From 52766bedb9316cd6cddacbb282809e3bdfba143e Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 29 Jan 2025 14:31:25 +0100 Subject: [PATCH 01/24] core/{.,state,vm},miner,eth/tracers,tests: implement 7709 with a syscall flag (#31036) Same as #31015 but requires the contract to exist. Not compatible with any verkle testnet up to now. This adds a `isSytemCall` flag so that it is possible to detect when a system call is executed, so that the code execution and other locations are not added to the witness. --------- Signed-off-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Co-authored-by: Ignacio Hagopian Co-authored-by: Felix Lange --- core/chain_makers.go | 23 +++++++++------- core/state/statedb_hooked.go | 4 +++ core/state_processor.go | 24 ++++++++++------ core/verkle_witness_test.go | 53 ++++++++++++++++++++---------------- core/vm/contract.go | 1 + core/vm/eips.go | 14 ++++++---- core/vm/evm.go | 9 ++++-- core/vm/gas_table.go | 4 +-- core/vm/interface.go | 3 ++ core/vm/interpreter.go | 2 +- core/vm/operations_verkle.go | 20 +++++++++++++- 11 files changed, 103 insertions(+), 54 deletions(-) diff --git a/core/chain_makers.go b/core/chain_makers.go index d52554a307..1f48f9d128 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -122,6 +122,11 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti if err != nil { panic(err) } + // Merge the tx-local access event into the "block-local" one, in order to collect + // all values, so that the witness can be built. + if b.statedb.GetTrie().IsVerkle() { + b.statedb.AccessEvents().Merge(evm.AccessEvents) + } b.txs = append(b.txs, tx) b.receipts = append(b.receipts, receipt) if b.header.BlobGasUsed != nil { @@ -379,7 +384,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse misc.ApplyDAOHardFork(statedb) } - if config.IsPrague(b.header.Number, b.header.Time) { + if config.IsPrague(b.header.Number, b.header.Time) || config.IsVerkle(b.header.Number, b.header.Time) { // EIP-2935 blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) blockContext.Random = &common.Hash{} // enable post-merge instruction set @@ -487,13 +492,11 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine // Save pre state for proof generation // preState := statedb.Copy() - // Pre-execution system calls. - if config.IsPrague(b.header.Number, b.header.Time) { - // EIP-2935 - blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) - evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) - ProcessParentBlockHash(b.header.ParentHash, evm) - } + // EIP-2935 / 7709 + blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) + blockContext.Random = &common.Hash{} // enable post-merge instruction set + evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) + ProcessParentBlockHash(b.header.ParentHash, evm) // Execute any user modifications to the block. if gen != nil { @@ -561,7 +564,7 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine return cm.chain, cm.receipts, proofs, keyvals } -func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { +func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (common.Hash, ethdb.Database, []*types.Block, []types.Receipts, []*verkle.VerkleProof, []verkle.StateDiff) { db := rawdb.NewMemoryDatabase() cacheConfig := DefaultCacheConfigWithScheme(rawdb.PathScheme) cacheConfig.SnapshotLimit = 0 @@ -572,7 +575,7 @@ func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n panic(err) } blocks, receipts, proofs, keyvals := GenerateVerkleChain(genesis.Config, genesisBlock, engine, db, triedb, n, gen) - return db, blocks, receipts, proofs, keyvals + return genesisBlock.Hash(), db, blocks, receipts, proofs, keyvals } func (cm *chainMaker) makeHeader(parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { diff --git a/core/state/statedb_hooked.go b/core/state/statedb_hooked.go index 31bdd06b46..25d823cc87 100644 --- a/core/state/statedb_hooked.go +++ b/core/state/statedb_hooked.go @@ -157,6 +157,10 @@ func (s *hookedStateDB) Witness() *stateless.Witness { return s.inner.Witness() } +func (s *hookedStateDB) AccessEvents() *AccessEvents { + return s.inner.AccessEvents() +} + func (s *hookedStateDB) SubBalance(addr common.Address, amount *uint256.Int, reason tracing.BalanceChangeReason) uint256.Int { prev := s.inner.SubBalance(addr, amount, reason) if s.hooks.OnBalanceChange != nil && !amount.IsZero() { diff --git a/core/state_processor.go b/core/state_processor.go index 3eb83a673a..902ff582e8 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -85,7 +85,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg if beaconRoot := block.BeaconRoot(); beaconRoot != nil { ProcessBeaconBlockRoot(*beaconRoot, evm) } - if p.config.IsPrague(block.Number(), block.Time()) { + if p.config.IsPrague(block.Number(), block.Time()) || p.config.IsVerkle(block.Number(), block.Time()) { ProcessParentBlockHash(block.ParentHash(), evm) } @@ -155,6 +155,12 @@ func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, } *usedGas += result.UsedGas + // Merge the tx-local access event into the "block-local" one, in order to collect + // all values, so that the witness can be built. + if statedb.GetTrie().IsVerkle() { + statedb.AccessEvents().Merge(evm.AccessEvents) + } + return MakeReceipt(evm, result, statedb, blockNumber, blockHash, tx, *usedGas, root), nil } @@ -181,12 +187,6 @@ func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, b receipt.ContractAddress = crypto.CreateAddress(evm.TxContext.Origin, tx.Nonce()) } - // Merge the tx-local access event into the "block-local" one, in order to collect - // all values, so that the witness can be built. - if statedb.GetTrie().IsVerkle() { - statedb.AccessEvents().Merge(evm.AccessEvents) - } - // Set the receipt logs and create the bloom filter. receipt.Logs = statedb.GetLogs(tx.Hash(), blockNumber.Uint64(), blockHash) receipt.Bloom = types.CreateBloom(types.Receipts{receipt}) @@ -234,7 +234,7 @@ func ProcessBeaconBlockRoot(beaconRoot common.Hash, evm *vm.EVM) { } // ProcessParentBlockHash stores the parent block hash in the history storage contract -// as per EIP-2935. +// as per EIP-2935/7709. func ProcessParentBlockHash(prevHash common.Hash, evm *vm.EVM) { if tracer := evm.Config.Tracer; tracer != nil { onSystemCallStart(tracer, evm.GetVMContext()) @@ -253,7 +253,13 @@ func ProcessParentBlockHash(prevHash common.Hash, evm *vm.EVM) { } evm.SetTxContext(NewEVMTxContext(msg)) evm.StateDB.AddAddressToAccessList(params.HistoryStorageAddress) - _, _, _ = evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) + _, _, err := evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) + if err != nil { + panic(err) + } + if evm.StateDB.AccessEvents() != nil { + evm.StateDB.AccessEvents().Merge(evm.AccessEvents) + } evm.StateDB.Finalise(true) } diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index 02e94963c4..763939735e 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -152,7 +152,7 @@ func TestProcessVerkle(t *testing.T) { txCost1*2 + txCost2, txCost1*2 + txCost2 + contractCreationCost + codeWithExtCodeCopyGas, } - _, chain, _, proofs, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { + _, _, chain, _, proofs, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { gen.SetPoS() // TODO need to check that the tx cost provided is the exact amount used (no remaining left-over) @@ -219,7 +219,7 @@ func TestProcessParentBlockHash(t *testing.T) { // block 1 parent hash is 0x0100.... // block 2 parent hash is 0x0200.... // etc - checkBlockHashes := func(statedb *state.StateDB) { + checkBlockHashes := func(statedb *state.StateDB, isVerkle bool) { statedb.SetNonce(params.HistoryStorageAddress, 1) statedb.SetCode(params.HistoryStorageAddress, params.HistoryStorageCode) // Process n blocks, from 1 .. num @@ -227,20 +227,24 @@ func TestProcessParentBlockHash(t *testing.T) { for i := 1; i <= num; i++ { header := &types.Header{ParentHash: common.Hash{byte(i)}, Number: big.NewInt(int64(i)), Difficulty: new(big.Int)} vmContext := NewEVMBlockContext(header, nil, new(common.Address)) - evm := vm.NewEVM(vmContext, statedb, params.MergedTestChainConfig, vm.Config{}) + chainConfig := params.MergedTestChainConfig + if isVerkle { + chainConfig = testVerkleChainConfig + } + evm := vm.NewEVM(vmContext, statedb, chainConfig, vm.Config{}) ProcessParentBlockHash(header.ParentHash, evm) } // Read block hashes for block 0 .. num-1 for i := 0; i < num; i++ { - have, want := getContractStoredBlockHash(statedb, uint64(i)), common.Hash{byte(i + 1)} + have, want := getContractStoredBlockHash(statedb, uint64(i), isVerkle), common.Hash{byte(i + 1)} if have != want { - t.Errorf("block %d, have parent hash %v, want %v", i, have, want) + t.Errorf("block %d, verkle=%v, have parent hash %v, want %v", i, isVerkle, have, want) } } } t.Run("MPT", func(t *testing.T) { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - checkBlockHashes(statedb) + checkBlockHashes(statedb, false) }) t.Run("Verkle", func(t *testing.T) { db := rawdb.NewMemoryDatabase() @@ -248,15 +252,18 @@ func TestProcessParentBlockHash(t *testing.T) { cacheConfig.SnapshotLimit = 0 triedb := triedb.NewDatabase(db, cacheConfig.triedbConfig(true)) statedb, _ := state.New(types.EmptyVerkleHash, state.NewDatabase(triedb, nil)) - checkBlockHashes(statedb) + checkBlockHashes(statedb, true) }) } // getContractStoredBlockHash is a utility method which reads the stored parent blockhash for block 'number' -func getContractStoredBlockHash(statedb *state.StateDB, number uint64) common.Hash { +func getContractStoredBlockHash(statedb *state.StateDB, number uint64, isVerkle bool) common.Hash { ringIndex := number % params.HistoryServeWindow var key common.Hash binary.BigEndian.PutUint64(key[24:], ringIndex) + if isVerkle { + return statedb.GetState(params.HistoryStorageAddress, key) + } return statedb.GetState(params.HistoryStorageAddress, key) } @@ -279,7 +286,7 @@ func TestProcessVerkleInvalidContractCreation(t *testing.T) { // // - The second block contains a single failing contract creation transaction, // that fails right off the bat. - _, chain, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { + genesisH, _, chain, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { gen.SetPoS() if i == 0 { @@ -364,8 +371,8 @@ func TestProcessVerkleInvalidContractCreation(t *testing.T) { if stemStateDiff.SuffixDiffs[0].NewValue == nil { t.Fatalf("nil new value in BLOCKHASH contract insert") } - if *stemStateDiff.SuffixDiffs[0].NewValue != chain[0].Hash() { - t.Fatalf("invalid BLOCKHASH value: %x != %x", *stemStateDiff.SuffixDiffs[0].NewValue, chain[0].Hash()) + if *stemStateDiff.SuffixDiffs[0].NewValue != genesisH { + t.Fatalf("invalid BLOCKHASH value: %x != %x", *stemStateDiff.SuffixDiffs[0].NewValue, genesisH) } } else { // For all other entries present in the witness, check that nothing beyond @@ -393,8 +400,8 @@ func TestProcessVerkleInvalidContractCreation(t *testing.T) { if stemStateDiff.SuffixDiffs[0].NewValue == nil { t.Fatalf("missing post state value for BLOCKHASH contract at block #2") } - if *stemStateDiff.SuffixDiffs[0].NewValue != common.HexToHash("0788c2c0f23aa07eb8bf76fe6c1ca9064a4821c1fd0af803913da488a58dba54") { - t.Fatalf("invalid post state value for BLOCKHASH contract at block #2: 0788c2c0f23aa07eb8bf76fe6c1ca9064a4821c1fd0af803913da488a58dba54 != %x", (*stemStateDiff.SuffixDiffs[0].NewValue)[:]) + if *stemStateDiff.SuffixDiffs[0].NewValue != chain[0].Hash() { + t.Fatalf("invalid post state value for BLOCKHASH contract at block #2: %x != %x", chain[0].Hash(), (*stemStateDiff.SuffixDiffs[0].NewValue)[:]) } } else if suffixDiff.Suffix > 4 { t.Fatalf("invalid suffix diff found for %x in block #2: %d\n", stemStateDiff.Stem, suffixDiff.Suffix) @@ -440,7 +447,7 @@ func TestProcessVerkleContractWithEmptyCode(t *testing.T) { config.ChainID.SetUint64(69421) gspec := verkleTestGenesis(&config) - _, chain, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { + genesisH, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { gen.SetPoS() var tx types.Transaction // a transaction that does some PUSH1n but returns a 0-sized contract @@ -472,8 +479,8 @@ func TestProcessVerkleContractWithEmptyCode(t *testing.T) { if stemStateDiff.SuffixDiffs[0].NewValue == nil { t.Fatalf("nil new value in BLOCKHASH contract insert") } - if *stemStateDiff.SuffixDiffs[0].NewValue != chain[0].Hash() { - t.Fatalf("invalid BLOCKHASH value: %x != %x", *stemStateDiff.SuffixDiffs[0].NewValue, chain[0].Hash()) + if *stemStateDiff.SuffixDiffs[0].NewValue != genesisH { + t.Fatalf("invalid BLOCKHASH value: %x != %x", *stemStateDiff.SuffixDiffs[0].NewValue, genesisH) } } else { for _, suffixDiff := range stemStateDiff.SuffixDiffs { @@ -532,7 +539,7 @@ func TestProcessVerkleExtCodeHashOpcode(t *testing.T) { } extCodeHashContractAddr := crypto.CreateAddress(deployer, 1) - _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { + _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { gen.SetPoS() if i == 0 { @@ -605,7 +612,7 @@ func TestProcessVerkleBalanceOpcode(t *testing.T) { account2 = common.HexToAddress("0x6177843db3138ae69679A54b95cf345ED759450d") gspec = verkleTestGenesis(&config) ) - _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { + _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { gen.SetPoS() txData := slices.Concat( []byte{byte(vm.PUSH20)}, @@ -686,7 +693,7 @@ func TestProcessVerkleSelfDestructInSeparateTx(t *testing.T) { deployer := crypto.PubkeyToAddress(testKey.PublicKey) contract := crypto.CreateAddress(deployer, 0) - _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { + _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { gen.SetPoS() if i == 0 { @@ -794,7 +801,7 @@ func TestProcessVerkleSelfDestructInSameTx(t *testing.T) { deployer := crypto.PubkeyToAddress(testKey.PublicKey) contract := crypto.CreateAddress(deployer, 0) - _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { + _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { gen.SetPoS() tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, Value: big.NewInt(42), @@ -897,7 +904,7 @@ func TestProcessVerkleSelfDestructInSeparateTxWithSelfBeneficiary(t *testing.T) deployer := crypto.PubkeyToAddress(testKey.PublicKey) contract := crypto.CreateAddress(deployer, 0) - _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { + _, _, _, _, _, statediffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 2, func(i int, gen *BlockGen) { gen.SetPoS() if i == 0 { // Create self-destruct contract, sending 42 wei. @@ -977,7 +984,7 @@ func TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiary(t *testing.T) { selfDestructContract := []byte{byte(vm.ADDRESS), byte(vm.SELFDESTRUCT)} - _, _, _, _, stateDiffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { + _, _, _, _, _, stateDiffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { gen.SetPoS() tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, Value: big.NewInt(42), @@ -1043,7 +1050,7 @@ func TestProcessVerkleSelfDestructInSameTxWithSelfBeneficiaryAndPrefundedAccount selfDestructContract := []byte{byte(vm.ADDRESS), byte(vm.SELFDESTRUCT)} - _, _, _, _, stateDiffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { + _, _, _, _, _, stateDiffs := GenerateVerkleChainWithGenesis(gspec, beacon.New(ethash.NewFaker()), 1, func(i int, gen *BlockGen) { gen.SetPoS() tx, _ := types.SignNewTx(testKey, signer, &types.LegacyTx{Nonce: 0, Value: big.NewInt(42), diff --git a/core/vm/contract.go b/core/vm/contract.go index cfda75b27e..5670244e17 100644 --- a/core/vm/contract.go +++ b/core/vm/contract.go @@ -59,6 +59,7 @@ type Contract struct { // is the execution frame represented by this object a contract deployment IsDeployment bool + IsSystemCall bool Gas uint64 value *uint256.Int diff --git a/core/vm/eips.go b/core/vm/eips.go index a51a18dc60..ab58702428 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -345,10 +345,12 @@ func opExtCodeCopyEIP4762(pc *uint64, interpreter *EVMInterpreter, scope *ScopeC self: AccountRef(addr), } paddedCodeCopy, copyOffset, nonPaddedCopyLength := getDataAndAdjustedBounds(code, uint64CodeOffset, length.Uint64()) - statelessGas := interpreter.evm.AccessEvents.CodeChunksRangeGas(addr, copyOffset, nonPaddedCopyLength, uint64(len(contract.Code)), false) - if !scope.Contract.UseGas(statelessGas, interpreter.evm.Config.Tracer, tracing.GasChangeUnspecified) { - scope.Contract.Gas = 0 - return nil, ErrOutOfGas + if !contract.IsSystemCall { + statelessGas := interpreter.evm.AccessEvents.CodeChunksRangeGas(addr, copyOffset, nonPaddedCopyLength, uint64(len(contract.Code)), false) + if !scope.Contract.UseGas(statelessGas, interpreter.evm.Config.Tracer, tracing.GasChangeUnspecified) { + scope.Contract.Gas = 0 + return nil, ErrOutOfGas + } } scope.Memory.Set(memOffset.Uint64(), length.Uint64(), paddedCodeCopy) @@ -367,7 +369,7 @@ func opPush1EIP4762(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext if *pc < codeLen { scope.Stack.push(integer.SetUint64(uint64(scope.Contract.Code[*pc]))) - if !scope.Contract.IsDeployment && *pc%31 == 0 { + if !scope.Contract.IsDeployment && !scope.Contract.IsSystemCall && *pc%31 == 0 { // touch next chunk if PUSH1 is at the boundary. if so, *pc has // advanced past this boundary. contractAddr := scope.Contract.Address() @@ -397,7 +399,7 @@ func makePushEIP4762(size uint64, pushByteSize int) executionFunc { )), ) - if !scope.Contract.IsDeployment { + if !scope.Contract.IsDeployment && !scope.Contract.IsSystemCall { contractAddr := scope.Contract.Address() statelessGas := interpreter.evm.AccessEvents.CodeChunksRangeGas(contractAddr, uint64(start), uint64(pushByteSize), uint64(len(scope.Contract.Code)), false) if !scope.Contract.UseGas(statelessGas, interpreter.evm.Config.Tracer, tracing.GasChangeUnspecified) { diff --git a/core/vm/evm.go b/core/vm/evm.go index 1e52215b7e..3f9ae621a3 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -165,8 +165,12 @@ func (evm *EVM) Interpreter() *EVMInterpreter { return evm.interpreter } +func isSystemCall(caller ContractRef) bool { + return caller.Address() == params.SystemAddress +} + // Call executes the contract associated with the addr with the given input as -// parameters. It also handles any necessary value transfer required and takes +// parameters. It also handles any necessary value transfer required and takse // the necessary steps to create accounts and reverses the state in case of an // execution error or failed value transfer. func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas uint64, value *uint256.Int) (ret []byte, leftOverGas uint64, err error) { @@ -189,7 +193,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas p, isPrecompile := evm.precompile(addr) if !evm.StateDB.Exist(addr) { - if !isPrecompile && evm.chainRules.IsEIP4762 { + if !isPrecompile && evm.chainRules.IsEIP4762 && !isSystemCall(caller) { // add proof of absence to witness wgas := evm.AccessEvents.AddAccount(addr, false) if gas < wgas { @@ -220,6 +224,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas // If the account has no code, we can abort here // The depth-check is already done, and precompiles handled above contract := NewContract(caller, AccountRef(addrCopy), value, gas) + contract.IsSystemCall = isSystemCall(caller) contract.SetCallCode(&addrCopy, evm.resolveCodeHash(addrCopy), code) ret, err = evm.interpreter.Run(contract, input, false) gas = contract.Gas diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index f3a9808251..55855727b5 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -394,7 +394,7 @@ func gasCall(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize if gas, overflow = math.SafeAdd(gas, memoryGas); overflow { return 0, ErrGasUintOverflow } - if evm.chainRules.IsEIP4762 { + if evm.chainRules.IsEIP4762 && !contract.IsSystemCall { if transfersValue { gas, overflow = math.SafeAdd(gas, evm.AccessEvents.ValueTransferGas(contract.Address(), address)) if overflow { @@ -428,7 +428,7 @@ func gasCallCode(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memory if gas, overflow = math.SafeAdd(gas, memoryGas); overflow { return 0, ErrGasUintOverflow } - if evm.chainRules.IsEIP4762 { + if evm.chainRules.IsEIP4762 && !contract.IsSystemCall { address := common.Address(stack.Back(1).Bytes20()) transfersValue := !stack.Back(2).IsZero() if transfersValue { diff --git a/core/vm/interface.go b/core/vm/interface.go index 011541dde3..3488526fc4 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -20,6 +20,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" @@ -98,6 +99,8 @@ type StateDB interface { Witness() *stateless.Witness + AccessEvents() *state.AccessEvents + // Finalise must be invoked at the end of a transaction Finalise(bool) } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 996ed6e56a..e3237936a0 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -233,7 +233,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( logged, pcCopy, gasCopy = false, pc, contract.Gas } - if in.evm.chainRules.IsEIP4762 && !contract.IsDeployment { + if in.evm.chainRules.IsEIP4762 && !contract.IsDeployment && !contract.IsSystemCall { // if the PC ends up in a new "chunk" of verkleized code, charge the // associated costs. contractAddr := contract.Address() diff --git a/core/vm/operations_verkle.go b/core/vm/operations_verkle.go index 3492994778..751761a911 100644 --- a/core/vm/operations_verkle.go +++ b/core/vm/operations_verkle.go @@ -41,6 +41,9 @@ func gasSLoad4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memor } func gasBalance4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + if contract.IsSystemCall { + return 0, nil + } address := stack.peek().Bytes20() gas := evm.AccessEvents.BasicDataGas(address, false) if gas == 0 { @@ -54,6 +57,9 @@ func gasExtCodeSize4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, if _, isPrecompile := evm.precompile(address); isPrecompile { return 0, nil } + if contract.IsSystemCall { + return 0, nil + } gas := evm.AccessEvents.BasicDataGas(address, false) if gas == 0 { gas = params.WarmStorageReadCostEIP2929 @@ -62,6 +68,9 @@ func gasExtCodeSize4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, } func gasExtCodeHash4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { + if contract.IsSystemCall { + return 0, nil + } address := stack.peek().Bytes20() if _, isPrecompile := evm.precompile(address); isPrecompile { return 0, nil @@ -79,6 +88,9 @@ func makeCallVariantGasEIP4762(oldCalculator gasFunc) gasFunc { if err != nil { return 0, err } + if contract.IsSystemCall { + return gas, nil + } if _, isPrecompile := evm.precompile(contract.Address()); isPrecompile { return gas, nil } @@ -102,6 +114,9 @@ func gasSelfdestructEIP4762(evm *EVM, contract *Contract, stack *Stack, mem *Mem if _, isPrecompile := evm.precompile(beneficiaryAddr); isPrecompile { return 0, nil } + if contract.IsSystemCall { + return 0, nil + } contractAddr := contract.Address() statelessGas := evm.AccessEvents.BasicDataGas(contractAddr, false) if contractAddr != beneficiaryAddr { @@ -131,7 +146,7 @@ func gasCodeCopyEip4762(evm *EVM, contract *Contract, stack *Stack, mem *Memory, uint64CodeOffset = gomath.MaxUint64 } _, copyOffset, nonPaddedCopyLength := getDataAndAdjustedBounds(contract.Code, uint64CodeOffset, length.Uint64()) - if !contract.IsDeployment { + if !contract.IsDeployment && !contract.IsSystemCall { gas += evm.AccessEvents.CodeChunksRangeGas(contract.Address(), copyOffset, nonPaddedCopyLength, uint64(len(contract.Code)), false) } return gas, nil @@ -143,6 +158,9 @@ func gasExtCodeCopyEIP4762(evm *EVM, contract *Contract, stack *Stack, mem *Memo if err != nil { return 0, err } + if contract.IsSystemCall { + return gas, nil + } addr := common.Address(stack.peek().Bytes20()) wgas := evm.AccessEvents.BasicDataGas(addr, false) if wgas == 0 { From 11e841e5992d6ffd5b59ed8219bce50bddbfdbb6 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Thu, 30 Jan 2025 10:58:44 +0100 Subject: [PATCH 02/24] build: bump test timeout (#31095) Travis often fails because the test times out. --- build/ci.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/ci.go b/build/ci.go index fd51b90d17..4a83fb9a88 100644 --- a/build/ci.go +++ b/build/ci.go @@ -298,8 +298,8 @@ func doTest(cmdline []string) { } gotest := tc.Go("test") - // CI needs a bit more time for the statetests (default 10m). - gotest.Args = append(gotest.Args, "-timeout=30m") + // CI needs a bit more time for the statetests (default 45m). + gotest.Args = append(gotest.Args, "-timeout=45m") // Enable CKZG backend in CI. gotest.Args = append(gotest.Args, "-tags=ckzg") From 7b96ec4dae8d4ddeffc761c7757c12e2d2b8bf74 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 30 Jan 2025 11:10:45 +0100 Subject: [PATCH 03/24] .travis.yml: change arch for Docker build to arm64 (#31096) This is an attempt to work around a gcc issue in the Docker build. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2ba3af9419..0353aa2e4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ jobs: - stage: build if: type = push os: linux - arch: amd64 + arch: arm64 dist: focal go: 1.23.x env: From d80da7c3b418d092a6a88c35a2bce341fdc23bb1 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 30 Jan 2025 11:21:23 +0100 Subject: [PATCH 04/24] Revert ".travis.yml: change arch for Docker build to arm64 (#31096)" This reverts commit 7b96ec4dae8d4ddeffc761c7757c12e2d2b8bf74. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0353aa2e4a..2ba3af9419 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ jobs: - stage: build if: type = push os: linux - arch: arm64 + arch: amd64 dist: focal go: 1.23.x env: From 7d0e197def6d50eebb8116e7f2b52f95a90c0aef Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 30 Jan 2025 14:06:50 +0100 Subject: [PATCH 05/24] build: retry PPA upload up to three times (#31099) --- build/ci.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/build/ci.go b/build/ci.go index 4a83fb9a88..de47e8e541 100644 --- a/build/ci.go +++ b/build/ci.go @@ -883,11 +883,17 @@ func ppaUpload(workdir, ppa, sshUser string, files []string) { os.WriteFile(idfile, sshkey, 0600) } } - // Upload + // Upload. This doesn't always work, so try up to three times. dest := sshUser + "@ppa.launchpad.net" - if err := build.UploadSFTP(idfile, dest, incomingDir, files); err != nil { - log.Fatal(err) + for i := 0; i < 3; i++ { + err := build.UploadSFTP(idfile, dest, incomingDir, files) + if err == nil { + return + } + log.Println("PPA upload failed:", err) + time.Sleep(5 * time.Second) } + log.Fatal("PPA upload failed all attempts.") } func getenvBase64(variable string) []byte { From fa9a2ff8687ec9efe57b4b9833d5590d20f8a83f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 30 Jan 2025 14:10:36 +0100 Subject: [PATCH 06/24] crypto: add IsOnCurve check (#31100) --- crypto/crypto.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crypto/crypto.go b/crypto/crypto.go index aaa5cc43a2..f9979aa4bb 100644 --- a/crypto/crypto.go +++ b/crypto/crypto.go @@ -178,6 +178,9 @@ func UnmarshalPubkey(pub []byte) (*ecdsa.PublicKey, error) { if x == nil { return nil, errInvalidPubkey } + if !S256().IsOnCurve(x, y) { + return nil, errInvalidPubkey + } return &ecdsa.PublicKey{Curve: S256(), X: x, Y: y}, nil } From b3833e5d980b7cca31988f41efdbb9ee7d7f4d15 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 30 Jan 2025 14:52:02 +0100 Subject: [PATCH 07/24] build: provide a flag to disable publishing in dockerx build (#31098) This changes the `-upload` flag to just toggle the upload. The remote image name is now configured using the `-hub` flag. --- .travis.yml | 2 +- build/ci.go | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2ba3af9419..8953450e84 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ jobs: before_install: - export DOCKER_CLI_EXPERIMENTAL=enabled script: - - go run build/ci.go dockerx -platform "linux/amd64,linux/arm64,linux/riscv64" -upload ethereum/client-go + - go run build/ci.go dockerx -platform "linux/amd64,linux/arm64,linux/riscv64" -hub ethereum/client-go -upload # This builder does the Linux Azure uploads - stage: build diff --git a/build/ci.go b/build/ci.go index de47e8e541..02a4d771f6 100644 --- a/build/ci.go +++ b/build/ci.go @@ -684,7 +684,8 @@ func maybeSkipArchive(env build.Environment) { func doDockerBuildx(cmdline []string) { var ( platform = flag.String("platform", "", `Push a multi-arch docker image for the specified architectures (usually "linux/amd64,linux/arm64")`) - upload = flag.String("upload", "", `Where to upload the docker image (usually "ethereum/client-go")`) + hubImage = flag.String("hub", "ethereum/client-go", `Where to upload the docker image`) + upload = flag.Bool("upload", false, `Whether to trigger upload`) ) flag.CommandLine.Parse(cmdline) @@ -719,25 +720,33 @@ func doDockerBuildx(cmdline []string) { tags = []string{"stable", fmt.Sprintf("release-%v", version.Family), "v" + version.Semantic} } // Need to create a mult-arch builder - build.MustRunCommand("docker", "buildx", "create", "--use", "--name", "multi-arch-builder", "--platform", *platform) + check := exec.Command("docker", "buildx", "inspect", "multi-arch-builder") + if check.Run() != nil { + build.MustRunCommand("docker", "buildx", "create", "--use", "--name", "multi-arch-builder", "--platform", *platform) + } for _, spec := range []struct { file string base string }{ - {file: "Dockerfile", base: fmt.Sprintf("%s:", *upload)}, - {file: "Dockerfile.alltools", base: fmt.Sprintf("%s:alltools-", *upload)}, + {file: "Dockerfile", base: fmt.Sprintf("%s:", *hubImage)}, + {file: "Dockerfile.alltools", base: fmt.Sprintf("%s:alltools-", *hubImage)}, } { for _, tag := range tags { // latest, stable etc gethImage := fmt.Sprintf("%s%s", spec.base, tag) - build.MustRunCommand("docker", "buildx", "build", + cmd := exec.Command("docker", "buildx", "build", "--build-arg", "COMMIT="+env.Commit, "--build-arg", "VERSION="+version.WithMeta, "--build-arg", "BUILDNUM="+env.Buildnum, "--tag", gethImage, "--platform", *platform, - "--push", - "--file", spec.file, ".") + "--file", spec.file, + ) + if *upload { + cmd.Args = append(cmd.Args, "--push") + } + cmd.Args = append(cmd.Args, ".") + build.MustRun(cmd) } } } From 9bc8256a09d3e3b30371460f6d1195bbef60c71c Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 30 Jan 2025 15:00:50 +0100 Subject: [PATCH 08/24] version: begin v1.15.0 release cycle --- version/version.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/version/version.go b/version/version.go index 52dfa0281e..01bc97df51 100644 --- a/version/version.go +++ b/version/version.go @@ -18,7 +18,7 @@ package version const ( Major = 1 // Major version component of the current release - Minor = 14 // Minor version component of the current release - Patch = 13 // Patch version component of the current release + Minor = 15 // Minor version component of the current release + Patch = 0 // Patch version component of the current release Meta = "unstable" // Version metadata to append to the version string ) From 9b9e7ccacffbc6f1db9c0a3b42bdba37109fea48 Mon Sep 17 00:00:00 2001 From: Ryan Tinianov Date: Thu, 30 Jan 2025 10:58:53 -0500 Subject: [PATCH 09/24] all: add build tags for wasip1 (#31090) --- cmd/utils/diskusage.go | 4 +-- crypto/kzg4844/kzg4844_ckzg_cgo.go | 2 +- crypto/kzg4844/kzg4844_ckzg_nocgo.go | 2 +- crypto/signature_cgo.go | 4 +-- crypto/signature_nocgo.go | 4 +-- ethdb/leveldb/leveldb.go | 4 +-- metrics/cpu_disabled.go | 4 +-- metrics/cpu_enabled.go | 4 +-- metrics/cputime_unix.go | 4 +-- rpc/ipc_wasip1.go | 38 ++++++++++++++++++++++++++++ 10 files changed, 54 insertions(+), 16 deletions(-) create mode 100644 rpc/ipc_wasip1.go diff --git a/cmd/utils/diskusage.go b/cmd/utils/diskusage.go index 0e88f91944..8174236ab7 100644 --- a/cmd/utils/diskusage.go +++ b/cmd/utils/diskusage.go @@ -14,8 +14,8 @@ // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . -//go:build !windows && !openbsd -// +build !windows,!openbsd +//go:build !windows && !openbsd && !wasip1 +// +build !windows,!openbsd,!wasip1 package utils diff --git a/crypto/kzg4844/kzg4844_ckzg_cgo.go b/crypto/kzg4844/kzg4844_ckzg_cgo.go index 11bc451b58..dce592b444 100644 --- a/crypto/kzg4844/kzg4844_ckzg_cgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_cgo.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build ckzg && !nacl && !js && cgo && !gofuzz +//go:build ckzg && !nacl && !js && !wasip1 && cgo && !gofuzz package kzg4844 diff --git a/crypto/kzg4844/kzg4844_ckzg_nocgo.go b/crypto/kzg4844/kzg4844_ckzg_nocgo.go index 70a78e80d1..0662b2632f 100644 --- a/crypto/kzg4844/kzg4844_ckzg_nocgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_nocgo.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build !ckzg || nacl || js || !cgo || gofuzz +//go:build !ckzg || nacl || js || wasip1 || !cgo || gofuzz package kzg4844 diff --git a/crypto/signature_cgo.go b/crypto/signature_cgo.go index 87289253c0..5c7810c14c 100644 --- a/crypto/signature_cgo.go +++ b/crypto/signature_cgo.go @@ -14,8 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build !nacl && !js && cgo && !gofuzz -// +build !nacl,!js,cgo,!gofuzz +//go:build !nacl && !js && !wasip1 && cgo && !gofuzz +// +build !nacl,!js,!wasip1,cgo,!gofuzz package crypto diff --git a/crypto/signature_nocgo.go b/crypto/signature_nocgo.go index 16a785a186..2d35b49403 100644 --- a/crypto/signature_nocgo.go +++ b/crypto/signature_nocgo.go @@ -14,8 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build nacl || js || !cgo || gofuzz -// +build nacl js !cgo gofuzz +//go:build nacl || js || wasip1 || !cgo || gofuzz +// +build nacl js wasip1 !cgo gofuzz package crypto diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index f18503c941..1d88ed460b 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -14,8 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build !js -// +build !js +//go:build !js && !wasip1 +// +build !js,!wasip1 // Package leveldb implements the key-value database layer based on LevelDB. package leveldb diff --git a/metrics/cpu_disabled.go b/metrics/cpu_disabled.go index 025d97aeb3..8404b0edd4 100644 --- a/metrics/cpu_disabled.go +++ b/metrics/cpu_disabled.go @@ -14,8 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build ios || js -// +build ios js +//go:build ios || js || wasip1 +// +build ios js wasip1 package metrics diff --git a/metrics/cpu_enabled.go b/metrics/cpu_enabled.go index 2359028a21..838c32fad8 100644 --- a/metrics/cpu_enabled.go +++ b/metrics/cpu_enabled.go @@ -14,8 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build !ios && !js -// +build !ios,!js +//go:build !ios && !js && !wasip1 +// +build !ios,!js,!wasip1 package metrics diff --git a/metrics/cputime_unix.go b/metrics/cputime_unix.go index ad4f812fd2..a44bf80876 100644 --- a/metrics/cputime_unix.go +++ b/metrics/cputime_unix.go @@ -14,8 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//go:build !windows && !js -// +build !windows,!js +//go:build !windows && !js && !wasip1 +// +build !windows,!js,!wasip1 package metrics diff --git a/rpc/ipc_wasip1.go b/rpc/ipc_wasip1.go new file mode 100644 index 0000000000..2608e29222 --- /dev/null +++ b/rpc/ipc_wasip1.go @@ -0,0 +1,38 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +//go:build wasip1 +// +build wasip1 + +package rpc + +import ( + "context" + "errors" + "net" +) + +var errNotSupported = errors.New("rpc: not supported") + +// ipcListen will create a named pipe on the given endpoint. +func ipcListen(endpoint string) (net.Listener, error) { + return nil, errNotSupported +} + +// newIPCConnection will connect to a named pipe with the given endpoint as name. +func newIPCConnection(ctx context.Context, endpoint string) (net.Conn, error) { + return nil, errNotSupported +} From 0e1a19da76d1761fdb3db78112a9d0e3571d43a4 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Thu, 30 Jan 2025 17:07:45 +0100 Subject: [PATCH 10/24] core: implement eip-7623 floor data gas (#30946) This PR builds on #29040 and updates it to the new version of the spec. I filled the EEST tests and they pass. Link to spec: https://eips.ethereum.org/EIPS/eip-7623 --------- Co-authored-by: Marius van der Wijden Co-authored-by: lightclient <14004106+lightclient@users.noreply.github.com> Co-authored-by: lightclient --- cmd/evm/internal/t8ntool/transaction.go | 26 ++++-- core/error.go | 4 + core/state_processor_test.go | 4 +- core/state_transition.go | 82 +++++++++++++------ core/tracing/hooks.go | 3 + core/txpool/validation.go | 10 +++ core/verkle_witness_test.go | 2 +- ...h_getBlockReceipts-block-with-blob-tx.json | 8 +- ...ockReceipts-block-with-dynamic-fee-tx.json | 8 +- .../eth_getBlockReceipts-tag-latest.json | 8 +- .../eth_getTransactionReceipt-blob-tx.json | 8 +- ...eipt-create-contract-with-access-list.json | 8 +- ...ansactionReceipt-dynamic-tx-with-logs.json | 8 +- params/protocol_params.go | 2 + 14 files changed, 125 insertions(+), 56 deletions(-) diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index d8650beca6..2bc4f73b60 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -133,15 +133,29 @@ func Transaction(ctx *cli.Context) error { r.Address = sender } // Check intrinsic gas - if gas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.SetCodeAuthorizations(), tx.To() == nil, - chainConfig.IsHomestead(new(big.Int)), chainConfig.IsIstanbul(new(big.Int)), chainConfig.IsShanghai(new(big.Int), 0)); err != nil { + rules := chainConfig.Rules(common.Big0, true, 0) + gas, err := core.IntrinsicGas(tx.Data(), tx.AccessList(), tx.SetCodeAuthorizations(), tx.To() == nil, rules.IsHomestead, rules.IsIstanbul, rules.IsShanghai) + if err != nil { r.Error = err results = append(results, r) continue - } else { - r.IntrinsicGas = gas - if tx.Gas() < gas { - r.Error = fmt.Errorf("%w: have %d, want %d", core.ErrIntrinsicGas, tx.Gas(), gas) + } + r.IntrinsicGas = gas + if tx.Gas() < gas { + r.Error = fmt.Errorf("%w: have %d, want %d", core.ErrIntrinsicGas, tx.Gas(), gas) + results = append(results, r) + continue + } + // For Prague txs, validate the floor data gas. + if rules.IsPrague { + floorDataGas, err := core.FloorDataGas(tx.Data()) + if err != nil { + r.Error = err + results = append(results, r) + continue + } + if tx.Gas() < floorDataGas { + r.Error = fmt.Errorf("%w: have %d, want %d", core.ErrFloorDataGas, tx.Gas(), floorDataGas) results = append(results, r) continue } diff --git a/core/error.go b/core/error.go index 82f7ddcf5d..ce3bbb7888 100644 --- a/core/error.go +++ b/core/error.go @@ -80,6 +80,10 @@ var ( // than required to start the invocation. ErrIntrinsicGas = errors.New("intrinsic gas too low") + // ErrFloorDataGas is returned if the transaction is specified to use less gas + // than required for the data floor cost. + ErrFloorDataGas = errors.New("insufficient gas for floor data gas cost") + // ErrTxTypeNotSupported is returned if a transaction is not supported in the // current network configuration. ErrTxTypeNotSupported = types.ErrTxTypeNotSupported diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 2f841eb64a..750f3708dc 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -251,9 +251,9 @@ func TestStateProcessorErrors(t *testing.T) { }, { // ErrMaxInitCodeSizeExceeded txs: []*types.Transaction{ - mkDynamicCreationTx(0, 500000, common.Big0, big.NewInt(params.InitialBaseFee), tooBigInitCode[:]), + mkDynamicCreationTx(0, 520000, common.Big0, big.NewInt(params.InitialBaseFee), tooBigInitCode[:]), }, - want: "could not apply tx 0 [0xd491405f06c92d118dd3208376fcee18a57c54bc52063ee4a26b1cf296857c25]: max initcode size exceeded: code size 49153 limit 49152", + want: "could not apply tx 0 [0x3a30404d42d6ccc843d7c391fd0c87b9b9795a0c174261b46d2ac95ca17b81cd]: max initcode size exceeded: code size 49153 limit 49152", }, { // ErrIntrinsicGas: Not enough gas to cover init code txs: []*types.Transaction{ diff --git a/core/state_transition.go b/core/state_transition.go index b6203e6aae..e9c88eaedf 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -17,6 +17,7 @@ package core import ( + "bytes" "fmt" "math" "math/big" @@ -79,12 +80,9 @@ func IntrinsicGas(data []byte, accessList types.AccessList, authList []types.Set // Bump the required gas by the amount of transactional data if dataLen > 0 { // Zero and non-zero bytes are priced differently - var nz uint64 - for _, byt := range data { - if byt != 0 { - nz++ - } - } + z := uint64(bytes.Count(data, []byte{0})) + nz := dataLen - z + // Make sure we don't exceed uint64 for all data combinations nonZeroGas := params.TxDataNonZeroGasFrontier if isEIP2028 { @@ -95,7 +93,6 @@ func IntrinsicGas(data []byte, accessList types.AccessList, authList []types.Set } gas += nz * nonZeroGas - z := dataLen - nz if (math.MaxUint64-gas)/params.TxDataZeroGas < z { return 0, ErrGasUintOverflow } @@ -119,6 +116,21 @@ func IntrinsicGas(data []byte, accessList types.AccessList, authList []types.Set return gas, nil } +// FloorDataGas computes the minimum gas required for a transaction based on its data tokens (EIP-7623). +func FloorDataGas(data []byte) (uint64, error) { + var ( + z = uint64(bytes.Count(data, []byte{0})) + nz = uint64(len(data)) - z + tokens = nz*params.TxTokenPerNonZeroByte + z + ) + // Check for overflow + if (math.MaxUint64-params.TxGas)/params.TxCostFloorPerToken < tokens { + return 0, ErrGasUintOverflow + } + // Minimum gas required for a transaction based on its data tokens (EIP-7623). + return params.TxGas + tokens*params.TxCostFloorPerToken, nil +} + // toWordSize returns the ceiled word size required for init code payment calculation. func toWordSize(size uint64) uint64 { if size > math.MaxUint64-31 { @@ -414,6 +426,7 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { sender = vm.AccountRef(msg.From) rules = st.evm.ChainConfig().Rules(st.evm.Context.BlockNumber, st.evm.Context.Random != nil, st.evm.Context.Time) contractCreation = msg.To == nil + floorDataGas uint64 ) // Check clauses 4-5, subtract intrinsic gas if everything is correct @@ -424,6 +437,16 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { if st.gasRemaining < gas { return nil, fmt.Errorf("%w: have %d, want %d", ErrIntrinsicGas, st.gasRemaining, gas) } + // Gas limit suffices for the floor data cost (EIP-7623) + if rules.IsPrague { + floorDataGas, err = FloorDataGas(msg.Data) + if err != nil { + return nil, err + } + if msg.GasLimit < floorDataGas { + return nil, fmt.Errorf("%w: have %d, want %d", ErrFloorDataGas, msg.GasLimit, floorDataGas) + } + } if t := st.evm.Config.Tracer; t != nil && t.OnGasChange != nil { t.OnGasChange(st.gasRemaining, st.gasRemaining-gas, tracing.GasChangeTxIntrinsicGas) } @@ -487,14 +510,21 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { ret, st.gasRemaining, vmerr = st.evm.Call(sender, st.to(), msg.Data, st.gasRemaining, value) } - var gasRefund uint64 - if !rules.IsLondon { - // Before EIP-3529: refunds were capped to gasUsed / 2 - gasRefund = st.refundGas(params.RefundQuotient) - } else { - // After EIP-3529: refunds are capped to gasUsed / 5 - gasRefund = st.refundGas(params.RefundQuotientEIP3529) + // Compute refund counter, capped to a refund quotient. + gasRefund := st.calcRefund() + st.gasRemaining += gasRefund + if rules.IsPrague { + // After EIP-7623: Data-heavy transactions pay the floor gas. + if st.gasUsed() < floorDataGas { + prev := st.gasRemaining + st.gasRemaining = st.initialGas - floorDataGas + if t := st.evm.Config.Tracer; t != nil && t.OnGasChange != nil { + t.OnGasChange(prev, st.gasRemaining, tracing.GasChangeTxDataFloor) + } + } } + st.returnGas() + effectiveTip := msg.GasPrice if rules.IsLondon { effectiveTip = new(big.Int).Sub(msg.GasFeeCap, st.evm.Context.BaseFee) @@ -585,20 +615,28 @@ func (st *stateTransition) applyAuthorization(auth *types.SetCodeAuthorization) return nil } -func (st *stateTransition) refundGas(refundQuotient uint64) uint64 { - // Apply refund counter, capped to a refund quotient - refund := st.gasUsed() / refundQuotient +// calcRefund computes refund counter, capped to a refund quotient. +func (st *stateTransition) calcRefund() uint64 { + var refund uint64 + if !st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { + // Before EIP-3529: refunds were capped to gasUsed / 2 + refund = st.gasUsed() / params.RefundQuotient + } else { + // After EIP-3529: refunds are capped to gasUsed / 5 + refund = st.gasUsed() / params.RefundQuotientEIP3529 + } if refund > st.state.GetRefund() { refund = st.state.GetRefund() } - if st.evm.Config.Tracer != nil && st.evm.Config.Tracer.OnGasChange != nil && refund > 0 { st.evm.Config.Tracer.OnGasChange(st.gasRemaining, st.gasRemaining+refund, tracing.GasChangeTxRefunds) } + return refund +} - st.gasRemaining += refund - - // Return ETH for remaining gas, exchanged at the original rate. +// returnGas returns ETH for remaining gas, +// exchanged at the original rate. +func (st *stateTransition) returnGas() { remaining := uint256.NewInt(st.gasRemaining) remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice)) st.state.AddBalance(st.msg.From, remaining, tracing.BalanceIncreaseGasReturn) @@ -610,8 +648,6 @@ func (st *stateTransition) refundGas(refundQuotient uint64) uint64 { // Also return remaining gas to the block gas counter so it is // available for the next transaction. st.gp.AddGas(st.gasRemaining) - - return refund } // gasUsed returns the amount of gas used up by the state transition. diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go index da364444ed..167bcb5c16 100644 --- a/core/tracing/hooks.go +++ b/core/tracing/hooks.go @@ -313,6 +313,9 @@ const ( GasChangeWitnessCodeChunk GasChangeReason = 17 // GasChangeWitnessContractCollisionCheck flags the event of adding to the witness when checking for contract address collision. GasChangeWitnessContractCollisionCheck GasChangeReason = 18 + // GasChangeTxDataFloor is the amount of extra gas the transaction has to pay to reach the minimum gas requirement for the + // transaction data. This change will always be a negative change. + GasChangeTxDataFloor GasChangeReason = 19 // GasChangeIgnored is a special value that can be used to indicate that the gas change should be ignored as // it will be "manually" tracked by a direct emit of the gas change event. diff --git a/core/txpool/validation.go b/core/txpool/validation.go index 412418dcc9..dec711b173 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -115,6 +115,16 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types if tx.Gas() < intrGas { return fmt.Errorf("%w: gas %v, minimum needed %v", core.ErrIntrinsicGas, tx.Gas(), intrGas) } + // Ensure the transaction can cover floor data gas. + if opts.Config.IsPrague(head.Number, head.Time) { + floorDataGas, err := core.FloorDataGas(tx.Data()) + if err != nil { + return err + } + if tx.Gas() < floorDataGas { + return fmt.Errorf("%w: gas %v, minimum needed %v", core.ErrFloorDataGas, tx.Gas(), floorDataGas) + } + } // Ensure the gasprice is high enough to cover the requirement of the calling pool if tx.GasTipCapIntCmp(opts.MinTip) < 0 { return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrUnderpriced, tx.GasTipCap(), opts.MinTip) diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index 763939735e..500fb37156 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -147,7 +147,7 @@ func TestProcessVerkle(t *testing.T) { params.WitnessChunkWriteCost + /* SSTORE in constructor */ params.WitnessChunkReadCost + params.WitnessChunkWriteCost + /* write code hash for tx creation */ 15*(params.WitnessChunkReadCost+params.WitnessChunkWriteCost) + /* code chunks #0..#14 */ - 4844 /* execution costs */ + uint64(4844) /* execution costs */ blockGasUsagesExpected := []uint64{ txCost1*2 + txCost2, txCost1*2 + txCost2 + contractCreationCost + codeWithExtCodeCopyGas, diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json index df2d9349fb..5853724cd6 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json @@ -2,19 +2,19 @@ { "blobGasPrice": "0x1", "blobGasUsed": "0x20000", - "blockHash": "0x17124e31fb075a301b1d7d4135683b0a09fe4e6d453c54e2e734d5ee00744a49", + "blockHash": "0x5f58514bcb3b216908f0aff6ced44666c3aa250df06093150ac850a7a7850f3c", "blockNumber": "0x6", "contractAddress": null, "cumulativeGasUsed": "0x5208", - "effectiveGasPrice": "0x1b09d63b", + "effectiveGasPrice": "0x1b0a08c4", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gasUsed": "0x5208", "logs": [], "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "status": "0x1", "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", - "transactionHash": "0xb51ee3d2a89ba5d5623c73133c8d7a6ba9fb41194c17f4302c21b30994a1180f", + "transactionHash": "0x80348f994fb5f3b05bd2e5f58bbdc73485e449c028612a2c0680f9ac6ff70add", "transactionIndex": "0x0", "type": "0x3" } -] \ No newline at end of file +] diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json index b01400e605..602b25e21f 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json @@ -1,12 +1,12 @@ [ { - "blockHash": "0x102e50de30318ee99a03a09db74387e79cad3165bf6840cc84249806a2a302f3", + "blockHash": "0x47cd44027bb55856a175e36be0396bad221e52172529d9c1bf12bf5424a041ae", "blockNumber": "0x4", "contractAddress": null, - "cumulativeGasUsed": "0x538d", + "cumulativeGasUsed": "0x5564", "effectiveGasPrice": "0x2325c42f", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", - "gasUsed": "0x538d", + "gasUsed": "0x5564", "logs": [], "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "status": "0x0", @@ -15,4 +15,4 @@ "transactionIndex": "0x0", "type": "0x2" } -] \ No newline at end of file +] diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json b/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json index df2d9349fb..5853724cd6 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json @@ -2,19 +2,19 @@ { "blobGasPrice": "0x1", "blobGasUsed": "0x20000", - "blockHash": "0x17124e31fb075a301b1d7d4135683b0a09fe4e6d453c54e2e734d5ee00744a49", + "blockHash": "0x5f58514bcb3b216908f0aff6ced44666c3aa250df06093150ac850a7a7850f3c", "blockNumber": "0x6", "contractAddress": null, "cumulativeGasUsed": "0x5208", - "effectiveGasPrice": "0x1b09d63b", + "effectiveGasPrice": "0x1b0a08c4", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gasUsed": "0x5208", "logs": [], "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "status": "0x1", "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", - "transactionHash": "0xb51ee3d2a89ba5d5623c73133c8d7a6ba9fb41194c17f4302c21b30994a1180f", + "transactionHash": "0x80348f994fb5f3b05bd2e5f58bbdc73485e449c028612a2c0680f9ac6ff70add", "transactionIndex": "0x0", "type": "0x3" } -] \ No newline at end of file +] diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json b/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json index 8e0669d10a..386f1225e6 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json @@ -1,18 +1,18 @@ { "blobGasPrice": "0x1", "blobGasUsed": "0x20000", - "blockHash": "0x17124e31fb075a301b1d7d4135683b0a09fe4e6d453c54e2e734d5ee00744a49", + "blockHash": "0x5f58514bcb3b216908f0aff6ced44666c3aa250df06093150ac850a7a7850f3c", "blockNumber": "0x6", "contractAddress": null, "cumulativeGasUsed": "0x5208", - "effectiveGasPrice": "0x1b09d63b", + "effectiveGasPrice": "0x1b0a08c4", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gasUsed": "0x5208", "logs": [], "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "status": "0x1", "to": "0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e", - "transactionHash": "0xb51ee3d2a89ba5d5623c73133c8d7a6ba9fb41194c17f4302c21b30994a1180f", + "transactionHash": "0x80348f994fb5f3b05bd2e5f58bbdc73485e449c028612a2c0680f9ac6ff70add", "transactionIndex": "0x0", "type": "0x3" -} \ No newline at end of file +} diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json index 49c06aad62..0519ae1da4 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json @@ -1,16 +1,16 @@ { - "blockHash": "0x53bffe54375c0a31fe7bc0db7455db7d48278234c2400efa4d40d1c57cbe868d", + "blockHash": "0x5bff93f8f94ba7ee52bef1a80062b9fed22c6d1eebb2b0e87a4a003365a7bd66", "blockNumber": "0x5", "contractAddress": "0xfdaa97661a584d977b4d3abb5370766ff5b86a18", "cumulativeGasUsed": "0xe01c", - "effectiveGasPrice": "0x1ecb3fb4", + "effectiveGasPrice": "0x1ecb7942", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", "gasUsed": "0xe01c", "logs": [], "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "status": "0x1", "to": null, - "transactionHash": "0xb5a1148819cfdfff9bfe70035524fec940eb735d89b76960b97751d01ae2a9f2", + "transactionHash": "0x173dffc76966c72542560c376ce512a871e31b86988f9169bf021da0937640f9", "transactionIndex": "0x0", "type": "0x1" -} \ No newline at end of file +} diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json b/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json index 13bd7bd12c..4601273a51 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json @@ -1,11 +1,11 @@ { - "blockHash": "0x102e50de30318ee99a03a09db74387e79cad3165bf6840cc84249806a2a302f3", + "blockHash": "0x47cd44027bb55856a175e36be0396bad221e52172529d9c1bf12bf5424a041ae", "blockNumber": "0x4", "contractAddress": null, - "cumulativeGasUsed": "0x538d", + "cumulativeGasUsed": "0x5564", "effectiveGasPrice": "0x2325c42f", "from": "0x703c4b2bd70c169f5717101caee543299fc946c7", - "gasUsed": "0x538d", + "gasUsed": "0x5564", "logs": [], "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", "status": "0x0", @@ -13,4 +13,4 @@ "transactionHash": "0xdcde2574628c9d7dff22b9afa19f235959a924ceec65a9df903a517ae91f5c84", "transactionIndex": "0x0", "type": "0x2" -} \ No newline at end of file +} diff --git a/params/protocol_params.go b/params/protocol_params.go index f06346db84..a56eed5683 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -92,6 +92,8 @@ const ( TxDataNonZeroGasFrontier uint64 = 68 // Per byte of data attached to a transaction that is not equal to zero. NOTE: Not payable on data of calls between transactions. TxDataNonZeroGasEIP2028 uint64 = 16 // Per byte of non zero data attached to a transaction after EIP 2028 (part in Istanbul) + TxTokenPerNonZeroByte uint64 = 4 // Token cost per non-zero byte as specified by EIP-7623. + TxCostFloorPerToken uint64 = 10 // Cost floor per byte of data as specified by EIP-7623. TxAccessListAddressGas uint64 = 2400 // Per address specified in EIP 2930 access list TxAccessListStorageKeyGas uint64 = 1900 // Per storage key specified in EIP 2930 access list TxAuthTupleGas uint64 = 12500 // Per auth tuple code specified in EIP-7702 From a50cac5bbe1aeb6fdd74f02a83cea5a459bcab19 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Thu, 30 Jan 2025 17:08:42 +0100 Subject: [PATCH 11/24] core/vm: EXTCODE* return delegation designator for 7702 (#31089) Implements https://github.com/ethereum/EIPs/pull/9248 --- core/vm/eips.go | 59 ------------------------------------------------- 1 file changed, 59 deletions(-) diff --git a/core/vm/eips.go b/core/vm/eips.go index ab58702428..515999bd19 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -23,8 +23,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/tracing" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" ) @@ -709,65 +707,8 @@ func enableEOF(jt *JumpTable) { } } -// opExtCodeCopyEIP7702 implements the EIP-7702 variation of opExtCodeCopy. -func opExtCodeCopyEIP7702(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { - var ( - stack = scope.Stack - a = stack.pop() - memOffset = stack.pop() - codeOffset = stack.pop() - length = stack.pop() - ) - uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow() - if overflow { - uint64CodeOffset = math.MaxUint64 - } - code := interpreter.evm.StateDB.GetCode(common.Address(a.Bytes20())) - if _, ok := types.ParseDelegation(code); ok { - code = types.DelegationPrefix[:2] - } - codeCopy := getData(code, uint64CodeOffset, length.Uint64()) - scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) - - return nil, nil -} - -// opExtCodeSizeEIP7702 implements the EIP-7702 variation of opExtCodeSize. -func opExtCodeSizeEIP7702(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { - slot := scope.Stack.peek() - code := interpreter.evm.StateDB.GetCode(common.Address(slot.Bytes20())) - if _, ok := types.ParseDelegation(code); ok { - code = types.DelegationPrefix[:2] - } - slot.SetUint64(uint64(len(code))) - return nil, nil -} - -// opExtCodeHashEIP7702 implements the EIP-7702 variation of opExtCodeHash. -func opExtCodeHashEIP7702(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { - slot := scope.Stack.peek() - addr := common.Address(slot.Bytes20()) - if interpreter.evm.StateDB.Empty(addr) { - slot.Clear() - return nil, nil - } - code := interpreter.evm.StateDB.GetCode(addr) - if _, ok := types.ParseDelegation(code); ok { - // If the code is a delegation, return the prefix without version. - slot.SetBytes(crypto.Keccak256(types.DelegationPrefix[:2])) - } else { - // Otherwise, return normal code hash. - slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(addr).Bytes()) - } - return nil, nil -} - // enable7702 the EIP-7702 changes to support delegation designators. func enable7702(jt *JumpTable) { - jt[EXTCODECOPY].execute = opExtCodeCopyEIP7702 - jt[EXTCODESIZE].execute = opExtCodeSizeEIP7702 - jt[EXTCODEHASH].execute = opExtCodeHashEIP7702 - jt[CALL].dynamicGas = gasCallEIP7702 jt[CALLCODE].dynamicGas = gasCallCodeEIP7702 jt[STATICCALL].dynamicGas = gasStaticCallEIP7702 From 8daefeb890e827441914e69edf18c86d8c4398c7 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Fri, 31 Jan 2025 02:36:49 -0700 Subject: [PATCH 12/24] params: update system contract addresses for devnet-6 (#31102) Finalize Prague system contract addresses. Reference: * https://github.com/ethereum/EIPs/pull/9287 * https://github.com/ethereum/EIPs/pull/9288 * https://github.com/ethereum/EIPs/pull/9289 --- params/protocol_params.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/params/protocol_params.go b/params/protocol_params.go index a56eed5683..c9179634a5 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -205,14 +205,14 @@ var ( BeaconRootsCode = common.FromHex("3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500") // EIP-2935 - Serve historical block hashes from state - HistoryStorageAddress = common.HexToAddress("0x0F792be4B0c0cb4DAE440Ef133E90C0eCD48CCCC") + HistoryStorageAddress = common.HexToAddress("0x0000F90827F1C53a10cb7A02335B175320002935") HistoryStorageCode = common.FromHex("3373fffffffffffffffffffffffffffffffffffffffe14604657602036036042575f35600143038111604257611fff81430311604257611fff9006545f5260205ff35b5f5ffd5b5f35611fff60014303065500") // EIP-7002 - Execution layer triggerable withdrawals - WithdrawalQueueAddress = common.HexToAddress("0x0c15F14308530b7CDB8460094BbB9cC28b9AaaAA") + WithdrawalQueueAddress = common.HexToAddress("0x00000961Ef480Eb55e80D19ad83579A64c007002") WithdrawalQueueCode = common.FromHex("3373fffffffffffffffffffffffffffffffffffffffe1460cb5760115f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff146101f457600182026001905f5b5f82111560685781019083028483029004916001019190604d565b909390049250505036603814608857366101f457346101f4575f5260205ff35b34106101f457600154600101600155600354806003026004013381556001015f35815560010160203590553360601b5f5260385f601437604c5fa0600101600355005b6003546002548082038060101160df575060105b5f5b8181146101835782810160030260040181604c02815460601b8152601401816001015481526020019060020154807fffffffffffffffffffffffffffffffff00000000000000000000000000000000168252906010019060401c908160381c81600701538160301c81600601538160281c81600501538160201c81600401538160181c81600301538160101c81600201538160081c81600101535360010160e1565b910180921461019557906002556101a0565b90505f6002555f6003555b5f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff14156101cd57505f5b6001546002828201116101e25750505f6101e8565b01600290035b5f555f600155604c025ff35b5f5ffd") // EIP-7251 - Increase the MAX_EFFECTIVE_BALANCE - ConsolidationQueueAddress = common.HexToAddress("0x00431F263cE400f4455c2dCf564e53007Ca4bbBb") + ConsolidationQueueAddress = common.HexToAddress("0x0000BBdDc7CE488642fb579F8B00f3a590007251") ConsolidationQueueCode = common.FromHex("3373fffffffffffffffffffffffffffffffffffffffe1460d35760115f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1461019a57600182026001905f5b5f82111560685781019083028483029004916001019190604d565b9093900492505050366060146088573661019a573461019a575f5260205ff35b341061019a57600154600101600155600354806004026004013381556001015f358155600101602035815560010160403590553360601b5f5260605f60143760745fa0600101600355005b6003546002548082038060021160e7575060025b5f5b8181146101295782810160040260040181607402815460601b815260140181600101548152602001816002015481526020019060030154905260010160e9565b910180921461013b5790600255610146565b90505f6002555f6003555b5f54807fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff141561017357505f5b6001546001828201116101885750505f61018e565b01600190035b5f555f6001556074025ff35b5f5ffd") ) From fc12dbe40bdb3fd77d0ea6049570b8c870421859 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Fri, 31 Jan 2025 18:34:22 +0100 Subject: [PATCH 13/24] eth/catalyst: fix validation of type 0 request (#31103) I caught this error on Hive. It was introduced by https://github.com/ethereum/go-ethereum/pull/31071 because after adding the equality check the request type 0 will be rejected. --- eth/catalyst/api.go | 6 ++--- eth/catalyst/api_test.go | 58 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 7a9707dc15..e6f29c970b 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -1272,18 +1272,16 @@ func convertRequests(hex []hexutil.Bytes) [][]byte { // validateRequests checks that requests are ordered by their type and are not empty. func validateRequests(requests [][]byte) error { - var last byte - for _, req := range requests { + for i, req := range requests { // No empty requests. if len(req) < 2 { return fmt.Errorf("empty request: %v", req) } // Check that requests are ordered by their type. // Each type must appear only once. - if req[0] <= last { + if i > 0 && req[0] <= requests[i-1][0] { return fmt.Errorf("invalid request order: %v", req) } - last = req[0] } return nil } diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 4698827731..407f0db3eb 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -1737,3 +1737,61 @@ func TestGetClientVersion(t *testing.T) { t.Fatalf("client info does match expected, got %s", info.String()) } } + +func TestValidateRequests(t *testing.T) { + tests := []struct { + name string + requests [][]byte + wantErr bool + }{ + { + name: "valid ascending", + requests: [][]byte{ + {0x00, 0xAA, 0xBB}, // type 0x00 + {0x01, 0xCC}, // type 0x01 + {0x02, 0xDD}, // type 0x02 + }, + wantErr: false, + }, + { + name: "empty request (too short)", + requests: [][]byte{ + {0x00}, // only 1 byte: type with no data + }, + wantErr: true, + }, + { + name: "duplicate type", + requests: [][]byte{ + {0x00, 0x11}, + {0x01, 0x22}, + {0x01, 0x33}, // duplicate type 0x01 + }, + wantErr: true, + }, + { + name: "out of order", + requests: [][]byte{ + {0x01, 0xAA}, // type 0x01 + {0x00, 0xBB}, // type 0x00 out of order (should be ascending) + }, + wantErr: true, + }, + { + name: "single request valid", + requests: [][]byte{ + {0x01, 0xAB}, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateRequests(tt.requests) + if (err != nil) != tt.wantErr { + t.Errorf("validateRequests(%v) error = %v, wantErr = %v", + tt.requests, err, tt.wantErr) + } + }) + } +} From 55a18616b1c4d2ffee498e07ddef23f49bddbb6d Mon Sep 17 00:00:00 2001 From: jwasinger Date: Mon, 3 Feb 2025 09:44:26 -0800 Subject: [PATCH 14/24] core/vm: simplify tracer hook invocation in interpreter loop (#31074) Removes duplicate code in the interpreter loop. --- core/vm/interpreter.go | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index e3237936a0..a0038d1aa8 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -258,9 +258,9 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( contract.Gas -= cost } + // All ops with a dynamic memory usage also has a dynamic gas cost. + var memorySize uint64 if operation.dynamicGas != nil { - // All ops with a dynamic memory usage also has a dynamic gas cost. - var memorySize uint64 // calculate the new memory size and expand the memory to fit // the operation // Memory check needs to be done prior to evaluating the dynamic gas portion, @@ -290,21 +290,10 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( } else { contract.Gas -= dynamicCost } + } - // Do tracing before memory expansion - if debug { - if in.evm.Config.Tracer.OnGasChange != nil { - in.evm.Config.Tracer.OnGasChange(gasCopy, gasCopy-cost, tracing.GasChangeCallOpCode) - } - if in.evm.Config.Tracer.OnOpcode != nil { - in.evm.Config.Tracer.OnOpcode(pc, byte(op), gasCopy, cost, callContext, in.returnData, in.evm.depth, VMErrorFromErr(err)) - logged = true - } - } - if memorySize > 0 { - mem.Resize(memorySize) - } - } else if debug { + // Do tracing before potential memory expansion + if debug { if in.evm.Config.Tracer.OnGasChange != nil { in.evm.Config.Tracer.OnGasChange(gasCopy, gasCopy-cost, tracing.GasChangeCallOpCode) } @@ -313,6 +302,9 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( logged = true } } + if memorySize > 0 { + mem.Resize(memorySize) + } // execute the operation res, err = operation.execute(&pc, in, callContext) From e26dd774a9c0f77f814673d64870d84da79cd549 Mon Sep 17 00:00:00 2001 From: kazak Date: Tue, 4 Feb 2025 08:09:06 +0200 Subject: [PATCH 15/24] tests/fuzzers/bls12381: fix error message in fuzzCrossG2Add (#31113) Fixes a typo in the error message within the `fuzzCrossG2Add` function. The panic message incorrectly references "G1 point addition mismatch" when it should be "G2 point addition mismatch," as the function deals with G2 points. This doesn't affect functionality but could cause confusion during debugging. I've updated the message to reflect the correct curve. --- tests/fuzzers/bls12381/bls12381_fuzz.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzzers/bls12381/bls12381_fuzz.go b/tests/fuzzers/bls12381/bls12381_fuzz.go index 5cb05b5660..a17c128b7f 100644 --- a/tests/fuzzers/bls12381/bls12381_fuzz.go +++ b/tests/fuzzers/bls12381/bls12381_fuzz.go @@ -168,7 +168,7 @@ func fuzzCrossG2Add(data []byte) int { bl3 := blst.P2AffinesAdd([]*blst.P2Affine{bl1, bl2}) if !(bytes.Equal(gp.Marshal(), bl3.Serialize())) { - panic("G1 point addition mismatch blst / geth ") + panic("G2 point addition mismatch blst / geth ") } return 1 From 0ad0966cecf0602bc554fd368d2a02ea762baf8c Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 4 Feb 2025 18:45:45 +0800 Subject: [PATCH 16/24] core/rawdb: introduce flush offset in freezer (#30392) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a follow-up PR to #29792 to get rid of the data file sync. **This is a non-backward compatible change, which increments the database version from 8 to 9**. We introduce a flushOffset for each freezer table, which tracks the position of the most recently fsync’d item in the index file. When this offset moves forward, it indicates that all index entries below it, along with their corresponding data items, have been properly persisted to disk. The offset can also be moved backward when truncating from either the head or tail of the file. Previously, the data file required an explicit fsync after every mutation, which was highly inefficient. With the introduction of the flush offset, the synchronization strategy becomes more flexible, allowing the freezer to sync every 30 seconds instead. The data items above the flush offset are regarded volatile and callers must ensure they are recoverable after the unclean shutdown, or explicitly sync the freezer before any proceeding operations. --------- Co-authored-by: Felix Lange --- core/blockchain.go | 10 +- core/rawdb/accessors_chain_test.go | 3 +- core/rawdb/ancient_scheme.go | 1 + core/rawdb/freezer_batch.go | 10 +- core/rawdb/freezer_meta.go | 192 +++++++++++----- core/rawdb/freezer_meta_test.go | 88 ++++++- core/rawdb/freezer_table.go | 264 ++++++++++++++------- core/rawdb/freezer_table_test.go | 356 ++++++++++++++++++++--------- 8 files changed, 664 insertions(+), 260 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index ab88f4b68e..f097dc9781 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -113,23 +113,29 @@ const ( // * the `BlockNumber`, `TxHash`, `TxIndex`, `BlockHash` and `Index` fields of log are deleted // * the `Bloom` field of receipt is deleted // * the `BlockIndex` and `TxIndex` fields of txlookup are deleted + // // - Version 5 // The following incompatible database changes were added: // * the `TxHash`, `GasCost`, and `ContractAddress` fields are no longer stored for a receipt // * the `TxHash`, `GasCost`, and `ContractAddress` fields are computed by looking up the // receipts' corresponding block + // // - Version 6 // The following incompatible database changes were added: // * Transaction lookup information stores the corresponding block number instead of block hash + // // - Version 7 // The following incompatible database changes were added: // * Use freezer as the ancient database to maintain all ancient data + // // - Version 8 // The following incompatible database changes were added: // * New scheme for contract code in order to separate the codes and trie nodes + // // - Version 9 - // Total difficulty has been removed from both the key-value store and the - // ancient store, the td freezer table has been deprecated since that. + // The following incompatible database changes were added: + // * Total difficulty has been removed from both the key-value store and the ancient store. + // * The metadata structure of freezer is changed by adding 'flushOffset' BlockChainVersion uint64 = 9 ) diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 5533c60ab9..b9684f8e17 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -849,6 +849,7 @@ func TestHeadersRLPStorage(t *testing.T) { t.Fatalf("failed to create database with ancient backend") } defer db.Close() + // Create blocks var chain []*types.Block var pHash common.Hash @@ -864,7 +865,7 @@ func TestHeadersRLPStorage(t *testing.T) { chain = append(chain, block) pHash = block.Hash() } - var receipts []types.Receipts = make([]types.Receipts, 100) + receipts := make([]types.Receipts, 100) // Write first half to ancients WriteAncientBlocks(db, chain[:50], receipts[:50]) // Write second half to db diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go index 54a3be391f..67bfa37ecc 100644 --- a/core/rawdb/ancient_scheme.go +++ b/core/rawdb/ancient_scheme.go @@ -58,6 +58,7 @@ const ( stateHistoryStorageData = "storage.data" ) +// stateFreezerNoSnappy configures whether compression is disabled for the state freezer. var stateFreezerNoSnappy = map[string]bool{ stateHistoryMeta: true, stateHistoryAccountIndex: false, diff --git a/core/rawdb/freezer_batch.go b/core/rawdb/freezer_batch.go index 013d0b9d13..801d30f73f 100644 --- a/core/rawdb/freezer_batch.go +++ b/core/rawdb/freezer_batch.go @@ -19,6 +19,7 @@ package rawdb import ( "fmt" "math" + "time" "github.com/ethereum/go-ethereum/rlp" "github.com/golang/snappy" @@ -188,9 +189,6 @@ func (batch *freezerTableBatch) commit() error { if err != nil { return err } - if err := batch.t.head.Sync(); err != nil { - return err - } dataSize := int64(len(batch.dataBuffer)) batch.dataBuffer = batch.dataBuffer[:0] @@ -208,6 +206,12 @@ func (batch *freezerTableBatch) commit() error { // Update metrics. batch.t.sizeGauge.Inc(dataSize + indexSize) batch.t.writeMeter.Mark(dataSize + indexSize) + + // Periodically sync the table, todo (rjl493456442) make it configurable? + if time.Since(batch.t.lastSync) > 30*time.Second { + batch.t.lastSync = time.Now() + return batch.t.Sync() + } return nil } diff --git a/core/rawdb/freezer_meta.go b/core/rawdb/freezer_meta.go index 9eef9df351..3cda9ae45c 100644 --- a/core/rawdb/freezer_meta.go +++ b/core/rawdb/freezer_meta.go @@ -17,93 +17,173 @@ package rawdb import ( + "errors" "io" + "math" "os" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) -const freezerVersion = 1 // The initial version tag of freezer table metadata +const ( + freezerTableV1 = 1 // Initial version of metadata struct + freezerTableV2 = 2 // Add field: 'flushOffset' + freezerVersion = freezerTableV2 // The current used version +) -// freezerTableMeta wraps all the metadata of the freezer table. +// freezerTableMeta is a collection of additional properties that describe the +// freezer table. These properties are designed with error resilience, allowing +// them to be automatically corrected after an error occurs without significantly +// impacting overall correctness. type freezerTableMeta struct { - // Version is the versioning descriptor of the freezer table. - Version uint16 + file *os.File // file handler of metadata + version uint16 // version descriptor of the freezer table - // VirtualTail indicates how many items have been marked as deleted. - // Its value is equal to the number of items removed from the table - // plus the number of items hidden in the table, so it should never - // be lower than the "actual tail". - VirtualTail uint64 + // virtualTail represents the number of items marked as deleted. It is + // calculated as the sum of items removed from the table and the items + // hidden within the table, and should never be less than the "actual + // tail". + // + // If lost due to a crash or other reasons, it will be reset to the number + // of items deleted from the table, causing the previously hidden items + // to become visible, which is an acceptable consequence. + virtualTail uint64 + + // flushOffset represents the offset in the index file up to which the index + // items along with the corresponding data items in data files has been flushed + // (fsync’d) to disk. Beyond this offset, data integrity is not guaranteed, + // the extra index items along with the associated data items should be removed + // during the startup. + // + // The principle is that all data items above the flush offset are considered + // volatile and should be recoverable if they are discarded after the unclean + // shutdown. If data integrity is required, manually force a sync of the + // freezer before proceeding with further operations (e.g. do freezer.Sync() + // first and then write data to key value store in some circumstances). + // + // The offset could be moved forward by applying sync operation, or be moved + // backward in cases of head/tail truncation, etc. + flushOffset int64 } -// newMetadata initializes the metadata object with the given virtual tail. -func newMetadata(tail uint64) *freezerTableMeta { +// decodeV1 attempts to decode the metadata structure in v1 format. If fails or +// the result is incompatible, nil is returned. +func decodeV1(file *os.File) *freezerTableMeta { + _, err := file.Seek(0, io.SeekStart) + if err != nil { + return nil + } + type obj struct { + Version uint16 + Tail uint64 + } + var o obj + if err := rlp.Decode(file, &o); err != nil { + return nil + } + if o.Version != freezerTableV1 { + return nil + } return &freezerTableMeta{ - Version: freezerVersion, - VirtualTail: tail, + file: file, + version: o.Version, + virtualTail: o.Tail, } } -// readMetadata reads the metadata of the freezer table from the -// given metadata file. -func readMetadata(file *os.File) (*freezerTableMeta, error) { +// decodeV2 attempts to decode the metadata structure in v2 format. If fails or +// the result is incompatible, nil is returned. +func decodeV2(file *os.File) *freezerTableMeta { _, err := file.Seek(0, io.SeekStart) if err != nil { - return nil, err + return nil } - var meta freezerTableMeta - if err := rlp.Decode(file, &meta); err != nil { - return nil, err + type obj struct { + Version uint16 + Tail uint64 + Offset uint64 + } + var o obj + if err := rlp.Decode(file, &o); err != nil { + return nil + } + if o.Version != freezerTableV2 { + return nil + } + if o.Offset > math.MaxInt64 { + log.Error("Invalid flushOffset %d in freezer metadata", o.Offset, "file", file.Name()) + return nil + } + return &freezerTableMeta{ + file: file, + version: freezerTableV2, + virtualTail: o.Tail, + flushOffset: int64(o.Offset), } - return &meta, nil } -// writeMetadata writes the metadata of the freezer table into the -// given metadata file. -func writeMetadata(file *os.File, meta *freezerTableMeta) error { - _, err := file.Seek(0, io.SeekStart) - if err != nil { - return err - } - return rlp.Encode(file, meta) -} - -// loadMetadata loads the metadata from the given metadata file. -// Initializes the metadata file with the given "actual tail" if -// it's empty. -func loadMetadata(file *os.File, tail uint64) (*freezerTableMeta, error) { +// newMetadata initializes the metadata object, either by loading it from the file +// or by constructing a new one from scratch. +func newMetadata(file *os.File) (*freezerTableMeta, error) { stat, err := file.Stat() if err != nil { return nil, err } - // Write the metadata with the given actual tail into metadata file - // if it's non-existent. There are two possible scenarios here: - // - the freezer table is empty - // - the freezer table is legacy - // In both cases, write the meta into the file with the actual tail - // as the virtual tail. if stat.Size() == 0 { - m := newMetadata(tail) - if err := writeMetadata(file, m); err != nil { + m := &freezerTableMeta{ + file: file, + version: freezerTableV2, + virtualTail: 0, + flushOffset: 0, + } + if err := m.write(true); err != nil { return nil, err } return m, nil } - m, err := readMetadata(file) - if err != nil { - return nil, err + if m := decodeV2(file); m != nil { + return m, nil } - // Update the virtual tail with the given actual tail if it's even - // lower than it. Theoretically it shouldn't happen at all, print - // a warning here. - if m.VirtualTail < tail { - log.Warn("Updated virtual tail", "have", m.VirtualTail, "now", tail) - m.VirtualTail = tail - if err := writeMetadata(file, m); err != nil { - return nil, err - } + if m := decodeV1(file); m != nil { + return m, nil // legacy metadata } - return m, nil + return nil, errors.New("failed to decode metadata") +} + +// setVirtualTail sets the virtual tail and flushes the metadata if sync is true. +func (m *freezerTableMeta) setVirtualTail(tail uint64, sync bool) error { + m.virtualTail = tail + return m.write(sync) +} + +// setFlushOffset sets the flush offset and flushes the metadata if sync is true. +func (m *freezerTableMeta) setFlushOffset(offset int64, sync bool) error { + m.flushOffset = offset + return m.write(sync) +} + +// write flushes the content of metadata into file and performs a fsync if required. +func (m *freezerTableMeta) write(sync bool) error { + type obj struct { + Version uint16 + Tail uint64 + Offset uint64 + } + var o obj + o.Version = freezerVersion // forcibly use the current version + o.Tail = m.virtualTail + o.Offset = uint64(m.flushOffset) + + _, err := m.file.Seek(0, io.SeekStart) + if err != nil { + return err + } + if err := rlp.Encode(m.file, &o); err != nil { + return err + } + if !sync { + return nil + } + return m.file.Sync() } diff --git a/core/rawdb/freezer_meta_test.go b/core/rawdb/freezer_meta_test.go index 409e811026..31f8c519c8 100644 --- a/core/rawdb/freezer_meta_test.go +++ b/core/rawdb/freezer_meta_test.go @@ -19,6 +19,8 @@ package rawdb import ( "os" "testing" + + "github.com/ethereum/go-ethereum/rlp" ) func TestReadWriteFreezerTableMeta(t *testing.T) { @@ -27,36 +29,98 @@ func TestReadWriteFreezerTableMeta(t *testing.T) { t.Fatalf("Failed to create file %v", err) } defer f.Close() - err = writeMetadata(f, newMetadata(100)) + + meta, err := newMetadata(f) if err != nil { - t.Fatalf("Failed to write metadata %v", err) + t.Fatalf("Failed to new metadata %v", err) } - meta, err := readMetadata(f) + meta.setVirtualTail(100, false) + + meta, err = newMetadata(f) if err != nil { - t.Fatalf("Failed to read metadata %v", err) + t.Fatalf("Failed to reload metadata %v", err) } - if meta.Version != freezerVersion { + if meta.version != freezerTableV2 { t.Fatalf("Unexpected version field") } - if meta.VirtualTail != uint64(100) { + if meta.virtualTail != uint64(100) { t.Fatalf("Unexpected virtual tail field") } } -func TestInitializeFreezerTableMeta(t *testing.T) { +func TestUpgradeMetadata(t *testing.T) { f, err := os.CreateTemp(t.TempDir(), "*") if err != nil { t.Fatalf("Failed to create file %v", err) } defer f.Close() - meta, err := loadMetadata(f, uint64(100)) + + // Write legacy metadata into file + type obj struct { + Version uint16 + Tail uint64 + } + var o obj + o.Version = freezerTableV1 + o.Tail = 100 + + if err := rlp.Encode(f, &o); err != nil { + t.Fatalf("Failed to encode %v", err) + } + + // Reload the metadata, a silent upgrade is expected + meta, err := newMetadata(f) if err != nil { t.Fatalf("Failed to read metadata %v", err) } - if meta.Version != freezerVersion { - t.Fatalf("Unexpected version field") + if meta.version != freezerTableV1 { + t.Fatal("Unexpected version field") } - if meta.VirtualTail != uint64(100) { - t.Fatalf("Unexpected virtual tail field") + if meta.virtualTail != uint64(100) { + t.Fatal("Unexpected virtual tail field") + } + if meta.flushOffset != 0 { + t.Fatal("Unexpected flush offset field") + } + + meta.setFlushOffset(100, true) + + meta, err = newMetadata(f) + if err != nil { + t.Fatalf("Failed to read metadata %v", err) + } + if meta.version != freezerTableV2 { + t.Fatal("Unexpected version field") + } + if meta.virtualTail != uint64(100) { + t.Fatal("Unexpected virtual tail field") + } + if meta.flushOffset != 100 { + t.Fatal("Unexpected flush offset field") + } +} + +func TestInvalidMetadata(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "*") + if err != nil { + t.Fatalf("Failed to create file %v", err) + } + defer f.Close() + + // Write invalid legacy metadata into file + type obj struct { + Version uint16 + Tail uint64 + } + var o obj + o.Version = freezerTableV2 // -> invalid version tag + o.Tail = 100 + + if err := rlp.Encode(f, &o); err != nil { + t.Fatalf("Failed to encode %v", err) + } + _, err = newMetadata(f) + if err == nil { + t.Fatal("Unexpected success") } } diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 38c47dc223..1ba8cf639f 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -108,11 +108,13 @@ type freezerTable struct { head *os.File // File descriptor for the data head of the table index *os.File // File descriptor for the indexEntry file of the table - meta *os.File // File descriptor for metadata of the table files map[uint32]*os.File // open files headId uint32 // number of the currently active head file tailId uint32 // number of the earliest file + metadata *freezerTableMeta // metadata of the table + lastSync time.Time // Timestamp when the last sync was performed + headBytes int64 // Number of bytes written to the head file readMeter *metrics.Meter // Meter for measuring the effective amount of data read writeMeter *metrics.Meter // Meter for measuring the effective amount of data written @@ -166,10 +168,17 @@ func newTable(path string, name string, readMeter, writeMeter *metrics.Meter, si return nil, err } } + // Load metadata from the file. The tag will be true if legacy metadata + // is detected. + metadata, err := newMetadata(meta) + if err != nil { + return nil, err + } // Create the table and repair any past inconsistency tab := &freezerTable{ index: index, - meta: meta, + metadata: metadata, + lastSync: time.Now(), files: make(map[uint32]*os.File), readMeter: readMeter, writeMeter: writeMeter, @@ -221,13 +230,11 @@ func (t *freezerTable) repair() error { return err } // New file can't trigger this path } - // Validate the index file as it might contain some garbage data after the - // power failures. if err := t.repairIndex(); err != nil { return err } // Retrieve the file sizes and prepare for truncation. Note the file size - // might be changed after index validation. + // might be changed after index repair. if stat, err = t.index.Stat(); err != nil { return err } @@ -253,12 +260,14 @@ func (t *freezerTable) repair() error { t.tailId = firstIndex.filenum t.itemOffset.Store(uint64(firstIndex.offset)) - // Load metadata from the file - meta, err := loadMetadata(t.meta, t.itemOffset.Load()) - if err != nil { - return err + // Adjust the number of hidden items if it is less than the number of items + // being removed. + if t.itemOffset.Load() > t.metadata.virtualTail { + if err := t.metadata.setVirtualTail(t.itemOffset.Load(), true); err != nil { + return err + } } - t.itemHidden.Store(meta.VirtualTail) + t.itemHidden.Store(t.metadata.virtualTail) // Read the last index, use the default value in case the freezer is empty if offsetsSize == indexEntrySize { @@ -267,12 +276,6 @@ func (t *freezerTable) repair() error { t.index.ReadAt(buffer, offsetsSize-indexEntrySize) lastIndex.unmarshalBinary(buffer) } - // Print an error log if the index is corrupted due to an incorrect - // last index item. While it is theoretically possible to have a zero offset - // by storing all zero-size items, it is highly unlikely to occur in practice. - if lastIndex.offset == 0 && offsetsSize/indexEntrySize > 1 { - log.Error("Corrupted index file detected", "lastOffset", lastIndex.offset, "indexes", offsetsSize/indexEntrySize) - } if t.readonly { t.head, err = t.openFile(lastIndex.filenum, openFreezerFileForReadOnly) } else { @@ -293,6 +296,7 @@ func (t *freezerTable) repair() error { return fmt.Errorf("freezer table(path: %s, name: %s, num: %d) is corrupted", t.path, t.name, lastIndex.filenum) } verbose = true + // Truncate the head file to the last offset pointer if contentExp < contentSize { t.logger.Warn("Truncating dangling head", "indexed", contentExp, "stored", contentSize) @@ -304,11 +308,23 @@ func (t *freezerTable) repair() error { // Truncate the index to point within the head file if contentExp > contentSize { t.logger.Warn("Truncating dangling indexes", "indexes", offsetsSize/indexEntrySize, "indexed", contentExp, "stored", contentSize) - if err := truncateFreezerFile(t.index, offsetsSize-indexEntrySize); err != nil { + + newOffset := offsetsSize - indexEntrySize + if err := truncateFreezerFile(t.index, newOffset); err != nil { return err } offsetsSize -= indexEntrySize + // If the index file is truncated beyond the flush offset, move the flush + // offset back to the new end of the file. A crash may occur before the + // offset is updated, leaving a dangling reference that points to a position + // outside the file. If so, the offset will be reset to the new end of the + // file during the next run. + if t.metadata.flushOffset > newOffset { + if err := t.metadata.setFlushOffset(newOffset, true); err != nil { + return err + } + } // Read the new head index, use the default value in case // the freezer is already empty. var newLastIndex indexEntry @@ -345,7 +361,7 @@ func (t *freezerTable) repair() error { if err := t.head.Sync(); err != nil { return err } - if err := t.meta.Sync(); err != nil { + if err := t.metadata.file.Sync(); err != nil { return err } } @@ -372,7 +388,65 @@ func (t *freezerTable) repair() error { return nil } -// repairIndex validates the integrity of the index file. According to the design, +func (t *freezerTable) repairIndex() error { + stat, err := t.index.Stat() + if err != nil { + return err + } + size := stat.Size() + + // Validate the items in the index file to ensure the data integrity. + // It's possible some garbage data is retained in the index file after + // the power failures and should be truncated first. + size, err = t.checkIndex(size) + if err != nil { + return err + } + // If legacy metadata is detected, attempt to recover the offset from the + // index file to avoid clearing the entire table. + if t.metadata.version == freezerTableV1 { + t.logger.Info("Recovering freezer flushOffset for legacy table", "offset", size) + return t.metadata.setFlushOffset(size, true) + } + + switch { + case size == indexEntrySize && t.metadata.flushOffset == 0: + // It's a new freezer table with no content. + // Move the flush offset to the end of the file. + return t.metadata.setFlushOffset(size, true) + + case size == t.metadata.flushOffset: + // flushOffset is aligned with the index file, all is well. + return nil + + case size > t.metadata.flushOffset: + // Extra index items have been detected beyond the flush offset. Since these + // entries correspond to data that has not been fully flushed to disk in the + // last run (because of unclean shutdown), their integrity cannot be guaranteed. + // To ensure consistency, these index items will be truncated, as there is no + // reliable way to validate or recover their associated data. + extraSize := size - t.metadata.flushOffset + if t.readonly { + return fmt.Errorf("index file(path: %s, name: %s) contains %d garbage data bytes", t.path, t.name, extraSize) + } + t.logger.Warn("Truncating freezer items after flushOffset", "size", extraSize) + return truncateFreezerFile(t.index, t.metadata.flushOffset) + + default: // size < flushOffset + // Flush offset refers to a position larger than index file. The only + // possible scenario for this is: a power failure or system crash has occurred after + // truncating the segment in index file from head or tail, but without updating + // the flush offset. In this case, automatically reset the flush offset with + // the file size which implies the entire index file is complete. + if t.readonly { + return nil // do nothing in read only mode + } + t.logger.Warn("Rewinding freezer flushOffset", "old", t.metadata.flushOffset, "new", size) + return t.metadata.setFlushOffset(size, true) + } +} + +// checkIndex validates the integrity of the index file. According to the design, // the initial entry in the file denotes the earliest data file along with the // count of deleted items. Following this, all subsequent entries in the file must // be in order. This function identifies any corrupted entries and truncates items @@ -392,18 +466,11 @@ func (t *freezerTable) repair() error { // leftover garbage or if all items in the table have zero size is impossible. // In such instances, the file will remain unchanged to prevent potential data // loss or misinterpretation. -func (t *freezerTable) repairIndex() error { - // Retrieve the file sizes and prepare for validation - stat, err := t.index.Stat() - if err != nil { - return err - } - size := stat.Size() - +func (t *freezerTable) checkIndex(size int64) (int64, error) { // Move the read cursor to the beginning of the file - _, err = t.index.Seek(0, io.SeekStart) + _, err := t.index.Seek(0, io.SeekStart) if err != nil { - return err + return 0, err } fr := bufio.NewReader(t.index) @@ -425,21 +492,21 @@ func (t *freezerTable) repairIndex() error { entry.unmarshalBinary(buff) return entry, nil } - truncate = func(offset int64) error { + truncate = func(offset int64) (int64, error) { if t.readonly { - return fmt.Errorf("index file is corrupted at %d, size: %d", offset, size) + return 0, fmt.Errorf("index file is corrupted at %d, size: %d", offset, size) } if err := truncateFreezerFile(t.index, offset); err != nil { - return err + return 0, err } log.Warn("Truncated index file", "offset", offset, "truncated", size-offset) - return nil + return offset, nil } ) for offset := int64(0); offset < size; offset += indexEntrySize { entry, err := read() if err != nil { - return err + return 0, err } if offset == 0 { head = entry @@ -468,10 +535,10 @@ func (t *freezerTable) repairIndex() error { // the seek operation anyway as a precaution. _, err = t.index.Seek(0, io.SeekEnd) if err != nil { - return err + return 0, err } log.Debug("Verified index file", "items", size/indexEntrySize, "elapsed", common.PrettyDuration(time.Since(start))) - return nil + return size, nil } // checkIndexItems validates the correctness of two consecutive index items based @@ -550,12 +617,23 @@ func (t *freezerTable) truncateHead(items uint64) error { // Truncate the index file first, the tail position is also considered // when calculating the new freezer table length. length := items - t.itemOffset.Load() - if err := truncateFreezerFile(t.index, int64(length+1)*indexEntrySize); err != nil { + newOffset := (length + 1) * indexEntrySize + if err := truncateFreezerFile(t.index, int64(newOffset)); err != nil { return err } if err := t.index.Sync(); err != nil { return err } + // If the index file is truncated beyond the flush offset, move the flush + // offset back to the new end of the file. A crash may occur before the + // offset is updated, leaving a dangling reference that points to a position + // outside the file. If so, the offset will be reset to the new end of the + // file during the next run. + if t.metadata.flushOffset > int64(newOffset) { + if err := t.metadata.setFlushOffset(int64(newOffset), true); err != nil { + return err + } + } // Calculate the new expected size of the data file and truncate it var expected indexEntry if length == 0 { @@ -652,7 +730,10 @@ func (t *freezerTable) truncateTail(items uint64) error { } // Update the virtual tail marker and hidden these entries in table. t.itemHidden.Store(items) - if err := writeMetadata(t.meta, newMetadata(items)); err != nil { + + // Update the virtual tail without fsync, otherwise it will significantly + // impact the overall performance. + if err := t.metadata.setVirtualTail(items, false); err != nil { return err } // Hidden items still fall in the current tail file, no data file @@ -664,6 +745,18 @@ func (t *freezerTable) truncateTail(items uint64) error { if t.tailId > newTailId { return fmt.Errorf("invalid index, tail-file %d, item-file %d", t.tailId, newTailId) } + // Sync the table before performing the index tail truncation. A crash may + // occur after truncating the index file without updating the flush offset, + // leaving a dangling offset that points to a position outside the file. + // The offset will be rewound to the end of file during the next run + // automatically and implicitly assumes all the items within the file are + // complete. + // + // Therefore, forcibly flush everything above the offset to ensure this + // assumption is satisfied! + if err := t.doSync(); err != nil { + return err + } // Count how many items can be deleted from the file. var ( newDeleted = items @@ -681,11 +774,6 @@ func (t *freezerTable) truncateTail(items uint64) error { } newDeleted = current } - // Commit the changes of metadata file first before manipulating - // the indexes file. - if err := t.meta.Sync(); err != nil { - return err - } // Close the index file before shorten it. if err := t.index.Close(); err != nil { return err @@ -716,6 +804,21 @@ func (t *freezerTable) truncateTail(items uint64) error { t.itemOffset.Store(newDeleted) t.releaseFilesBefore(t.tailId, true) + // Move the index flush offset backward due to the deletion of an index segment. + // A crash may occur before the offset is updated, leaving a dangling reference + // that points to a position outside the file. If so, the offset will be reset + // to the new end of the file during the next run. + // + // Note, both the index and head data file has been persisted before performing + // tail truncation and all the items in these files are regarded as complete. + shorten := indexEntrySize * int64(newDeleted-deleted) + if t.metadata.flushOffset <= shorten { + return fmt.Errorf("invalid index flush offset: %d, shorten: %d", t.metadata.flushOffset, shorten) + } else { + if err := t.metadata.setFlushOffset(t.metadata.flushOffset-shorten, true); err != nil { + return err + } + } // Retrieve the new size and update the total size counter newSize, err := t.sizeNolock() if err != nil { @@ -725,40 +828,30 @@ func (t *freezerTable) truncateTail(items uint64) error { return nil } -// Close closes all opened files. +// Close closes all opened files and finalizes the freezer table for use. +// This operation must be completed before shutdown to prevent the loss of +// recent writes. func (t *freezerTable) Close() error { t.lock.Lock() defer t.lock.Unlock() + if err := t.doSync(); err != nil { + return err + } var errs []error - doClose := func(f *os.File, sync bool, close bool) { - if sync && !t.readonly { - if err := f.Sync(); err != nil { - errs = append(errs, err) - } - } - if close { - if err := f.Close(); err != nil { - errs = append(errs, err) - } + doClose := func(f *os.File) { + if err := f.Close(); err != nil { + errs = append(errs, err) } } - // Trying to fsync a file opened in rdonly causes "Access denied" - // error on Windows. - doClose(t.index, true, true) - doClose(t.meta, true, true) - - // The preopened non-head data-files are all opened in readonly. - // The head is opened in rw-mode, so we sync it here - but since it's also - // part of t.files, it will be closed in the loop below. - doClose(t.head, true, false) // sync but do not close - + doClose(t.index) + doClose(t.metadata.file) for _, f := range t.files { - doClose(f, false, true) // close but do not sync + doClose(f) } t.index = nil - t.meta = nil t.head = nil + t.metadata.file = nil if errs != nil { return fmt.Errorf("%v", errs) @@ -917,7 +1010,7 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i defer t.lock.RUnlock() // Ensure the table and the item are accessible - if t.index == nil || t.head == nil || t.meta == nil { + if t.index == nil || t.head == nil || t.metadata.file == nil { return nil, nil, errClosed } var ( @@ -1042,6 +1135,9 @@ func (t *freezerTable) advanceHead() error { t.lock.Lock() defer t.lock.Unlock() + if err := t.doSync(); err != nil { + return err + } // We open the next file in truncated mode -- if this file already // exists, we need to start over from scratch on it. nextID := t.headId + 1 @@ -1069,7 +1165,18 @@ func (t *freezerTable) advanceHead() error { func (t *freezerTable) Sync() error { t.lock.Lock() defer t.lock.Unlock() - if t.index == nil || t.head == nil || t.meta == nil { + + return t.doSync() +} + +// doSync is the internal version of Sync which assumes the lock is already held. +func (t *freezerTable) doSync() error { + // Trying to fsync a file opened in rdonly causes "Access denied" + // error on Windows. + if t.readonly { + return nil + } + if t.index == nil || t.head == nil || t.metadata.file == nil { return errClosed } var err error @@ -1078,10 +1185,18 @@ func (t *freezerTable) Sync() error { err = e } } - trackError(t.index.Sync()) - trackError(t.meta.Sync()) trackError(t.head.Sync()) + + // A crash may occur before the offset is updated, leaving the offset + // points to a old position. If so, the extra items above the offset + // will be truncated during the next run. + stat, err := t.index.Stat() + if err != nil { + return err + } + offset := stat.Size() + trackError(t.metadata.setFlushOffset(offset, true)) return err } @@ -1097,13 +1212,8 @@ func (t *freezerTable) dumpIndexString(start, stop int64) string { } func (t *freezerTable) dumpIndex(w io.Writer, start, stop int64) { - meta, err := readMetadata(t.meta) - if err != nil { - fmt.Fprintf(w, "Failed to decode freezer table %v\n", err) - return - } - fmt.Fprintf(w, "Version %d count %d, deleted %d, hidden %d\n", meta.Version, - t.items.Load(), t.itemOffset.Load(), t.itemHidden.Load()) + fmt.Fprintf(w, "Version %d count %d, deleted %d, hidden %d\n", + t.metadata.version, t.items.Load(), t.itemOffset.Load(), t.itemHidden.Load()) buf := make([]byte, indexEntrySize) diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index fd6e3cf199..9a72af6ccc 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -262,18 +262,6 @@ func TestSnappyDetection(t *testing.T) { f.Close() } - // Open without snappy - { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false, false) - if err != nil { - t.Fatal(err) - } - if _, err = f.Retrieve(0); err == nil { - f.Close() - t.Fatalf("expected empty table") - } - } - // Open with snappy { f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) @@ -286,6 +274,18 @@ func TestSnappyDetection(t *testing.T) { t.Fatalf("expected no error, got %v", err) } } + + // Open without snappy + { + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false, false) + if err != nil { + t.Fatal(err) + } + if _, err = f.Retrieve(0); err == nil { + f.Close() + t.Fatalf("expected empty table") + } + } } func assertFileSize(f string, size int64) error { @@ -521,93 +521,53 @@ func TestFreezerOffset(t *testing.T) { fname := fmt.Sprintf("offset-%d", rand.Uint64()) // Fill table - { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) - if err != nil { - t.Fatal(err) - } - - // Write 6 x 20 bytes, splitting out into three files - batch := f.newBatch() - require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF))) - require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE))) - - require.NoError(t, batch.AppendRaw(2, getChunk(20, 0xdd))) - require.NoError(t, batch.AppendRaw(3, getChunk(20, 0xcc))) - - require.NoError(t, batch.AppendRaw(4, getChunk(20, 0xbb))) - require.NoError(t, batch.AppendRaw(5, getChunk(20, 0xaa))) - require.NoError(t, batch.commit()) - - t.Log(f.dumpIndexString(0, 100)) - f.Close() + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + if err != nil { + t.Fatal(err) } + // Write 6 x 20 bytes, splitting out into three files + batch := f.newBatch() + require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF))) + require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE))) + + require.NoError(t, batch.AppendRaw(2, getChunk(20, 0xdd))) + require.NoError(t, batch.AppendRaw(3, getChunk(20, 0xcc))) + + require.NoError(t, batch.AppendRaw(4, getChunk(20, 0xbb))) + require.NoError(t, batch.AppendRaw(5, getChunk(20, 0xaa))) + require.NoError(t, batch.commit()) + + t.Log(f.dumpIndexString(0, 100)) + // Now crop it. - { - // delete files 0 and 1 - for i := 0; i < 2; i++ { - p := filepath.Join(os.TempDir(), fmt.Sprintf("%v.%04d.rdat", fname, i)) - if err := os.Remove(p); err != nil { - t.Fatal(err) - } - } - // Read the index file - p := filepath.Join(os.TempDir(), fmt.Sprintf("%v.ridx", fname)) - indexFile, err := os.OpenFile(p, os.O_RDWR, 0644) - if err != nil { - t.Fatal(err) - } - indexBuf := make([]byte, 7*indexEntrySize) - indexFile.Read(indexBuf) - - // Update the index file, so that we store - // [ file = 2, offset = 4 ] at index zero - - zeroIndex := indexEntry{ - filenum: uint32(2), // First file is 2 - offset: uint32(4), // We have removed four items - } - buf := zeroIndex.append(nil) - - // Overwrite index zero - copy(indexBuf, buf) - - // Remove the four next indices by overwriting - copy(indexBuf[indexEntrySize:], indexBuf[indexEntrySize*5:]) - indexFile.WriteAt(indexBuf, 0) - - // Need to truncate the moved index items - indexFile.Truncate(indexEntrySize * (1 + 2)) - indexFile.Close() - } + f.truncateTail(4) + f.Close() // Now open again - { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) - if err != nil { - t.Fatal(err) - } - defer f.Close() - t.Log(f.dumpIndexString(0, 100)) - - // It should allow writing item 6. - batch := f.newBatch() - require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x99))) - require.NoError(t, batch.commit()) - - checkRetrieveError(t, f, map[uint64]error{ - 0: errOutOfBounds, - 1: errOutOfBounds, - 2: errOutOfBounds, - 3: errOutOfBounds, - }) - checkRetrieve(t, f, map[uint64][]byte{ - 4: getChunk(20, 0xbb), - 5: getChunk(20, 0xaa), - 6: getChunk(20, 0x99), - }) + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + if err != nil { + t.Fatal(err) } + t.Log(f.dumpIndexString(0, 100)) + + // It should allow writing item 6. + batch = f.newBatch() + require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x99))) + require.NoError(t, batch.commit()) + + checkRetrieveError(t, f, map[uint64]error{ + 0: errOutOfBounds, + 1: errOutOfBounds, + 2: errOutOfBounds, + 3: errOutOfBounds, + }) + checkRetrieve(t, f, map[uint64][]byte{ + 4: getChunk(20, 0xbb), + 5: getChunk(20, 0xaa), + 6: getChunk(20, 0x99), + }) + f.Close() // Edit the index again, with a much larger initial offset of 1M. { @@ -1369,45 +1329,63 @@ func TestRandom(t *testing.T) { } func TestIndexValidation(t *testing.T) { - const ( - items = 30 - dataSize = 10 - ) + const dataSize = 10 + garbage := indexEntry{ filenum: 100, offset: 200, } var cases = []struct { - offset int64 - data []byte - expItems int + write int + offset int64 + data []byte + expItems int + hasCorruption bool }{ // extend index file with zero bytes at the end { - offset: (items + 1) * indexEntrySize, + write: 5, + offset: (5 + 1) * indexEntrySize, data: make([]byte, indexEntrySize), - expItems: 30, + expItems: 5, + }, + // extend index file with unaligned zero bytes at the end + { + write: 5, + offset: (5 + 1) * indexEntrySize, + data: make([]byte, indexEntrySize*1.5), + expItems: 5, }, // write garbage in the first non-head item { + write: 5, offset: indexEntrySize, data: garbage.append(nil), expItems: 0, }, - // write garbage in the first non-head item + // write garbage in the middle { - offset: (items/2 + 1) * indexEntrySize, + write: 5, + offset: 3 * indexEntrySize, data: garbage.append(nil), - expItems: items / 2, + expItems: 2, + }, + // fulfill the first data file (but not yet advanced), the zero bytes + // at tail should be truncated. + { + write: 10, + offset: 11 * indexEntrySize, + data: garbage.append(nil), + expItems: 10, }, } for _, c := range cases { fn := fmt.Sprintf("t-%d", rand.Uint64()) - f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 100, true, false) + f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 10*dataSize, true, false) if err != nil { t.Fatal(err) } - writeChunks(t, f, items, dataSize) + writeChunks(t, f, c.write, dataSize) // write corrupted data f.index.WriteAt(c.data, c.offset) @@ -1421,10 +1399,10 @@ func TestIndexValidation(t *testing.T) { for i := 0; i < c.expItems; i++ { exp := getChunk(10, i) got, err := f.Retrieve(uint64(i)) - if err != nil { + if err != nil && !c.hasCorruption { t.Fatalf("Failed to read from table, %v", err) } - if !bytes.Equal(exp, got) { + if !bytes.Equal(exp, got) && !c.hasCorruption { t.Fatalf("Unexpected item data, want: %v, got: %v", exp, got) } } @@ -1433,3 +1411,163 @@ func TestIndexValidation(t *testing.T) { } } } + +// TestFlushOffsetTracking tests the flush offset tracking. The offset moving +// in the test is mostly triggered by the advanceHead (new data file) and +// heda/tail truncation. +func TestFlushOffsetTracking(t *testing.T) { + const ( + items = 35 + dataSize = 10 + fileSize = 100 + ) + fn := fmt.Sprintf("t-%d", rand.Uint64()) + f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false) + if err != nil { + t.Fatal(err) + } + // Data files: + // F1(10 items) -> F2(10 items) -> F3(10 items) -> F4(5 items, non-full) + writeChunks(t, f, items, dataSize) + + var cases = []struct { + op func(*freezerTable) + offset int64 + }{ + { + // Data files: + // F1(10 items) -> F2(10 items) -> F3(10 items) -> F4(5 items, non-full) + func(f *freezerTable) {}, // no-op + 31 * indexEntrySize, + }, + { + // Write more items to fulfill the newest data file, but the file advance + // is not triggered. + + // Data files: + // F1(10 items) -> F2(10 items) -> F3(10 items) -> F4(10 items, full) + func(f *freezerTable) { + batch := f.newBatch() + for i := 0; i < 5; i++ { + batch.AppendRaw(items+uint64(i), make([]byte, dataSize)) + } + batch.commit() + }, + 31 * indexEntrySize, + }, + { + // Write more items to trigger the data file advance + + // Data files: + // F1(10 items) -> F2(10 items) -> F3(10 items) -> F4(10 items) -> F5(1 item) + func(f *freezerTable) { + batch := f.newBatch() + batch.AppendRaw(items+5, make([]byte, dataSize)) + batch.commit() + }, + 41 * indexEntrySize, + }, + { + // Head truncate + + // Data files: + // F1(10 items) -> F2(10 items) -> F3(10 items) -> F4(10 items) -> F5(0 item) + func(f *freezerTable) { + f.truncateHead(items + 5) + }, + 41 * indexEntrySize, + }, + { + // Tail truncate + + // Data files: + // F1(1 hidden, 9 visible) -> F2(10 items) -> F3(10 items) -> F4(10 items) -> F5(0 item) + func(f *freezerTable) { + f.truncateTail(1) + }, + 41 * indexEntrySize, + }, + { + // Tail truncate + + // Data files: + // F2(10 items) -> F3(10 items) -> F4(10 items) -> F5(0 item) + func(f *freezerTable) { + f.truncateTail(10) + }, + 31 * indexEntrySize, + }, + { + // Tail truncate + + // Data files: + // F4(10 items) -> F5(0 item) + func(f *freezerTable) { + f.truncateTail(30) + }, + 11 * indexEntrySize, + }, + { + // Head truncate + + // Data files: + // F4(9 items) + func(f *freezerTable) { + f.truncateHead(items + 4) + }, + 10 * indexEntrySize, + }, + } + for _, c := range cases { + c.op(f) + if f.metadata.flushOffset != c.offset { + t.Fatalf("Unexpected index flush offset, want: %d, got: %d", c.offset, f.metadata.flushOffset) + } + } +} + +func TestTailTruncationCrash(t *testing.T) { + const ( + items = 35 + dataSize = 10 + fileSize = 100 + ) + fn := fmt.Sprintf("t-%d", rand.Uint64()) + f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false) + if err != nil { + t.Fatal(err) + } + // Data files: + // F1(10 items) -> F2(10 items) -> F3(10 items) -> F4(5 items, non-full) + writeChunks(t, f, items, dataSize) + + // The latest 5 items are not persisted yet + if f.metadata.flushOffset != 31*indexEntrySize { + t.Fatalf("Unexpected index flush offset, want: %d, got: %d", 31*indexEntrySize, f.metadata.flushOffset) + } + + f.truncateTail(5) + if f.metadata.flushOffset != 31*indexEntrySize { + t.Fatalf("Unexpected index flush offset, want: %d, got: %d", 31*indexEntrySize, f.metadata.flushOffset) + } + + // Truncate the first 10 items which results in the first data file + // being removed. The offset should be moved to 26*indexEntrySize. + f.truncateTail(10) + if f.metadata.flushOffset != 26*indexEntrySize { + t.Fatalf("Unexpected index flush offset, want: %d, got: %d", 26*indexEntrySize, f.metadata.flushOffset) + } + + // Write the offset back to 31*indexEntrySize to simulate a crash + // which occurs after truncating the index file without updating + // the offset + f.metadata.setFlushOffset(31*indexEntrySize, true) + + f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false) + if err != nil { + t.Fatal(err) + } + if f.metadata.flushOffset != 26*indexEntrySize { + t.Fatalf("Unexpected index flush offset, want: %d, got: %d", 26*indexEntrySize, f.metadata.flushOffset) + } +} From 665c8512f388bd79053dce81b3920f65eb617289 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 4 Feb 2025 14:22:30 +0100 Subject: [PATCH 17/24] core: copy genesis before modifying (#31097) This PR fixes a data race in SetupGenesisWithOverride. --- core/genesis.go | 45 +++++++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/core/genesis.go b/core/genesis.go index b579e33c77..d1ee1b6e92 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -75,6 +75,19 @@ type Genesis struct { BlobGasUsed *uint64 `json:"blobGasUsed"` // EIP-4844 } +// copy copies the genesis. +func (g *Genesis) copy() *Genesis { + if g != nil { + cpy := *g + if g.Config != nil { + conf := *g.Config + cpy.Config = &conf + } + return &cpy + } + return nil +} + func ReadGenesis(db ethdb.Database) (*Genesis, error) { var genesis Genesis stored := rawdb.ReadCanonicalHash(db, 0) @@ -248,21 +261,17 @@ type ChainOverrides struct { } // apply applies the chain overrides on the supplied chain config. -func (o *ChainOverrides) apply(cfg *params.ChainConfig) (*params.ChainConfig, error) { +func (o *ChainOverrides) apply(cfg *params.ChainConfig) error { if o == nil || cfg == nil { - return cfg, nil + return nil } - cpy := *cfg if o.OverrideCancun != nil { - cpy.CancunTime = o.OverrideCancun + cfg.CancunTime = o.OverrideCancun } if o.OverrideVerkle != nil { - cpy.VerkleTime = o.OverrideVerkle + cfg.VerkleTime = o.OverrideVerkle } - if err := cpy.CheckConfigForkOrder(); err != nil { - return nil, err - } - return &cpy, nil + return cfg.CheckConfigForkOrder() } // SetupGenesisBlock writes or updates the genesis block in db. @@ -281,6 +290,8 @@ func SetupGenesisBlock(db ethdb.Database, triedb *triedb.Database, genesis *Gene } func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, genesis *Genesis, overrides *ChainOverrides) (*params.ChainConfig, common.Hash, *params.ConfigCompatError, error) { + // Copy the genesis, so we can operate on a copy. + genesis = genesis.copy() // Sanitize the supplied genesis, ensuring it has the associated chain // config attached. if genesis != nil && genesis.Config == nil { @@ -295,17 +306,15 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g } else { log.Info("Writing custom genesis block") } - chainCfg, err := overrides.apply(genesis.Config) - if err != nil { + if err := overrides.apply(genesis.Config); err != nil { return nil, common.Hash{}, nil, err } - genesis.Config = chainCfg block, err := genesis.Commit(db, triedb) if err != nil { return nil, common.Hash{}, nil, err } - return chainCfg, block.Hash(), nil, nil + return genesis.Config, block.Hash(), nil, nil } // Commit the genesis if the genesis block exists in the ancient database // but the key-value database is empty without initializing the genesis @@ -322,11 +331,9 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g } else { log.Info("Writing custom genesis block") } - chainCfg, err := overrides.apply(genesis.Config) - if err != nil { + if err := overrides.apply(genesis.Config); err != nil { return nil, common.Hash{}, nil, err } - genesis.Config = chainCfg if hash := genesis.ToBlock().Hash(); hash != ghash { return nil, common.Hash{}, nil, &GenesisMismatchError{ghash, hash} @@ -335,17 +342,15 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *triedb.Database, g if err != nil { return nil, common.Hash{}, nil, err } - return chainCfg, block.Hash(), nil, nil + return genesis.Config, block.Hash(), nil, nil } // The genesis block has already been committed previously. Verify that the // provided genesis with chain overrides matches the existing one, and update // the stored chain config if necessary. if genesis != nil { - chainCfg, err := overrides.apply(genesis.Config) - if err != nil { + if err := overrides.apply(genesis.Config); err != nil { return nil, common.Hash{}, nil, err } - genesis.Config = chainCfg if hash := genesis.ToBlock().Hash(); hash != ghash { return nil, common.Hash{}, nil, &GenesisMismatchError{ghash, hash} From eee868226a3d4504994fa176ea7c866b5d8f9d18 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 4 Feb 2025 15:29:51 +0100 Subject: [PATCH 18/24] params: start osaka fork (#31125) This PR defines the Osaka fork. An easy first step to start our work on the next hardfork (This is needed for EOF testing as well) --------- Co-authored-by: lightclient <14004106+lightclient@users.noreply.github.com> --- core/genesis_test.go | 1 + core/vm/jump_table_export.go | 4 +++- eth/tracers/api.go | 4 ++++ params/config.go | 23 ++++++++++++++++++- params/forks/forks.go | 1 + tests/init.go | 44 ++++++++++++++++++++++++++++++++++++ 6 files changed, 75 insertions(+), 2 deletions(-) diff --git a/core/genesis_test.go b/core/genesis_test.go index 4e7662d14c..714dc4d6b3 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -272,6 +272,7 @@ func TestVerkleGenesisCommit(t *testing.T) { ShanghaiTime: &verkleTime, CancunTime: &verkleTime, PragueTime: &verkleTime, + OsakaTime: &verkleTime, VerkleTime: &verkleTime, TerminalTotalDifficulty: big.NewInt(0), EnableVerkleAtGenesis: true, diff --git a/core/vm/jump_table_export.go b/core/vm/jump_table_export.go index b74109da0a..b8fa6049bb 100644 --- a/core/vm/jump_table_export.go +++ b/core/vm/jump_table_export.go @@ -28,8 +28,10 @@ func LookupInstructionSet(rules params.Rules) (JumpTable, error) { switch { case rules.IsVerkle: return newCancunInstructionSet(), errors.New("verkle-fork not defined yet") + case rules.IsOsaka: + return newPragueInstructionSet(), errors.New("osaka-fork not defined yet") case rules.IsPrague: - return newCancunInstructionSet(), errors.New("prague-fork not defined yet") + return newPragueInstructionSet(), nil case rules.IsCancun: return newCancunInstructionSet(), nil case rules.IsShanghai: diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 22163030de..d13aee555f 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -1088,6 +1088,10 @@ func overrideConfig(original *params.ChainConfig, override *params.ChainConfig) copy.PragueTime = timestamp canon = false } + if timestamp := override.OsakaTime; timestamp != nil { + copy.OsakaTime = timestamp + canon = false + } if timestamp := override.VerkleTime; timestamp != nil { copy.VerkleTime = timestamp canon = false diff --git a/params/config.go b/params/config.go index f1e139608c..9269cb2310 100644 --- a/params/config.go +++ b/params/config.go @@ -133,6 +133,7 @@ var ( ShanghaiTime: nil, CancunTime: nil, PragueTime: nil, + OsakaTime: nil, VerkleTime: nil, Ethash: new(EthashConfig), Clique: nil, @@ -182,6 +183,7 @@ var ( ShanghaiTime: nil, CancunTime: nil, PragueTime: nil, + OsakaTime: nil, VerkleTime: nil, TerminalTotalDifficulty: big.NewInt(math.MaxInt64), Ethash: nil, @@ -211,6 +213,7 @@ var ( ShanghaiTime: nil, CancunTime: nil, PragueTime: nil, + OsakaTime: nil, VerkleTime: nil, TerminalTotalDifficulty: big.NewInt(math.MaxInt64), Ethash: new(EthashConfig), @@ -240,6 +243,7 @@ var ( ShanghaiTime: newUint64(0), CancunTime: newUint64(0), PragueTime: newUint64(0), + OsakaTime: nil, VerkleTime: nil, TerminalTotalDifficulty: big.NewInt(0), Ethash: new(EthashConfig), @@ -269,6 +273,7 @@ var ( ShanghaiTime: nil, CancunTime: nil, PragueTime: nil, + OsakaTime: nil, VerkleTime: nil, TerminalTotalDifficulty: big.NewInt(math.MaxInt64), Ethash: new(EthashConfig), @@ -318,6 +323,7 @@ type ChainConfig struct { ShanghaiTime *uint64 `json:"shanghaiTime,omitempty"` // Shanghai switch time (nil = no fork, 0 = already on shanghai) CancunTime *uint64 `json:"cancunTime,omitempty"` // Cancun switch time (nil = no fork, 0 = already on cancun) PragueTime *uint64 `json:"pragueTime,omitempty"` // Prague switch time (nil = no fork, 0 = already on prague) + OsakaTime *uint64 `json:"osakaTime,omitempty"` // Osaka switch time (nil = no fork, 0 = already on osaka) VerkleTime *uint64 `json:"verkleTime,omitempty"` // Verkle switch time (nil = no fork, 0 = already on verkle) // TerminalTotalDifficulty is the amount of total difficulty reached by @@ -432,6 +438,9 @@ func (c *ChainConfig) Description() string { if c.PragueTime != nil { banner += fmt.Sprintf(" - Prague: @%-10v\n", *c.PragueTime) } + if c.OsakaTime != nil { + banner += fmt.Sprintf(" - Osaka: @%-10v\n", *c.OsakaTime) + } if c.VerkleTime != nil { banner += fmt.Sprintf(" - Verkle: @%-10v\n", *c.VerkleTime) } @@ -533,6 +542,11 @@ func (c *ChainConfig) IsPrague(num *big.Int, time uint64) bool { return c.IsLondon(num) && isTimestampForked(c.PragueTime, time) } +// IsOsaka returns whether time is either equal to the Osaka fork time or greater. +func (c *ChainConfig) IsOsaka(num *big.Int, time uint64) bool { + return c.IsLondon(num) && isTimestampForked(c.OsakaTime, time) +} + // IsVerkle returns whether time is either equal to the Verkle fork time or greater. func (c *ChainConfig) IsVerkle(num *big.Int, time uint64) bool { return c.IsLondon(num) && isTimestampForked(c.VerkleTime, time) @@ -611,6 +625,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { {name: "shanghaiTime", timestamp: c.ShanghaiTime}, {name: "cancunTime", timestamp: c.CancunTime, optional: true}, {name: "pragueTime", timestamp: c.PragueTime, optional: true}, + {name: "osakaTime", timestamp: c.OsakaTime, optional: true}, {name: "verkleTime", timestamp: c.VerkleTime, optional: true}, } { if lastFork.name != "" { @@ -715,6 +730,9 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, headNumber *big.Int, if isForkTimestampIncompatible(c.PragueTime, newcfg.PragueTime, headTimestamp) { return newTimestampCompatError("Prague fork timestamp", c.PragueTime, newcfg.PragueTime) } + if isForkTimestampIncompatible(c.OsakaTime, newcfg.OsakaTime, headTimestamp) { + return newTimestampCompatError("Osaka fork timestamp", c.OsakaTime, newcfg.OsakaTime) + } if isForkTimestampIncompatible(c.VerkleTime, newcfg.VerkleTime, headTimestamp) { return newTimestampCompatError("Verkle fork timestamp", c.VerkleTime, newcfg.VerkleTime) } @@ -737,6 +755,8 @@ func (c *ChainConfig) LatestFork(time uint64) forks.Fork { london := c.LondonBlock switch { + case c.IsOsaka(london, time): + return forks.Osaka case c.IsPrague(london, time): return forks.Prague case c.IsCancun(london, time): @@ -888,7 +908,7 @@ type Rules struct { IsEIP2929, IsEIP4762 bool IsByzantium, IsConstantinople, IsPetersburg, IsIstanbul bool IsBerlin, IsLondon bool - IsMerge, IsShanghai, IsCancun, IsPrague bool + IsMerge, IsShanghai, IsCancun, IsPrague, IsOsaka bool IsVerkle bool } @@ -918,6 +938,7 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules IsShanghai: isMerge && c.IsShanghai(num, timestamp), IsCancun: isMerge && c.IsCancun(num, timestamp), IsPrague: isMerge && c.IsPrague(num, timestamp), + IsOsaka: isMerge && c.IsOsaka(num, timestamp), IsVerkle: isVerkle, IsEIP4762: isVerkle, } diff --git a/params/forks/forks.go b/params/forks/forks.go index 4f50ff5aed..2d44e13b04 100644 --- a/params/forks/forks.go +++ b/params/forks/forks.go @@ -39,4 +39,5 @@ const ( Shanghai Cancun Prague + Osaka ) diff --git a/tests/init.go b/tests/init.go index 4bb83f9300..8429f38e44 100644 --- a/tests/init.go +++ b/tests/init.go @@ -396,6 +396,50 @@ var Forks = map[string]*params.ChainConfig{ PragueTime: u64(15_000), DepositContractAddress: params.MainnetChainConfig.DepositContractAddress, }, + "Osaka": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + MergeNetsplitBlock: big.NewInt(0), + TerminalTotalDifficulty: big.NewInt(0), + ShanghaiTime: u64(0), + CancunTime: u64(0), + PragueTime: u64(0), + OsakaTime: u64(0), + DepositContractAddress: params.MainnetChainConfig.DepositContractAddress, + }, + "PragueToOsakaAtTime15k": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + MergeNetsplitBlock: big.NewInt(0), + TerminalTotalDifficulty: big.NewInt(0), + ShanghaiTime: u64(0), + CancunTime: u64(0), + PragueTime: u64(0), + OsakaTime: u64(15_000), + DepositContractAddress: params.MainnetChainConfig.DepositContractAddress, + }, } // AvailableForks returns the set of defined fork names From e6f3ce7b168b8f346de621a8f60d2fa57c2ebfb0 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 4 Feb 2025 07:43:18 -0700 Subject: [PATCH 19/24] params,core: add max and target value to chain config (#31002) Implements [EIP-7840](https://github.com/ethereum/EIPs/pull/9129) and [EIP-7691](https://github.com/ethereum/EIPs/blob/d96625a4dcbbe2572fa006f062bd02b4582eefd5/EIPS/eip-7691.md). --------- Co-authored-by: Marius van der Wijden Co-authored-by: Felix Lange --- cmd/devp2p/internal/ethtest/suite.go | 2 +- .../internal/ethtest/testdata/genesis.json | 11 +- cmd/evm/internal/t8ntool/execution.go | 22 ++- consensus/beacon/consensus.go | 2 +- consensus/misc/eip4844/eip4844.go | 110 +++++++++--- consensus/misc/eip4844/eip4844_test.go | 38 +++-- core/blockchain.go | 5 +- core/chain_makers.go | 18 +- core/chain_makers_test.go | 6 +- core/evm.go | 6 +- core/genesis_test.go | 5 + core/rawdb/accessors_chain.go | 2 +- core/state_processor_test.go | 27 +-- core/txpool/blobpool/blobpool.go | 22 +-- core/txpool/blobpool/blobpool_test.go | 158 +++++++++++++++++- core/txpool/blobpool/evictheap_test.go | 34 ++-- core/txpool/blobpool/limbo.go | 4 +- core/txpool/blobpool/priority_test.go | 6 +- core/txpool/blobpool/slotter.go | 4 +- core/txpool/blobpool/slotter_test.go | 2 +- core/txpool/validation.go | 6 +- core/verkle_witness_test.go | 8 +- core/vm/runtime/runtime_test.go | 4 + eth/api_backend.go | 2 +- eth/catalyst/api_test.go | 3 + eth/fetcher/tx_fetcher_test.go | 6 +- eth/gasprice/feehistory.go | 10 +- eth/gasprice/gasprice_test.go | 1 + eth/tracers/internal/tracetest/supply_test.go | 1 + .../testdata/call_tracer/blob_tx.json | 9 +- .../testdata/prestate_tracer/blob_tx.json | 13 +- .../testdata/prestate_tracer/setcode_tx.json | 14 +- eth/tracers/internal/tracetest/util.go | 6 +- eth/tracers/live/supply.go | 26 +-- ethclient/ethclient_test.go | 9 +- internal/ethapi/api.go | 5 + internal/ethapi/simulate.go | 8 +- internal/ethapi/transaction_args.go | 22 +-- internal/ethapi/transaction_args_test.go | 1 + miner/worker.go | 29 ++-- params/config.go | 96 ++++++++++- params/protocol_params.go | 4 - tests/init.go | 14 ++ tests/state_test.go | 2 +- tests/state_test_util.go | 34 +++- 45 files changed, 613 insertions(+), 204 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 12dc5711ac..2f1731b60c 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -743,7 +743,7 @@ func (s *Suite) makeBlobTxs(count, blobs int, discriminator byte) (txs types.Tra GasTipCap: uint256.NewInt(1), GasFeeCap: uint256.MustFromBig(s.chain.Head().BaseFee()), Gas: 100000, - BlobFeeCap: uint256.MustFromBig(eip4844.CalcBlobFee(*s.chain.Head().ExcessBlobGas())), + BlobFeeCap: uint256.MustFromBig(eip4844.CalcBlobFee(s.chain.config, s.chain.Head().Header())), BlobHashes: makeSidecar(blobdata...).BlobHashes(), Sidecar: makeSidecar(blobdata...), } diff --git a/cmd/devp2p/internal/ethtest/testdata/genesis.json b/cmd/devp2p/internal/ethtest/testdata/genesis.json index 4cfebdcac1..ace2f78815 100644 --- a/cmd/devp2p/internal/ethtest/testdata/genesis.json +++ b/cmd/devp2p/internal/ethtest/testdata/genesis.json @@ -18,7 +18,14 @@ "shanghaiTime": 780, "cancunTime": 840, "terminalTotalDifficulty": 9454784, - "ethash": {} + "ethash": {}, + "blobSchedule": { + "cancun": { + "target": 3, + "max": 6, + "baseFeeUpdateFraction": 3338477 + } + } }, "nonce": "0x0", "timestamp": "0x0", @@ -108,4 +115,4 @@ "baseFeePerGas": null, "excessBlobGas": null, "blobGasUsed": null -} \ No newline at end of file +} diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 9ff5a05290..9332f4901b 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -178,15 +178,28 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, var excessBlobGas uint64 if pre.Env.ExcessBlobGas != nil { excessBlobGas = *pre.Env.ExcessBlobGas - vmContext.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas) + header := &types.Header{ + Time: pre.Env.Timestamp, + ExcessBlobGas: pre.Env.ExcessBlobGas, + } + vmContext.BlobBaseFee = eip4844.CalcBlobFee(chainConfig, header) } else { // If it is not explicitly defined, but we have the parent values, we try // to calculate it ourselves. parentExcessBlobGas := pre.Env.ParentExcessBlobGas parentBlobGasUsed := pre.Env.ParentBlobGasUsed if parentExcessBlobGas != nil && parentBlobGasUsed != nil { - excessBlobGas = eip4844.CalcExcessBlobGas(*parentExcessBlobGas, *parentBlobGasUsed) - vmContext.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas) + parent := &types.Header{ + Time: pre.Env.ParentTimestamp, + ExcessBlobGas: pre.Env.ParentExcessBlobGas, + BlobGasUsed: pre.Env.ParentBlobGasUsed, + } + excessBlobGas = eip4844.CalcExcessBlobGas(chainConfig, parent) + header := &types.Header{ + Time: pre.Env.Timestamp, + ExcessBlobGas: &excessBlobGas, + } + vmContext.BlobBaseFee = eip4844.CalcBlobFee(chainConfig, header) } } // If DAO is supported/enabled, we need to handle it here. In geth 'proper', it's @@ -229,7 +242,8 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, txBlobGas := uint64(0) if tx.Type() == types.BlobTxType { txBlobGas = uint64(params.BlobTxBlobGasPerBlob * len(tx.BlobHashes())) - if used, max := blobGasUsed+txBlobGas, uint64(params.MaxBlobGasPerBlock); used > max { + max := eip4844.MaxBlobGasPerBlock(chainConfig, pre.Env.Timestamp) + if used := blobGasUsed + txBlobGas; used > max { err := fmt.Errorf("blob gas (%d) would exceed maximum allowance %d", used, max) log.Warn("rejected tx", "index", i, "err", err) rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()}) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 20df6d0091..bdfddfff06 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -282,7 +282,7 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa if header.ParentBeaconRoot == nil { return errors.New("header is missing beaconRoot") } - if err := eip4844.VerifyEIP4844Header(parent, header); err != nil { + if err := eip4844.VerifyEIP4844Header(chain.Config(), parent, header); err != nil { return err } } diff --git a/consensus/misc/eip4844/eip4844.go b/consensus/misc/eip4844/eip4844.go index 2dad9a0cd3..1d76b21b30 100644 --- a/consensus/misc/eip4844/eip4844.go +++ b/consensus/misc/eip4844/eip4844.go @@ -23,17 +23,17 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/params/forks" ) var ( - minBlobGasPrice = big.NewInt(params.BlobTxMinBlobGasprice) - blobGaspriceUpdateFraction = big.NewInt(params.BlobTxBlobGaspriceUpdateFraction) + minBlobGasPrice = big.NewInt(params.BlobTxMinBlobGasprice) ) // VerifyEIP4844Header verifies the presence of the excessBlobGas field and that // if the current block contains no transactions, the excessBlobGas is updated // accordingly. -func VerifyEIP4844Header(parent, header *types.Header) error { +func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Header) error { // Verify the header is not malformed if header.ExcessBlobGas == nil { return errors.New("header is missing excessBlobGas") @@ -42,14 +42,26 @@ func VerifyEIP4844Header(parent, header *types.Header) error { return errors.New("header is missing blobGasUsed") } // Verify that the blob gas used remains within reasonable limits. - if *header.BlobGasUsed > params.MaxBlobGasPerBlock { - return fmt.Errorf("blob gas used %d exceeds maximum allowance %d", *header.BlobGasUsed, params.MaxBlobGasPerBlock) + maxBlobGas := MaxBlobGasPerBlock(config, header.Time) + if *header.BlobGasUsed > maxBlobGas { + return fmt.Errorf("blob gas used %d exceeds maximum allowance %d", *header.BlobGasUsed, maxBlobGas) } if *header.BlobGasUsed%params.BlobTxBlobGasPerBlob != 0 { return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", header.BlobGasUsed, params.BlobTxBlobGasPerBlob) } // Verify the excessBlobGas is correct based on the parent header + expectedExcessBlobGas := CalcExcessBlobGas(config, parent) + if *header.ExcessBlobGas != expectedExcessBlobGas { + return fmt.Errorf("invalid excessBlobGas: have %d, want %d", *header.ExcessBlobGas, expectedExcessBlobGas) + } + return nil +} + +// CalcExcessBlobGas calculates the excess blob gas after applying the set of +// blobs on top of the excess blob gas. +func CalcExcessBlobGas(config *params.ChainConfig, parent *types.Header) uint64 { var ( + targetGas = uint64(targetBlobsPerBlock(config, parent.Time)) * params.BlobTxBlobGasPerBlob parentExcessBlobGas uint64 parentBlobGasUsed uint64 ) @@ -57,27 +69,85 @@ func VerifyEIP4844Header(parent, header *types.Header) error { parentExcessBlobGas = *parent.ExcessBlobGas parentBlobGasUsed = *parent.BlobGasUsed } - expectedExcessBlobGas := CalcExcessBlobGas(parentExcessBlobGas, parentBlobGasUsed) - if *header.ExcessBlobGas != expectedExcessBlobGas { - return fmt.Errorf("invalid excessBlobGas: have %d, want %d, parent excessBlobGas %d, parent blobDataUsed %d", - *header.ExcessBlobGas, expectedExcessBlobGas, parentExcessBlobGas, parentBlobGasUsed) - } - return nil -} - -// CalcExcessBlobGas calculates the excess blob gas after applying the set of -// blobs on top of the excess blob gas. -func CalcExcessBlobGas(parentExcessBlobGas uint64, parentBlobGasUsed uint64) uint64 { excessBlobGas := parentExcessBlobGas + parentBlobGasUsed - if excessBlobGas < params.BlobTxTargetBlobGasPerBlock { + if excessBlobGas < targetGas { return 0 } - return excessBlobGas - params.BlobTxTargetBlobGasPerBlock + return excessBlobGas - targetGas } // CalcBlobFee calculates the blobfee from the header's excess blob gas field. -func CalcBlobFee(excessBlobGas uint64) *big.Int { - return fakeExponential(minBlobGasPrice, new(big.Int).SetUint64(excessBlobGas), blobGaspriceUpdateFraction) +func CalcBlobFee(config *params.ChainConfig, header *types.Header) *big.Int { + var frac uint64 + switch config.LatestFork(header.Time) { + case forks.Prague: + frac = config.BlobScheduleConfig.Prague.UpdateFraction + case forks.Cancun: + frac = config.BlobScheduleConfig.Cancun.UpdateFraction + default: + panic("calculating blob fee on unsupported fork") + } + return fakeExponential(minBlobGasPrice, new(big.Int).SetUint64(*header.ExcessBlobGas), new(big.Int).SetUint64(frac)) +} + +// MaxBlobsPerBlock returns the max blobs per block for a block at the given timestamp. +func MaxBlobsPerBlock(cfg *params.ChainConfig, time uint64) int { + if cfg.BlobScheduleConfig == nil { + return 0 + } + var ( + london = cfg.LondonBlock + s = cfg.BlobScheduleConfig + ) + switch { + case cfg.IsPrague(london, time) && s.Prague != nil: + return s.Prague.Max + case cfg.IsCancun(london, time) && s.Cancun != nil: + return s.Cancun.Max + default: + return 0 + } +} + +// MaxBlobsPerBlock returns the maximum blob gas that can be spent in a block at the given timestamp. +func MaxBlobGasPerBlock(cfg *params.ChainConfig, time uint64) uint64 { + return uint64(MaxBlobsPerBlock(cfg, time)) * params.BlobTxBlobGasPerBlob +} + +// LatestMaxBlobsPerBlock returns the latest max blobs per block defined by the +// configuration, regardless of the currently active fork. +func LatestMaxBlobsPerBlock(cfg *params.ChainConfig) int { + s := cfg.BlobScheduleConfig + if s == nil { + return 0 + } + switch { + case s.Prague != nil: + return s.Prague.Max + case s.Cancun != nil: + return s.Cancun.Max + default: + return 0 + } +} + +// targetBlobsPerBlock returns the target number of blobs in a block at the given timestamp. +func targetBlobsPerBlock(cfg *params.ChainConfig, time uint64) int { + if cfg.BlobScheduleConfig == nil { + return 0 + } + var ( + london = cfg.LondonBlock + s = cfg.BlobScheduleConfig + ) + switch { + case cfg.IsPrague(london, time) && s.Prague != nil: + return s.Prague.Target + case cfg.IsCancun(london, time) && s.Cancun != nil: + return s.Cancun.Target + default: + return 0 + } } // fakeExponential approximates factor * e ** (numerator / denominator) using diff --git a/consensus/misc/eip4844/eip4844_test.go b/consensus/misc/eip4844/eip4844_test.go index ec417380fc..839ea8df22 100644 --- a/consensus/misc/eip4844/eip4844_test.go +++ b/consensus/misc/eip4844/eip4844_test.go @@ -21,36 +21,48 @@ import ( "math/big" "testing" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" ) func TestCalcExcessBlobGas(t *testing.T) { + var ( + config = params.MainnetChainConfig + targetBlobs = targetBlobsPerBlock(config, *config.CancunTime) + targetBlobGas = uint64(targetBlobs) * params.BlobTxBlobGasPerBlob + ) var tests = []struct { excess uint64 - blobs uint64 + blobs int want uint64 }{ // The excess blob gas should not increase from zero if the used blob // slots are below - or equal - to the target. {0, 0, 0}, {0, 1, 0}, - {0, params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob, 0}, + {0, targetBlobs, 0}, // If the target blob gas is exceeded, the excessBlobGas should increase // by however much it was overshot - {0, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) + 1, params.BlobTxBlobGasPerBlob}, - {1, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) + 1, params.BlobTxBlobGasPerBlob + 1}, - {1, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) + 2, 2*params.BlobTxBlobGasPerBlob + 1}, + {0, targetBlobs + 1, params.BlobTxBlobGasPerBlob}, + {1, targetBlobs + 1, params.BlobTxBlobGasPerBlob + 1}, + {1, targetBlobs + 2, 2*params.BlobTxBlobGasPerBlob + 1}, // The excess blob gas should decrease by however much the target was // under-shot, capped at zero. - {params.BlobTxTargetBlobGasPerBlock, params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob, params.BlobTxTargetBlobGasPerBlock}, - {params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, params.BlobTxTargetBlobGasPerBlock - params.BlobTxBlobGasPerBlob}, - {params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 2, params.BlobTxTargetBlobGasPerBlock - (2 * params.BlobTxBlobGasPerBlob)}, - {params.BlobTxBlobGasPerBlob - 1, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, 0}, + {targetBlobGas, targetBlobs, targetBlobGas}, + {targetBlobGas, targetBlobs - 1, targetBlobGas - params.BlobTxBlobGasPerBlob}, + {targetBlobGas, targetBlobs - 2, targetBlobGas - (2 * params.BlobTxBlobGasPerBlob)}, + {params.BlobTxBlobGasPerBlob - 1, targetBlobs - 1, 0}, } for i, tt := range tests { - result := CalcExcessBlobGas(tt.excess, tt.blobs*params.BlobTxBlobGasPerBlob) + blobGasUsed := uint64(tt.blobs) * params.BlobTxBlobGasPerBlob + parent := &types.Header{ + Time: *config.CancunTime, + ExcessBlobGas: &tt.excess, + BlobGasUsed: &blobGasUsed, + } + result := CalcExcessBlobGas(config, parent) if result != tt.want { t.Errorf("test %d: excess blob gas mismatch: have %v, want %v", i, result, tt.want) } @@ -58,6 +70,8 @@ func TestCalcExcessBlobGas(t *testing.T) { } func TestCalcBlobFee(t *testing.T) { + zero := uint64(0) + tests := []struct { excessBlobGas uint64 blobfee int64 @@ -68,7 +82,9 @@ func TestCalcBlobFee(t *testing.T) { {10 * 1024 * 1024, 23}, } for i, tt := range tests { - have := CalcBlobFee(tt.excessBlobGas) + config := ¶ms.ChainConfig{LondonBlock: big.NewInt(0), CancunTime: &zero, BlobScheduleConfig: params.DefaultBlobSchedule} + header := &types.Header{ExcessBlobGas: &tt.excessBlobGas} + have := CalcBlobFee(config, header) if have.Int64() != tt.blobfee { t.Errorf("test %d: blobfee mismatch: have %v want %v", i, have, tt.blobfee) } diff --git a/core/blockchain.go b/core/blockchain.go index f097dc9781..b98c2d43aa 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2139,9 +2139,8 @@ func (bc *BlockChain) recoverAncestors(block *types.Block, makeWitness bool) (co // processing of a block. These logs are later announced as deleted or reborn. func (bc *BlockChain) collectLogs(b *types.Block, removed bool) []*types.Log { var blobGasPrice *big.Int - excessBlobGas := b.ExcessBlobGas() - if excessBlobGas != nil { - blobGasPrice = eip4844.CalcBlobFee(*excessBlobGas) + if b.ExcessBlobGas() != nil { + blobGasPrice = eip4844.CalcBlobFee(bc.chainConfig, b.Header()) } receipts := rawdb.ReadRawReceipts(bc.db, b.Hash(), b.NumberU64()) if err := receipts.DeriveFields(bc.chainConfig, b.Hash(), b.NumberU64(), b.Time(), b.BaseFee(), blobGasPrice, b.Transactions()); err != nil { diff --git a/core/chain_makers.go b/core/chain_makers.go index 1f48f9d128..19f433d7b0 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -143,7 +143,9 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti // instruction will panic during execution if it attempts to access a block number outside // of the range created by GenerateChain. func (b *BlockGen) AddTx(tx *types.Transaction) { - b.addTx(nil, vm.Config{}, tx) + // Wrap the chain config in an empty BlockChain object to satisfy ChainContext. + bc := &BlockChain{chainConfig: b.cm.config} + b.addTx(bc, vm.Config{}, tx) } // AddTxWithChain adds a transaction to the generated block. If no coinbase has @@ -445,7 +447,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } var blobGasPrice *big.Int if block.ExcessBlobGas() != nil { - blobGasPrice = eip4844.CalcBlobFee(*block.ExcessBlobGas()) + blobGasPrice = eip4844.CalcBlobFee(cm.config, block.Header()) } if err := receipts.DeriveFields(config, block.Hash(), block.NumberU64(), block.Time(), block.BaseFee(), blobGasPrice, txs); err != nil { panic(err) @@ -548,7 +550,7 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine } var blobGasPrice *big.Int if block.ExcessBlobGas() != nil { - blobGasPrice = eip4844.CalcBlobFee(*block.ExcessBlobGas()) + blobGasPrice = eip4844.CalcBlobFee(cm.config, block.Header()) } if err := receipts.DeriveFields(config, block.Hash(), block.NumberU64(), block.Time(), block.BaseFee(), blobGasPrice, txs); err != nil { panic(err) @@ -598,15 +600,7 @@ func (cm *chainMaker) makeHeader(parent *types.Block, state *state.StateDB, engi } } if cm.config.IsCancun(header.Number, header.Time) { - var ( - parentExcessBlobGas uint64 - parentBlobGasUsed uint64 - ) - if parent.ExcessBlobGas() != nil { - parentExcessBlobGas = *parent.ExcessBlobGas() - parentBlobGasUsed = *parent.BlobGasUsed() - } - excessBlobGas := eip4844.CalcExcessBlobGas(parentExcessBlobGas, parentBlobGasUsed) + excessBlobGas := eip4844.CalcExcessBlobGas(cm.config, parent.Header()) header.ExcessBlobGas = &excessBlobGas header.BlobGasUsed = new(uint64) header.ParentBeaconRoot = new(common.Hash) diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go index cb1087b4b3..d81a52e915 100644 --- a/core/chain_makers_test.go +++ b/core/chain_makers_test.go @@ -42,7 +42,7 @@ func TestGeneratePOSChain(t *testing.T) { aa = common.Address{0xaa} bb = common.Address{0xbb} funds = big.NewInt(0).Mul(big.NewInt(1337), big.NewInt(params.Ether)) - config = *params.AllEthashProtocolChanges + config = *params.MergedTestChainConfig gspec = &Genesis{ Config: &config, Alloc: types.GenesisAlloc{ @@ -57,10 +57,6 @@ func TestGeneratePOSChain(t *testing.T) { db = rawdb.NewMemoryDatabase() ) - config.TerminalTotalDifficulty = common.Big0 - config.ShanghaiTime = u64(0) - config.CancunTime = u64(0) - // init 0xaa with some storage elements storage := make(map[common.Hash]common.Hash) storage[common.Hash{0x00}] = common.Hash{0x00} diff --git a/core/evm.go b/core/evm.go index 5d3c454d7c..41b4e6ac58 100644 --- a/core/evm.go +++ b/core/evm.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" ) @@ -36,6 +37,9 @@ type ChainContext interface { // GetHeader returns the header corresponding to the hash/number argument pair. GetHeader(common.Hash, uint64) *types.Header + + // Config returns the chain's configuration. + Config() *params.ChainConfig } // NewEVMBlockContext creates a new context for use in the EVM. @@ -57,7 +61,7 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common baseFee = new(big.Int).Set(header.BaseFee) } if header.ExcessBlobGas != nil { - blobBaseFee = eip4844.CalcBlobFee(*header.ExcessBlobGas) + blobBaseFee = eip4844.CalcBlobFee(chain.Config(), header) } if header.Difficulty.Sign() == 0 { random = &header.MixDigest diff --git a/core/genesis_test.go b/core/genesis_test.go index 714dc4d6b3..3e120a9307 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -278,6 +278,11 @@ func TestVerkleGenesisCommit(t *testing.T) { EnableVerkleAtGenesis: true, Ethash: nil, Clique: nil, + BlobScheduleConfig: ¶ms.BlobScheduleConfig{ + Cancun: params.DefaultCancunBlobConfig, + Prague: params.DefaultPragueBlobConfig, + Verkle: params.DefaultPragueBlobConfig, + }, } genesis := &Genesis{ diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index a586046d5f..1a2b31b360 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -587,7 +587,7 @@ func ReadReceipts(db ethdb.Reader, hash common.Hash, number uint64, time uint64, // Compute effective blob gas price. var blobGasPrice *big.Int if header != nil && header.ExcessBlobGas != nil { - blobGasPrice = eip4844.CalcBlobFee(*header.ExcessBlobGas) + blobGasPrice = eip4844.CalcBlobFee(config, header) } if err := receipts.DeriveFields(config, hash, number, time, baseFee, blobGasPrice, body.Transactions); err != nil { log.Error("Failed to derive block receipts fields", "hash", hash, "number", number, "err", err) diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 750f3708dc..7d62e90ad4 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -46,25 +46,7 @@ func u64(val uint64) *uint64 { return &val } // contain invalid transactions func TestStateProcessorErrors(t *testing.T) { var ( - config = ¶ms.ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - Ethash: new(params.EthashConfig), - TerminalTotalDifficulty: big.NewInt(0), - ShanghaiTime: new(uint64), - CancunTime: new(uint64), - PragueTime: new(uint64), - } + config = params.MergedTestChainConfig signer = types.LatestSigner(config) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") key2, _ = crypto.HexToECDSA("0202020202020202020202020202020202020202020202020202002020202020") @@ -425,12 +407,7 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr } header.Root = common.BytesToHash(hasher.Sum(nil)) if config.IsCancun(header.Number, header.Time) { - var pExcess, pUsed = uint64(0), uint64(0) - if parent.ExcessBlobGas() != nil { - pExcess = *parent.ExcessBlobGas() - pUsed = *parent.BlobGasUsed() - } - excess := eip4844.CalcExcessBlobGas(pExcess, pUsed) + excess := eip4844.CalcExcessBlobGas(config, parent.Header()) used := uint64(nBlobs * params.BlobTxBlobGasPerBlob) header.ExcessBlobGas = &excess header.BlobGasUsed = &used diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 4ab14bbcc0..78cf2c05af 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -51,11 +51,6 @@ const ( // transaction. There can be multiple of these embedded into a single tx. blobSize = params.BlobTxFieldElementsPerBlob * params.BlobTxBytesPerFieldElement - // maxBlobsPerTransaction is the maximum number of blobs a single transaction - // is allowed to contain. Whilst the spec states it's unlimited, the block - // data slots are protocol bound, which implicitly also limit this. - maxBlobsPerTransaction = params.MaxBlobGasPerBlock / params.BlobTxBlobGasPerBlob - // txAvgSize is an approximate byte size of a transaction metadata to avoid // tiny overflows causing all txs to move a shelf higher, wasting disk space. txAvgSize = 4 * 1024 @@ -223,6 +218,11 @@ func newBlobTxMeta(id uint64, size uint32, tx *types.Transaction) *blobTxMeta { // very relaxed ones can be included even if the fees go up, when the closer // ones could already be invalid. // +// - Because the maximum number of blobs allowed in a block can change per +// fork, the pool is designed to handle the maximum number of blobs allowed +// in the chain's latest defined fork -- even if it isn't active. This +// avoids needing to upgrade the database around the fork boundary. +// // When the pool eventually reaches saturation, some old transactions - that may // never execute - will need to be evicted in favor of newer ones. The eviction // strategy is quite complex: @@ -387,7 +387,8 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres fails = append(fails, id) } } - store, err := billy.Open(billy.Options{Path: queuedir, Repair: true}, newSlotter(), index) + slotter := newSlotter(eip4844.LatestMaxBlobsPerBlock(p.chain.Config())) + store, err := billy.Open(billy.Options{Path: queuedir, Repair: true}, slotter, index) if err != nil { return err } @@ -414,13 +415,13 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice) ) if p.head.ExcessBlobGas != nil { - blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(*p.head.ExcessBlobGas)) + blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(p.chain.Config(), p.head)) } p.evict = newPriceHeap(basefee, blobfee, p.index) // Pool initialized, attach the blob limbo to it to track blobs included // recently but not yet finalized - p.limbo, err = newLimbo(limbodir) + p.limbo, err = newLimbo(limbodir, eip4844.LatestMaxBlobsPerBlock(p.chain.Config())) if err != nil { p.Close() return err @@ -834,7 +835,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { blobfee = uint256.MustFromBig(big.NewInt(params.BlobTxMinBlobGasprice)) ) if newHead.ExcessBlobGas != nil { - blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(*newHead.ExcessBlobGas)) + blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(p.chain.Config(), newHead)) } p.evict.reinit(basefee, blobfee, false) @@ -1598,7 +1599,8 @@ func (p *BlobPool) updateStorageMetrics() { metrics.GetOrRegisterGauge(fmt.Sprintf(shelfSlotusedGaugeName, shelf.SlotSize/blobSize), nil).Update(int64(shelf.FilledSlots)) metrics.GetOrRegisterGauge(fmt.Sprintf(shelfSlotgapsGaugeName, shelf.SlotSize/blobSize), nil).Update(int64(shelf.GappedSlots)) - if shelf.SlotSize/blobSize > maxBlobsPerTransaction { + maxBlobs := eip4844.LatestMaxBlobsPerBlock(p.chain.Config()) + if shelf.SlotSize/blobSize > uint32(maxBlobs) { oversizedDataused += slotDataused oversizedDatagaps += slotDatagaps oversizedSlotused += shelf.FilledSlots diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 3d90ec4412..0187e8d9a0 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -51,8 +51,10 @@ var ( testBlobVHashes [][32]byte ) +const testMaxBlobsPerBlock = 6 + func init() { - for i := 0; i < 10; i++ { + for i := 0; i < 24; i++ { testBlob := &kzg4844.Blob{byte(i)} testBlobs = append(testBlobs, testBlob) @@ -121,7 +123,12 @@ func (bc *testBlockChain) CurrentBlock() *types.Header { mid := new(big.Int).Add(lo, hi) mid.Div(mid, big.NewInt(2)) - if eip4844.CalcBlobFee(mid.Uint64()).Cmp(bc.blobfee.ToBig()) > 0 { + tmp := mid.Uint64() + if eip4844.CalcBlobFee(bc.Config(), &types.Header{ + Number: blockNumber, + Time: blockTime, + ExcessBlobGas: &tmp, + }).Cmp(bc.blobfee.ToBig()) > 0 { hi = mid } else { lo = mid @@ -194,10 +201,43 @@ func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64, return types.MustSignNewTx(key, types.LatestSigner(params.MainnetChainConfig), blobtx) } +// makeMultiBlobTx is a utility method to construct a ramdom blob tx with +// certain number of blobs in its sidecar. +func makeMultiBlobTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64, blobCount int, key *ecdsa.PrivateKey) *types.Transaction { + var ( + blobs []kzg4844.Blob + blobHashes []common.Hash + commitments []kzg4844.Commitment + proofs []kzg4844.Proof + ) + for i := 0; i < blobCount; i++ { + blobs = append(blobs, *testBlobs[i]) + commitments = append(commitments, testBlobCommits[i]) + proofs = append(proofs, testBlobProofs[i]) + blobHashes = append(blobHashes, testBlobVHashes[i]) + } + blobtx := &types.BlobTx{ + ChainID: uint256.MustFromBig(params.MainnetChainConfig.ChainID), + Nonce: nonce, + GasTipCap: uint256.NewInt(gasTipCap), + GasFeeCap: uint256.NewInt(gasFeeCap), + Gas: 21000, + BlobFeeCap: uint256.NewInt(blobFeeCap), + BlobHashes: blobHashes, + Value: uint256.NewInt(100), + Sidecar: &types.BlobTxSidecar{ + Blobs: blobs, + Commitments: commitments, + Proofs: proofs, + }, + } + return types.MustSignNewTx(key, types.LatestSigner(params.MainnetChainConfig), blobtx) +} + // makeUnsignedTx is a utility method to construct a random blob transaction // without signing it. func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64) *types.BlobTx { - return makeUnsignedTxWithTestBlob(nonce, gasTipCap, gasFeeCap, blobFeeCap, rand.Intn(len(testBlobs))) + return makeUnsignedTxWithTestBlob(nonce, gasTipCap, gasFeeCap, blobFeeCap, rnd.Intn(len(testBlobs))) } // makeUnsignedTx is a utility method to construct a random blob transaction @@ -415,7 +455,7 @@ func TestOpenDrops(t *testing.T) { defer os.RemoveAll(storage) os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700) - store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(), nil) + store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(testMaxBlobsPerBlock), nil) // Insert a malformed transaction to verify that decoding errors (or format // changes) are handled gracefully (case 1) @@ -738,7 +778,7 @@ func TestOpenIndex(t *testing.T) { defer os.RemoveAll(storage) os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700) - store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(), nil) + store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(testMaxBlobsPerBlock), nil) // Insert a sequence of transactions with varying price points to check that // the cumulative minimum will be maintained. @@ -827,7 +867,7 @@ func TestOpenHeap(t *testing.T) { defer os.RemoveAll(storage) os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700) - store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(), nil) + store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(testMaxBlobsPerBlock), nil) // Insert a few transactions from a few accounts. To remove randomness from // the heap initialization, use a deterministic account/tx/priority ordering. @@ -914,7 +954,7 @@ func TestOpenCap(t *testing.T) { defer os.RemoveAll(storage) os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700) - store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(), nil) + store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(testMaxBlobsPerBlock), nil) // Insert a few transactions from a few accounts var ( @@ -992,6 +1032,108 @@ func TestOpenCap(t *testing.T) { } } +// TestChangingSlotterSize attempts to mimic a scenario where the max blob count +// of the pool is increased. This would happen during a client release where a +// new fork is added with a max blob count higher than the previous fork. We +// want to make sure transactions a persisted between those runs. +func TestChangingSlotterSize(t *testing.T) { + //log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true))) + + // Create a temporary folder for the persistent backend + storage, _ := os.MkdirTemp("", "blobpool-") + defer os.RemoveAll(storage) + + os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700) + store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(6), nil) + + // Create transactions from a few accounts. + var ( + key1, _ = crypto.GenerateKey() + key2, _ = crypto.GenerateKey() + key3, _ = crypto.GenerateKey() + + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + addr2 = crypto.PubkeyToAddress(key2.PublicKey) + addr3 = crypto.PubkeyToAddress(key3.PublicKey) + + tx1 = makeMultiBlobTx(0, 1, 1000, 100, 6, key1) + tx2 = makeMultiBlobTx(0, 1, 800, 70, 6, key2) + tx3 = makeMultiBlobTx(0, 1, 800, 110, 24, key3) + + blob1, _ = rlp.EncodeToBytes(tx1) + blob2, _ = rlp.EncodeToBytes(tx2) + ) + + // Write the two safely sized txs to store. note: although the store is + // configured for a blob count of 6, it can also support around ~1mb of call + // data - all this to say that we aren't using the the absolute largest shelf + // available. + store.Put(blob1) + store.Put(blob2) + store.Close() + + // Mimic a blobpool with max blob count of 6 upgrading to a max blob count of 24. + for _, maxBlobs := range []int{6, 24} { + statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) + statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(addr2, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) + statedb.AddBalance(addr3, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) + statedb.Commit(0, true, false) + + // Make custom chain config where the max blob count changes based on the loop variable. + cancunTime := uint64(0) + config := ¶ms.ChainConfig{ + ChainID: big.NewInt(1), + LondonBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + CancunTime: &cancunTime, + BlobScheduleConfig: ¶ms.BlobScheduleConfig{ + Cancun: ¶ms.BlobConfig{ + Target: maxBlobs / 2, + Max: maxBlobs, + UpdateFraction: params.DefaultCancunBlobConfig.UpdateFraction, + }, + }, + } + chain := &testBlockChain{ + config: config, + basefee: uint256.NewInt(1050), + blobfee: uint256.NewInt(105), + statedb: statedb, + } + pool := New(Config{Datadir: storage}, chain) + if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + t.Fatalf("failed to create blob pool: %v", err) + } + + // Try to add the big blob tx. In the initial iteration it should overflow + // the pool. On the subsequent iteration it should be accepted. + errs := pool.Add([]*types.Transaction{tx3}, false, true) + if _, ok := pool.index[addr3]; ok && maxBlobs == 6 { + t.Errorf("expected insert of oversized blob tx to fail: blobs=24, maxBlobs=%d, err=%v", maxBlobs, errs[0]) + } else if !ok && maxBlobs == 10 { + t.Errorf("expected insert of oversized blob tx to succeed: blobs=24, maxBlobs=%d, err=%v", maxBlobs, errs[0]) + } + + // Verify the regular two txs are always available. + if got := pool.Get(tx1.Hash()); got == nil { + t.Errorf("expected tx %s from %s in pool", tx1.Hash(), addr1) + } + if got := pool.Get(tx2.Hash()); got == nil { + t.Errorf("expected tx %s from %s in pool", tx2.Hash(), addr2) + } + + // Verify all the calculated pool internals. Interestingly, this is **not** + // a duplication of the above checks, this actually validates the verifier + // using the above already hard coded checks. + // + // Do not remove this, nor alter the above to be generic. + verifyPoolInternals(t, pool) + + pool.Close() + } +} + // Tests that adding transaction will correctly store it in the persistent store // and update all the indices. // @@ -1369,7 +1511,7 @@ func TestAdd(t *testing.T) { defer os.RemoveAll(storage) // late defer, still ok os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700) - store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(), nil) + store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(testMaxBlobsPerBlock), nil) // Insert the seed transactions for the pool startup var ( diff --git a/core/txpool/blobpool/evictheap_test.go b/core/txpool/blobpool/evictheap_test.go index b03dd83d69..e392932401 100644 --- a/core/txpool/blobpool/evictheap_test.go +++ b/core/txpool/blobpool/evictheap_test.go @@ -26,7 +26,7 @@ import ( "github.com/holiman/uint256" ) -var rand = mrand.New(mrand.NewSource(1)) +var rnd = mrand.New(mrand.NewSource(1)) // verifyHeapInternals verifies that all accounts present in the index are also // present in the heap and internals are consistent across various indices. @@ -193,12 +193,12 @@ func benchmarkPriceHeapReinit(b *testing.B, datacap uint64) { index := make(map[common.Address][]*blobTxMeta) for i := 0; i < int(blobs); i++ { var addr common.Address - rand.Read(addr[:]) + rnd.Read(addr[:]) var ( - execTip = uint256.NewInt(rand.Uint64()) - execFee = uint256.NewInt(rand.Uint64()) - blobFee = uint256.NewInt(rand.Uint64()) + execTip = uint256.NewInt(rnd.Uint64()) + execFee = uint256.NewInt(rnd.Uint64()) + blobFee = uint256.NewInt(rnd.Uint64()) basefeeJumps = dynamicFeeJumps(execFee) blobfeeJumps = dynamicFeeJumps(blobFee) @@ -218,13 +218,13 @@ func benchmarkPriceHeapReinit(b *testing.B, datacap uint64) { }} } // Create a price heap and reinit it over and over - heap := newPriceHeap(uint256.NewInt(rand.Uint64()), uint256.NewInt(rand.Uint64()), index) + heap := newPriceHeap(uint256.NewInt(rnd.Uint64()), uint256.NewInt(rnd.Uint64()), index) basefees := make([]*uint256.Int, b.N) blobfees := make([]*uint256.Int, b.N) for i := 0; i < b.N; i++ { - basefees[i] = uint256.NewInt(rand.Uint64()) - blobfees[i] = uint256.NewInt(rand.Uint64()) + basefees[i] = uint256.NewInt(rnd.Uint64()) + blobfees[i] = uint256.NewInt(rnd.Uint64()) } b.ResetTimer() b.ReportAllocs() @@ -269,12 +269,12 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) { index := make(map[common.Address][]*blobTxMeta) for i := 0; i < int(blobs); i++ { var addr common.Address - rand.Read(addr[:]) + rnd.Read(addr[:]) var ( - execTip = uint256.NewInt(rand.Uint64()) - execFee = uint256.NewInt(rand.Uint64()) - blobFee = uint256.NewInt(rand.Uint64()) + execTip = uint256.NewInt(rnd.Uint64()) + execFee = uint256.NewInt(rnd.Uint64()) + blobFee = uint256.NewInt(rnd.Uint64()) basefeeJumps = dynamicFeeJumps(execFee) blobfeeJumps = dynamicFeeJumps(blobFee) @@ -294,18 +294,18 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) { }} } // Create a price heap and overflow it over and over - evict := newPriceHeap(uint256.NewInt(rand.Uint64()), uint256.NewInt(rand.Uint64()), index) + evict := newPriceHeap(uint256.NewInt(rnd.Uint64()), uint256.NewInt(rnd.Uint64()), index) var ( addrs = make([]common.Address, b.N) metas = make([]*blobTxMeta, b.N) ) for i := 0; i < b.N; i++ { - rand.Read(addrs[i][:]) + rnd.Read(addrs[i][:]) var ( - execTip = uint256.NewInt(rand.Uint64()) - execFee = uint256.NewInt(rand.Uint64()) - blobFee = uint256.NewInt(rand.Uint64()) + execTip = uint256.NewInt(rnd.Uint64()) + execFee = uint256.NewInt(rnd.Uint64()) + blobFee = uint256.NewInt(rnd.Uint64()) basefeeJumps = dynamicFeeJumps(execFee) blobfeeJumps = dynamicFeeJumps(blobFee) diff --git a/core/txpool/blobpool/limbo.go b/core/txpool/blobpool/limbo.go index 32381a3936..d5992f2906 100644 --- a/core/txpool/blobpool/limbo.go +++ b/core/txpool/blobpool/limbo.go @@ -48,7 +48,7 @@ type limbo struct { } // newLimbo opens and indexes a set of limboed blob transactions. -func newLimbo(datadir string) (*limbo, error) { +func newLimbo(datadir string, maxBlobsPerTransaction int) (*limbo, error) { l := &limbo{ index: make(map[common.Hash]uint64), groups: make(map[uint64]map[uint64]common.Hash), @@ -60,7 +60,7 @@ func newLimbo(datadir string) (*limbo, error) { fails = append(fails, id) } } - store, err := billy.Open(billy.Options{Path: datadir, Repair: true}, newSlotter(), index) + store, err := billy.Open(billy.Options{Path: datadir, Repair: true}, newSlotter(maxBlobsPerTransaction), index) if err != nil { return nil, err } diff --git a/core/txpool/blobpool/priority_test.go b/core/txpool/blobpool/priority_test.go index cf0e0454a0..1eaee6d7df 100644 --- a/core/txpool/blobpool/priority_test.go +++ b/core/txpool/blobpool/priority_test.go @@ -52,7 +52,7 @@ func TestPriorityCalculation(t *testing.T) { func BenchmarkDynamicFeeJumpCalculation(b *testing.B) { fees := make([]*uint256.Int, b.N) for i := 0; i < b.N; i++ { - fees[i] = uint256.NewInt(rand.Uint64()) + fees[i] = uint256.NewInt(rnd.Uint64()) } b.ResetTimer() b.ReportAllocs() @@ -76,8 +76,8 @@ func BenchmarkPriorityCalculation(b *testing.B) { txBasefeeJumps := make([]float64, b.N) txBlobfeeJumps := make([]float64, b.N) for i := 0; i < b.N; i++ { - txBasefeeJumps[i] = dynamicFeeJumps(uint256.NewInt(rand.Uint64())) - txBlobfeeJumps[i] = dynamicFeeJumps(uint256.NewInt(rand.Uint64())) + txBasefeeJumps[i] = dynamicFeeJumps(uint256.NewInt(rnd.Uint64())) + txBlobfeeJumps[i] = dynamicFeeJumps(uint256.NewInt(rnd.Uint64())) } b.ResetTimer() b.ReportAllocs() diff --git a/core/txpool/blobpool/slotter.go b/core/txpool/blobpool/slotter.go index 35349c3445..84ccc0f27b 100644 --- a/core/txpool/blobpool/slotter.go +++ b/core/txpool/blobpool/slotter.go @@ -25,13 +25,13 @@ package blobpool // The slotter also creates a shelf for 0-blob transactions. Whilst those are not // allowed in the current protocol, having an empty shelf is not a relevant use // of resources, but it makes stress testing with junk transactions simpler. -func newSlotter() func() (uint32, bool) { +func newSlotter(maxBlobsPerTransaction int) func() (uint32, bool) { slotsize := uint32(txAvgSize) slotsize -= uint32(blobSize) // underflows, it's ok, will overflow back in the first return return func() (size uint32, done bool) { slotsize += blobSize - finished := slotsize > maxBlobsPerTransaction*blobSize+txMaxSize + finished := slotsize > uint32(maxBlobsPerTransaction)*blobSize+txMaxSize return slotsize, finished } diff --git a/core/txpool/blobpool/slotter_test.go b/core/txpool/blobpool/slotter_test.go index a7b43b4d22..8d46f47d2c 100644 --- a/core/txpool/blobpool/slotter_test.go +++ b/core/txpool/blobpool/slotter_test.go @@ -21,7 +21,7 @@ import "testing" // Tests that the slotter creates the expected database shelves. func TestNewSlotter(t *testing.T) { // Generate the database shelve sizes - slotter := newSlotter() + slotter := newSlotter(6) var shelves []uint32 for { diff --git a/core/txpool/validation.go b/core/txpool/validation.go index dec711b173..8cd13776a6 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -23,6 +23,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -144,8 +145,9 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types if len(hashes) == 0 { return errors.New("blobless blob transaction") } - if len(hashes) > params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob { - return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob) + maxBlobs := eip4844.MaxBlobsPerBlock(opts.Config, head.Time) + if len(hashes) > maxBlobs { + return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), maxBlobs) } // Ensure commitments, proofs and hashes are valid if err := validateBlobSidecar(hashes, sidecar); err != nil { diff --git a/core/verkle_witness_test.go b/core/verkle_witness_test.go index 500fb37156..4e873eedff 100644 --- a/core/verkle_witness_test.go +++ b/core/verkle_witness_test.go @@ -58,6 +58,9 @@ var ( VerkleTime: u64(0), TerminalTotalDifficulty: common.Big0, EnableVerkleAtGenesis: true, + BlobScheduleConfig: ¶ms.BlobScheduleConfig{ + Verkle: params.DefaultPragueBlobConfig, + }, // TODO uncomment when proof generation is merged // ProofInBlocks: true, } @@ -79,6 +82,9 @@ var ( VerkleTime: u64(0), TerminalTotalDifficulty: common.Big0, EnableVerkleAtGenesis: true, + BlobScheduleConfig: ¶ms.BlobScheduleConfig{ + Verkle: params.DefaultPragueBlobConfig, + }, } ) @@ -226,11 +232,11 @@ func TestProcessParentBlockHash(t *testing.T) { var num = 2 for i := 1; i <= num; i++ { header := &types.Header{ParentHash: common.Hash{byte(i)}, Number: big.NewInt(int64(i)), Difficulty: new(big.Int)} - vmContext := NewEVMBlockContext(header, nil, new(common.Address)) chainConfig := params.MergedTestChainConfig if isVerkle { chainConfig = testVerkleChainConfig } + vmContext := NewEVMBlockContext(header, nil, new(common.Address)) evm := vm.NewEVM(vmContext, statedb, chainConfig, vm.Config{}) ProcessParentBlockHash(header.ParentHash, evm) } diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 6074e9a096..44f5bc8273 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -314,6 +314,10 @@ func (d *dummyChain) GetHeader(h common.Hash, n uint64) *types.Header { return fakeHeader(n, parentHash) } +func (d *dummyChain) Config() *params.ChainConfig { + return nil +} + // TestBlockhash tests the blockhash operation. It's a bit special, since it internally // requires access to a chain reader. func TestBlockhash(t *testing.T) { diff --git a/eth/api_backend.go b/eth/api_backend.go index 90439e8eab..52ecd91789 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -356,7 +356,7 @@ func (b *EthAPIBackend) FeeHistory(ctx context.Context, blockCount uint64, lastB func (b *EthAPIBackend) BlobBaseFee(ctx context.Context) *big.Int { if excess := b.CurrentHeader().ExcessBlobGas; excess != nil { - return eip4844.CalcBlobFee(*excess) + return eip4844.CalcBlobFee(b.ChainConfig(), b.CurrentHeader()) } return nil } diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 407f0db3eb..9840d9c6ad 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -1227,6 +1227,7 @@ func setupBodies(t *testing.T) (*node.Node, *eth.Ethereum, []*types.Block) { genesis.Config.ShanghaiTime = &time genesis.Config.CancunTime = &time genesis.Config.PragueTime = &time + genesis.Config.BlobScheduleConfig = params.DefaultBlobSchedule n, ethservice := startEthService(t, genesis, blocks) @@ -1543,6 +1544,7 @@ func TestParentBeaconBlockRoot(t *testing.T) { time := blocks[len(blocks)-1].Time() + 5 genesis.Config.ShanghaiTime = &time genesis.Config.CancunTime = &time + genesis.Config.BlobScheduleConfig = params.DefaultBlobSchedule n, ethservice := startEthService(t, genesis, blocks) defer n.Close() @@ -1625,6 +1627,7 @@ func TestWitnessCreationAndConsumption(t *testing.T) { timestamp := blocks[len(blocks)-2].Time() + 5 genesis.Config.ShanghaiTime = ×tamp genesis.Config.CancunTime = ×tamp + genesis.Config.BlobScheduleConfig = params.DefaultBlobSchedule n, ethservice := startEthService(t, genesis, blocks[:9]) defer n.Close() diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index f80b1d6096..52b3591086 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -1108,7 +1108,7 @@ func TestTransactionFetcherBandwidthLimiting(t *testing.T) { doTxNotify{peer: "C", hashes: []common.Hash{{0x07}, {0x08}}, types: []byte{types.BlobTxType, types.BlobTxType}, - sizes: []uint32{params.MaxBlobGasPerBlock, params.MaxBlobGasPerBlock}, + sizes: []uint32{params.BlobTxBlobGasPerBlob * 10, params.BlobTxBlobGasPerBlob * 10}, }, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), @@ -1125,8 +1125,8 @@ func TestTransactionFetcherBandwidthLimiting(t *testing.T) { {common.Hash{0x06}, types.LegacyTxType, maxTxRetrievalSize}, }, "C": { - {common.Hash{0x07}, types.BlobTxType, params.MaxBlobGasPerBlock}, - {common.Hash{0x08}, types.BlobTxType, params.MaxBlobGasPerBlock}, + {common.Hash{0x07}, types.BlobTxType, params.BlobTxBlobGasPerBlob * 10}, + {common.Hash{0x08}, types.BlobTxType, params.BlobTxBlobGasPerBlob * 10}, }, }, fetching: map[string][]common.Hash{ diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 1e625e21c0..697263f20b 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -31,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) @@ -97,8 +96,10 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { } // Fill in blob base fee and next blob base fee. if excessBlobGas := bf.header.ExcessBlobGas; excessBlobGas != nil { - bf.results.blobBaseFee = eip4844.CalcBlobFee(*excessBlobGas) - bf.results.nextBlobBaseFee = eip4844.CalcBlobFee(eip4844.CalcExcessBlobGas(*excessBlobGas, *bf.header.BlobGasUsed)) + bf.results.blobBaseFee = eip4844.CalcBlobFee(config, bf.header) + excess := eip4844.CalcExcessBlobGas(config, bf.header) + next := &types.Header{Number: bf.header.Number, Time: bf.header.Time, ExcessBlobGas: &excess} + bf.results.nextBlobBaseFee = eip4844.CalcBlobFee(config, next) } else { bf.results.blobBaseFee = new(big.Int) bf.results.nextBlobBaseFee = new(big.Int) @@ -106,7 +107,8 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { // Compute gas used ratio for normal and blob gas. bf.results.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit) if blobGasUsed := bf.header.BlobGasUsed; blobGasUsed != nil { - bf.results.blobGasUsedRatio = float64(*blobGasUsed) / params.MaxBlobGasPerBlock + maxBlobs := eip4844.MaxBlobsPerBlock(config, bf.header.Time) + bf.results.blobGasUsedRatio = float64(*blobGasUsed) / float64(maxBlobs) } if len(percentiles) == 0 { diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 185224d834..0a32c278cb 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -157,6 +157,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, cancunBlock *big.Int, pe ts := gspec.Timestamp + cancunBlock.Uint64()*10 // fixed 10 sec block time in blockgen config.ShanghaiTime = &ts config.CancunTime = &ts + config.BlobScheduleConfig = params.DefaultBlobSchedule signer = types.LatestSigner(gspec.Config) } diff --git a/eth/tracers/internal/tracetest/supply_test.go b/eth/tracers/internal/tracetest/supply_test.go index d918f5aca4..dc46b6d5ef 100644 --- a/eth/tracers/internal/tracetest/supply_test.go +++ b/eth/tracers/internal/tracetest/supply_test.go @@ -360,6 +360,7 @@ func TestSupplySelfdestruct(t *testing.T) { cancunTime := uint64(0) gspec.Config.ShanghaiTime = &cancunTime gspec.Config.CancunTime = &cancunTime + gspec.Config.BlobScheduleConfig = params.DefaultBlobSchedule postCancunOutput, postCancunChain, err := testSupplyTracer(t, gspec, testBlockGenerationFunc) if err != nil { diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer/blob_tx.json b/eth/tracers/internal/tracetest/testdata/call_tracer/blob_tx.json index 05da3b42e1..f8e08532a4 100644 --- a/eth/tracers/internal/tracetest/testdata/call_tracer/blob_tx.json +++ b/eth/tracers/internal/tracetest/testdata/call_tracer/blob_tx.json @@ -41,7 +41,14 @@ "grayGlacierBlock": 0, "shanghaiTime": 0, "cancunTime": 0, - "terminalTotalDifficulty": 0 + "terminalTotalDifficulty": 0, + "blobSchedule": { + "cancun": { + "target": 3, + "max": 6, + "baseFeeUpdateFraction": 3338477 + } + } } }, "context": { diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json index f8adbabf63..9f452ca5bd 100644 --- a/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer/blob_tx.json @@ -41,7 +41,14 @@ "grayGlacierBlock": 0, "shanghaiTime": 0, "cancunTime": 0, - "terminalTotalDifficulty": 0 + "terminalTotalDifficulty": 0, + "blobSchedule": { + "cancun": { + "target": 3, + "max": 6, + "baseFeeUpdateFraction": 3338477 + } + } } }, "context": { @@ -54,7 +61,9 @@ }, "input": "0x03f8b1820539806485174876e800825208940c2c51a0990aee1d73c1228de1586883415575088080c083020000f842a00100c9fbdf97f747e85847b4f3fff408f89c26842f77c882858bf2c89923849aa00138e3896f3c27f2389147507f8bcec52028b0efca6ee842ed83c9158873943880a0dbac3f97a532c9b00e6239b29036245a5bfbb96940b9d848634661abee98b945a03eec8525f261c2e79798f7b45a5d6ccaefa24576d53ba5023e919b86841c0675", "result": { - "0x0000000000000000000000000000000000000000": { "balance": "0x272e0528" }, + "0x0000000000000000000000000000000000000000": { + "balance": "0x272e0528" + }, "0x0c2c51a0990aee1d73c1228de158688341557508": { "balance": "0xde0b6b3a7640000" } diff --git a/eth/tracers/internal/tracetest/testdata/prestate_tracer/setcode_tx.json b/eth/tracers/internal/tracetest/testdata/prestate_tracer/setcode_tx.json index b7d5ee1c54..043130a072 100644 --- a/eth/tracers/internal/tracetest/testdata/prestate_tracer/setcode_tx.json +++ b/eth/tracers/internal/tracetest/testdata/prestate_tracer/setcode_tx.json @@ -53,7 +53,19 @@ "shanghaiTime": 0, "cancunTime": 0, "pragueTime": 0, - "terminalTotalDifficulty": 0 + "terminalTotalDifficulty": 0, + "blobSchedule": { + "cancun": { + "target": 3, + "max": 6, + "baseFeeUpdateFraction": 3338477 + }, + "prague": { + "target": 3, + "max": 6, + "baseFeeUpdateFraction": 5007716 + } + } } }, "context": { diff --git a/eth/tracers/internal/tracetest/util.go b/eth/tracers/internal/tracetest/util.go index abc2699498..e29144e04e 100644 --- a/eth/tracers/internal/tracetest/util.go +++ b/eth/tracers/internal/tracetest/util.go @@ -9,6 +9,7 @@ import ( "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" // Force-load native and js packages, to trigger registration @@ -53,8 +54,9 @@ func (c *callContext) toBlockContext(genesis *core.Genesis) vm.BlockContext { } if genesis.ExcessBlobGas != nil && genesis.BlobGasUsed != nil { - excessBlobGas := eip4844.CalcExcessBlobGas(*genesis.ExcessBlobGas, *genesis.BlobGasUsed) - context.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas) + excess := eip4844.CalcExcessBlobGas(genesis.Config, genesis.ToBlock().Header()) + header := &types.Header{ExcessBlobGas: &excess, Number: genesis.Config.LondonBlock, Time: *genesis.Config.CancunTime} + context.BlobBaseFee = eip4844.CalcBlobFee(genesis.Config, header) } return context } diff --git a/eth/tracers/live/supply.go b/eth/tracers/live/supply.go index fa4e5b1904..bae7445cb4 100644 --- a/eth/tracers/live/supply.go +++ b/eth/tracers/live/supply.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" "gopkg.in/natefinch/lumberjack.v2" ) @@ -83,6 +84,7 @@ type supplyTracer struct { delta supplyInfo txCallstack []supplyTxCallstack // Callstack for current transaction logger *lumberjack.Logger + chainConfig *params.ChainConfig } type supplyTracerConfig struct { @@ -112,14 +114,15 @@ func newSupplyTracer(cfg json.RawMessage) (*tracing.Hooks, error) { logger: logger, } return &tracing.Hooks{ - OnBlockStart: t.onBlockStart, - OnBlockEnd: t.onBlockEnd, - OnGenesisBlock: t.onGenesisBlock, - OnTxStart: t.onTxStart, - OnBalanceChange: t.onBalanceChange, - OnEnter: t.onEnter, - OnExit: t.onExit, - OnClose: t.onClose, + OnBlockchainInit: t.onBlockchainInit, + OnBlockStart: t.onBlockStart, + OnBlockEnd: t.onBlockEnd, + OnGenesisBlock: t.onGenesisBlock, + OnTxStart: t.onTxStart, + OnBalanceChange: t.onBalanceChange, + OnEnter: t.onEnter, + OnExit: t.onExit, + OnClose: t.onClose, }, nil } @@ -146,6 +149,10 @@ func (s *supplyTracer) resetDelta() { s.delta = newSupplyInfo() } +func (s *supplyTracer) onBlockchainInit(chainConfig *params.ChainConfig) { + s.chainConfig = chainConfig +} + func (s *supplyTracer) onBlockStart(ev tracing.BlockEvent) { s.resetDelta() @@ -161,8 +168,7 @@ func (s *supplyTracer) onBlockStart(ev tracing.BlockEvent) { // Blob burnt gas if blobGas := ev.Block.BlobGasUsed(); blobGas != nil && *blobGas > 0 && ev.Block.ExcessBlobGas() != nil { var ( - excess = *ev.Block.ExcessBlobGas() - baseFee = eip4844.CalcBlobFee(excess) + baseFee = eip4844.CalcBlobFee(s.chainConfig, ev.Block.Header()) burn = new(big.Int).Mul(new(big.Int).SetUint64(*blobGas), baseFee) ) s.delta.Burn.Blob = burn diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 4ad8a552d2..5fc7ed68f9 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" @@ -65,7 +66,7 @@ var ( ) var genesis = &core.Genesis{ - Config: params.AllEthashProtocolChanges, + Config: params.AllDevChainProtocolChanges, Alloc: types.GenesisAlloc{ testAddr: {Balance: testBalance}, revertContractAddr: {Code: revertCode}, @@ -136,7 +137,7 @@ func generateTestChain() []*types.Block { g.AddTx(testTx2) } } - _, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), 2, generate) + _, blocks, _ := core.GenerateChainWithGenesis(genesis, beacon.New(ethash.NewFaker()), 2, generate) return append([]*types.Block{genesis.ToBlock()}, blocks...) } @@ -223,7 +224,7 @@ func testHeader(t *testing.T, chain []*types.Block, client *rpc.Client) { if got != nil && got.Number != nil && got.Number.Sign() == 0 { got.Number = big.NewInt(0) // hack to make DeepEqual work } - if !reflect.DeepEqual(got, tt.want) { + if got.Hash() != tt.want.Hash() { t.Fatalf("HeaderByNumber(%v) got = %v, want %v", tt.block, got, tt.want) } }) @@ -314,7 +315,7 @@ func testChainID(t *testing.T, client *rpc.Client) { if err != nil { t.Fatalf("unexpected error: %v", err) } - if id == nil || id.Cmp(params.AllEthashProtocolChanges.ChainID) != 0 { + if id == nil || id.Cmp(params.AllDevChainProtocolChanges.ChainID) != 0 { t.Fatalf("ChainID returned wrong number: %+v", id) } } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index d9cec560ea..c0b37c516b 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -624,6 +624,7 @@ func (api *BlockChainAPI) GetBlockReceipts(ctx context.Context, blockNrOrHash rp type ChainContextBackend interface { Engine() consensus.Engine HeaderByNumber(context.Context, rpc.BlockNumber) (*types.Header, error) + ChainConfig() *params.ChainConfig } // ChainContext is an implementation of core.ChainContext. It's main use-case @@ -652,6 +653,10 @@ func (context *ChainContext) GetHeader(hash common.Hash, number uint64) *types.H return header } +func (context *ChainContext) Config() *params.ChainConfig { + return context.b.ChainConfig() +} + func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.StateDB, header *types.Header, overrides *override.StateOverride, blockOverrides *override.BlockOverrides, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { blockCtx := core.NewEVMBlockContext(header, NewChainContext(ctx, b), nil) if blockOverrides != nil { diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 130eaa9724..d364b80485 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -159,9 +159,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, if sim.chainConfig.IsCancun(header.Number, header.Time) { var excess uint64 if sim.chainConfig.IsCancun(parent.Number, parent.Time) { - excess = eip4844.CalcExcessBlobGas(*parent.ExcessBlobGas, *parent.BlobGasUsed) - } else { - excess = eip4844.CalcExcessBlobGas(0, 0) + excess = eip4844.CalcExcessBlobGas(sim.chainConfig, parent) } header.ExcessBlobGas = &excess } @@ -415,3 +413,7 @@ func (b *simBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) } return nil, errors.New("header not found") } + +func (b *simBackend) ChainConfig() *params.ChainConfig { + return b.b.ChainConfig() +} diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 175ac13a0f..7a7d63c535 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -37,10 +37,6 @@ import ( "github.com/holiman/uint256" ) -var ( - maxBlobsPerTransaction = params.MaxBlobGasPerBlock / params.BlobTxBlobGasPerBlob -) - // TransactionArgs represents the arguments to construct a new transaction // or a message call. type TransactionArgs struct { @@ -125,8 +121,9 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, skipGas if args.BlobHashes != nil && len(args.BlobHashes) == 0 { return errors.New(`need at least 1 blob for a blob transaction`) } - if args.BlobHashes != nil && len(args.BlobHashes) > maxBlobsPerTransaction { - return fmt.Errorf(`too many blobs in transaction (have=%d, max=%d)`, len(args.BlobHashes), maxBlobsPerTransaction) + maxBlobs := eip4844.MaxBlobsPerBlock(b.ChainConfig(), b.CurrentHeader().Time) + if args.BlobHashes != nil && len(args.BlobHashes) > maxBlobs { + return fmt.Errorf(`too many blobs in transaction (have=%d, max=%d)`, len(args.BlobHashes), maxBlobs) } // create check @@ -191,7 +188,9 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend, head if args.BlobFeeCap != nil && args.BlobFeeCap.ToInt().Sign() == 0 { return errors.New("maxFeePerBlobGas, if specified, must be non-zero") } - args.setCancunFeeDefaults(head) + if b.ChainConfig().IsCancun(head.Number, head.Time) { + args.setCancunFeeDefaults(b.ChainConfig(), head) + } // If both gasPrice and at least one of the EIP-1559 fee parameters are specified, error. if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") @@ -243,15 +242,10 @@ func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend, head } // setCancunFeeDefaults fills in reasonable default fee values for unspecified fields. -func (args *TransactionArgs) setCancunFeeDefaults(head *types.Header) { +func (args *TransactionArgs) setCancunFeeDefaults(config *params.ChainConfig, head *types.Header) { // Set maxFeePerBlobGas if it is missing. if args.BlobHashes != nil && args.BlobFeeCap == nil { - var excessBlobGas uint64 - if head.ExcessBlobGas != nil { - excessBlobGas = *head.ExcessBlobGas - } - // ExcessBlobGas must be set for a Cancun block. - blobBaseFee := eip4844.CalcBlobFee(excessBlobGas) + blobBaseFee := eip4844.CalcBlobFee(config, head) // Set the max fee to be 2 times larger than the previous block's blob base fee. // The additional slack allows the tx to not become invalidated if the base // fee is rising. diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index 7355c2463c..e017804861 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -279,6 +279,7 @@ func newBackendMock() *backendMock { BerlinBlock: big.NewInt(0), LondonBlock: big.NewInt(1000), CancunTime: &cancunTime, + BlobScheduleConfig: params.DefaultBlobSchedule, } return &backendMock{ current: &types.Header{ diff --git a/miner/worker.go b/miner/worker.go index b5aa080025..f8f4bae833 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -210,10 +210,7 @@ func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*envir if miner.chainConfig.IsCancun(header.Number, header.Time) { var excessBlobGas uint64 if miner.chainConfig.IsCancun(parent.Number, parent.Time) { - excessBlobGas = eip4844.CalcExcessBlobGas(*parent.ExcessBlobGas, *parent.BlobGasUsed) - } else { - // For the first post-fork block, both parent.data_gas_used and parent.excess_data_gas are evaluated as 0 - excessBlobGas = eip4844.CalcExcessBlobGas(0, 0) + excessBlobGas = eip4844.CalcExcessBlobGas(miner.chainConfig, parent) } header.BlobGasUsed = new(uint64) header.ExcessBlobGas = &excessBlobGas @@ -284,7 +281,8 @@ func (miner *Miner) commitBlobTransaction(env *environment, tx *types.Transactio // isn't really a better place right now. The blob gas limit is checked at block validation time // and not during execution. This means core.ApplyTransaction will not return an error if the // tx has too many blobs. So we have to explicitly check it here. - if (env.blobs+len(sc.Blobs))*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { + maxBlobs := eip4844.MaxBlobsPerBlock(miner.chainConfig, env.header.Time) + if env.blobs+len(sc.Blobs) > maxBlobs { return errors.New("max data blobs reached") } receipt, err := miner.applyTransaction(env, tx) @@ -333,7 +331,7 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran } // If we don't have enough blob space for any further blob transactions, // skip that list altogether - if !blobTxs.Empty() && env.blobs*params.BlobTxBlobGasPerBlob >= params.MaxBlobGasPerBlock { + if !blobTxs.Empty() && env.blobs >= eip4844.MaxBlobsPerBlock(miner.chainConfig, env.header.Time) { log.Trace("Not enough blob space for further blob transactions") blobTxs.Clear() // Fall though to pick up any plain txs @@ -367,11 +365,19 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran txs.Pop() continue } - if left := uint64(params.MaxBlobGasPerBlock - env.blobs*params.BlobTxBlobGasPerBlob); left < ltx.BlobGas { - log.Trace("Not enough blob gas left for transaction", "hash", ltx.Hash, "left", left, "needed", ltx.BlobGas) - txs.Pop() - continue + + // Most of the blob gas logic here is agnostic as to if the chain supports + // blobs or not, however the max check panics when called on a chain without + // a defined schedule, so we need to verify it's safe to call. + if miner.chainConfig.IsCancun(env.header.Number, env.header.Time) { + left := eip4844.MaxBlobsPerBlock(miner.chainConfig, env.header.Time) - env.blobs + if left < int(ltx.BlobGas/params.BlobTxBlobGasPerBlob) { + log.Trace("Not enough blob space left for transaction", "hash", ltx.Hash, "left", left, "needed", ltx.BlobGas/params.BlobTxBlobGasPerBlob) + txs.Pop() + continue + } } + // Transaction seems to fit, pull it up from the pool tx := ltx.Resolve() if tx == nil { @@ -379,6 +385,7 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran txs.Pop() continue } + // Error may be ignored here. The error has already been checked // during transaction acceptance in the transaction pool. from, _ := types.Sender(env.signer, tx) @@ -430,7 +437,7 @@ func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) filter.BaseFee = uint256.MustFromBig(env.header.BaseFee) } if env.header.ExcessBlobGas != nil { - filter.BlobFee = uint256.MustFromBig(eip4844.CalcBlobFee(*env.header.ExcessBlobGas)) + filter.BlobFee = uint256.MustFromBig(eip4844.CalcBlobFee(miner.chainConfig, env.header)) } filter.OnlyPlainTxs, filter.OnlyBlobTxs = true, false pendingPlainTxs := miner.txpool.Pending(filter) diff --git a/params/config.go b/params/config.go index 9269cb2310..0dbe4c509a 100644 --- a/params/config.go +++ b/params/config.go @@ -17,6 +17,7 @@ package params import ( + "errors" "fmt" "math" "math/big" @@ -60,6 +61,9 @@ var ( CancunTime: newUint64(1710338135), DepositContractAddress: common.HexToAddress("0x00000000219ab540356cbb839cbe05303d7705fa"), Ethash: new(EthashConfig), + BlobScheduleConfig: &BlobScheduleConfig{ + Cancun: DefaultCancunBlobConfig, + }, } // HoleskyChainConfig contains the chain parameters to run a node on the Holesky test network. HoleskyChainConfig = &ChainConfig{ @@ -84,6 +88,9 @@ var ( ShanghaiTime: newUint64(1696000704), CancunTime: newUint64(1707305664), Ethash: new(EthashConfig), + BlobScheduleConfig: &BlobScheduleConfig{ + Cancun: DefaultCancunBlobConfig, + }, } // SepoliaChainConfig contains the chain parameters to run a node on the Sepolia test network. SepoliaChainConfig = &ChainConfig{ @@ -108,6 +115,9 @@ var ( ShanghaiTime: newUint64(1677557088), CancunTime: newUint64(1706655072), Ethash: new(EthashConfig), + BlobScheduleConfig: &BlobScheduleConfig{ + Cancun: DefaultCancunBlobConfig, + }, } // AllEthashProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Ethash consensus. @@ -158,6 +168,10 @@ var ( CancunTime: newUint64(0), TerminalTotalDifficulty: big.NewInt(0), PragueTime: newUint64(0), + BlobScheduleConfig: &BlobScheduleConfig{ + Cancun: DefaultCancunBlobConfig, + Prague: DefaultPragueBlobConfig, + }, } // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced @@ -248,6 +262,10 @@ var ( TerminalTotalDifficulty: big.NewInt(0), Ethash: new(EthashConfig), Clique: nil, + BlobScheduleConfig: &BlobScheduleConfig{ + Cancun: DefaultCancunBlobConfig, + Prague: DefaultPragueBlobConfig, + }, } // NonActivatedConfig defines the chain configuration without activating @@ -282,6 +300,26 @@ var ( TestRules = TestChainConfig.Rules(new(big.Int), false, 0) ) +var ( + // DefaultCancunBlobConfig is the default blob configuration for the Cancun fork. + DefaultCancunBlobConfig = &BlobConfig{ + Target: 3, + Max: 6, + UpdateFraction: 3338477, + } + // DefaultPragueBlobConfig is the default blob configuration for the Prague fork. + DefaultPragueBlobConfig = &BlobConfig{ + Target: 6, + Max: 9, + UpdateFraction: 5007716, + } + // DefaultBlobSchedule is the latest configured blob schedule for test chains. + DefaultBlobSchedule = &BlobScheduleConfig{ + Cancun: DefaultCancunBlobConfig, + Prague: DefaultPragueBlobConfig, + } +) + // NetworkNames are user friendly names to use in the chain spec banner. var NetworkNames = map[string]string{ MainnetChainConfig.ChainID.String(): "mainnet", @@ -346,8 +384,9 @@ type ChainConfig struct { EnableVerkleAtGenesis bool `json:"enableVerkleAtGenesis,omitempty"` // Various consensus engines - Ethash *EthashConfig `json:"ethash,omitempty"` - Clique *CliqueConfig `json:"clique,omitempty"` + Ethash *EthashConfig `json:"ethash,omitempty"` + Clique *CliqueConfig `json:"clique,omitempty"` + BlobScheduleConfig *BlobScheduleConfig `json:"blobSchedule,omitempty"` } // EthashConfig is the consensus engine configs for proof-of-work based sealing. @@ -447,6 +486,20 @@ func (c *ChainConfig) Description() string { return banner } +// BlobConfig specifies the target and max blobs per block for the associated fork. +type BlobConfig struct { + Target int `json:"target"` + Max int `json:"max"` + UpdateFraction uint64 `json:"baseFeeUpdateFraction"` +} + +// BlobScheduleConfig determines target and max number of blobs allow per fork. +type BlobScheduleConfig struct { + Cancun *BlobConfig `json:"cancun,omitempty"` + Prague *BlobConfig `json:"prague,omitempty"` + Verkle *BlobConfig `json:"verkle,omitempty"` +} + // IsHomestead returns whether num is either equal to the homestead block or greater. func (c *ChainConfig) IsHomestead(num *big.Int) bool { return isBlockForked(c.HomesteadBlock, num) @@ -662,6 +715,45 @@ func (c *ChainConfig) CheckConfigForkOrder() error { lastFork = cur } } + + // Check that all forks with blobs explicitly define the blob schedule configuration. + bsc := c.BlobScheduleConfig + if bsc == nil { + bsc = new(BlobScheduleConfig) + } + for _, cur := range []struct { + name string + timestamp *uint64 + config *BlobConfig + }{ + {name: "cancun", timestamp: c.CancunTime, config: bsc.Cancun}, + {name: "prague", timestamp: c.PragueTime, config: bsc.Prague}, + } { + if cur.config != nil { + if err := cur.config.validate(); err != nil { + return fmt.Errorf("invalid blob configuration for fork %s: %v", cur.name, err) + } + } + if cur.timestamp != nil { + // If the fork is configured, a blob schedule must be defined for it. + if cur.config == nil { + return fmt.Errorf("unsupported fork configuration: missing blob configuration entry for %v in schedule", cur.name) + } + } + } + return nil +} + +func (bc *BlobConfig) validate() error { + if bc.Max < 0 { + return errors.New("max < 0") + } + if bc.Target < 0 { + return errors.New("target < 0") + } + if bc.UpdateFraction == 0 { + return errors.New("update fraction must be defined and non-zero") + } return nil } diff --git a/params/protocol_params.go b/params/protocol_params.go index c9179634a5..6b06dadaef 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -172,12 +172,8 @@ const ( BlobTxFieldElementsPerBlob = 4096 // Number of field elements stored in a single data blob BlobTxBlobGasPerBlob = 1 << 17 // Gas consumption of a single data blob (== blob byte size) BlobTxMinBlobGasprice = 1 // Minimum gas price for data blobs - BlobTxBlobGaspriceUpdateFraction = 3338477 // Controls the maximum rate of change for blob gas price BlobTxPointEvaluationPrecompileGas = 50000 // Gas price for the point evaluation precompile. - BlobTxTargetBlobGasPerBlock = 3 * BlobTxBlobGasPerBlob // Target consumable blob gas for data blobs per block (for 1559-like pricing) - MaxBlobGasPerBlock = 6 * BlobTxBlobGasPerBlob // Maximum consumable blob gas for data blobs per block - HistoryServeWindow = 8192 // Number of blocks to serve historical block hashes for, EIP-2935. ) diff --git a/tests/init.go b/tests/init.go index 8429f38e44..ee8d354224 100644 --- a/tests/init.go +++ b/tests/init.go @@ -334,6 +334,9 @@ var Forks = map[string]*params.ChainConfig{ TerminalTotalDifficulty: big.NewInt(0), ShanghaiTime: u64(0), CancunTime: u64(0), + BlobScheduleConfig: ¶ms.BlobScheduleConfig{ + Cancun: params.DefaultCancunBlobConfig, + }, }, "ShanghaiToCancunAtTime15k": { ChainID: big.NewInt(1), @@ -353,6 +356,9 @@ var Forks = map[string]*params.ChainConfig{ TerminalTotalDifficulty: big.NewInt(0), ShanghaiTime: u64(0), CancunTime: u64(15_000), + BlobScheduleConfig: ¶ms.BlobScheduleConfig{ + Cancun: params.DefaultCancunBlobConfig, + }, }, "Prague": { ChainID: big.NewInt(1), @@ -374,6 +380,10 @@ var Forks = map[string]*params.ChainConfig{ CancunTime: u64(0), PragueTime: u64(0), DepositContractAddress: params.MainnetChainConfig.DepositContractAddress, + BlobScheduleConfig: ¶ms.BlobScheduleConfig{ + Cancun: params.DefaultCancunBlobConfig, + Prague: params.DefaultPragueBlobConfig, + }, }, "CancunToPragueAtTime15k": { ChainID: big.NewInt(1), @@ -395,6 +405,10 @@ var Forks = map[string]*params.ChainConfig{ CancunTime: u64(0), PragueTime: u64(15_000), DepositContractAddress: params.MainnetChainConfig.DepositContractAddress, + BlobScheduleConfig: ¶ms.BlobScheduleConfig{ + Cancun: params.DefaultCancunBlobConfig, + Prague: params.DefaultPragueBlobConfig, + }, }, "Osaka": { ChainID: big.NewInt(1), diff --git a/tests/state_test.go b/tests/state_test.go index da4b8ef79d..0c9553c075 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -303,7 +303,7 @@ func runBenchmark(b *testing.B, t *StateTest) { // Prepare the EVM. txContext := core.NewEVMTxContext(msg) - context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase) + context := core.NewEVMBlockContext(block.Header(), &dummyChain{config: config}, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash context.BaseFee = baseFee evm := vm.NewEVM(context, state.StateDB, config, vmconfig) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index e658b62ebf..24caf41ed9 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -275,13 +276,14 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh return st, common.Hash{}, 0, err } - { // Blob transactions may be present after the Cancun fork. - // In production, - // - the header is verified against the max in eip4844.go:VerifyEIP4844Header - // - the block body is verified against the header in block_validator.go:ValidateBody - // Here, we just do this shortcut smaller fix, since state tests do not - // utilize those codepaths - if len(msg.BlobHashes)*params.BlobTxBlobGasPerBlob > params.MaxBlobGasPerBlock { + // Blob transactions may be present after the Cancun fork. + // In production, + // - the header is verified against the max in eip4844.go:VerifyEIP4844Header + // - the block body is verified against the header in block_validator.go:ValidateBody + // Here, we just do this shortcut smaller fix, since state tests do not + // utilize those codepaths. + if config.IsCancun(new(big.Int), block.Time()) { + if len(msg.BlobHashes) > eip4844.MaxBlobsPerBlock(config, block.Time()) { return st, common.Hash{}, 0, errors.New("blob gas exceeds maximum") } } @@ -299,7 +301,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh } // Prepare the EVM. - context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase) + context := core.NewEVMBlockContext(block.Header(), &dummyChain{config: config}, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash context.BaseFee = baseFee context.Random = nil @@ -312,8 +314,13 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh context.Difficulty = big.NewInt(0) } if config.IsCancun(new(big.Int), block.Time()) && t.json.Env.ExcessBlobGas != nil { - context.BlobBaseFee = eip4844.CalcBlobFee(*t.json.Env.ExcessBlobGas) + header := &types.Header{ + Time: block.Time(), + ExcessBlobGas: t.json.Env.ExcessBlobGas, + } + context.BlobBaseFee = eip4844.CalcBlobFee(config, header) } + evm := vm.NewEVM(context, st.StateDB, config, vmconfig) if tracer := vmconfig.Tracer; tracer != nil && tracer.OnTxStart != nil { @@ -543,3 +550,12 @@ func (st *StateTestState) Close() { st.Snapshots = nil } } + +// dummyChain implements the core.ChainContext interface. +type dummyChain struct { + config *params.ChainConfig +} + +func (d *dummyChain) Engine() consensus.Engine { return nil } +func (d *dummyChain) GetHeader(h common.Hash, n uint64) *types.Header { return nil } +func (d *dummyChain) Config() *params.ChainConfig { return d.config } From e332431cb28c6d374504fc5343fafaf3157a6aee Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 4 Feb 2025 17:16:26 +0100 Subject: [PATCH 20/24] core: assign default difficulty to zero for chain without ethash (#31067) I hit this case while trying something with the simulated backend. The EVM only enables instruction set forks after the merge when 'Random' is set. In the simulated backend, the random value will be set via the engine API for all blocks after genesis. But for the genesis block itself, the random value will not be assigned in the vm.BlockContext because the genesis has a non-zero difficulty. For my case, this meant that estimateGas did not work for the first transaction sent on the simulated chain, since the contract contained a PUSH0 instruction. This could also be fixed by explicitly configuring a zero difficulty in the simulated backend. However, I think that zero difficulty is a better default these days. --------- Co-authored-by: lightclient --- core/genesis.go | 8 ++++++-- core/genesis_test.go | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/core/genesis.go b/core/genesis.go index d1ee1b6e92..8f71f9ef1e 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -464,8 +464,12 @@ func (g *Genesis) toBlockWithRoot(root common.Hash) *types.Block { if g.GasLimit == 0 { head.GasLimit = params.GenesisGasLimit } - if g.Difficulty == nil && g.Mixhash == (common.Hash{}) { - head.Difficulty = params.GenesisDifficulty + if g.Difficulty == nil { + if g.Config != nil && g.Config.Ethash == nil { + head.Difficulty = big.NewInt(0) + } else if g.Mixhash == (common.Hash{}) { + head.Difficulty = params.GenesisDifficulty + } } if g.Config != nil && g.Config.IsLondon(common.Big0) { if g.BaseFee != nil { diff --git a/core/genesis_test.go b/core/genesis_test.go index 3e120a9307..ad75afcd9e 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -44,14 +44,14 @@ func testSetupGenesis(t *testing.T, scheme string) { var ( customghash = common.HexToHash("0x89c99d90b79719238d2645c7642f2c9295246e80775b38cfd162b696817fbd50") customg = Genesis{ - Config: ¶ms.ChainConfig{HomesteadBlock: big.NewInt(3)}, + Config: ¶ms.ChainConfig{HomesteadBlock: big.NewInt(3), Ethash: ¶ms.EthashConfig{}}, Alloc: types.GenesisAlloc{ {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, }, } oldcustomg = customg ) - oldcustomg.Config = ¶ms.ChainConfig{HomesteadBlock: big.NewInt(2)} + oldcustomg.Config = ¶ms.ChainConfig{HomesteadBlock: big.NewInt(2), Ethash: ¶ms.EthashConfig{}} tests := []struct { name string From 7c7b7f6ab1b3f56c527ce1fb39db3061a9ec0ac6 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Tue, 4 Feb 2025 17:23:01 +0100 Subject: [PATCH 21/24] core/txpool: remove locals-tracking from txpools (#30559) Replaces #29297, descendant from #27535 --------- This PR removes `locals` as a concept from transaction pools. Therefore, the pool now acts as very a good simulation/approximation of how our peers' pools behave. What this PR does instead, is implement a locals-tracker, which basically is a little thing which, from time to time, asks the pool "did you forget this transaction?". If it did, the tracker resubmits it. If the txpool _had_ forgotten it, chances are that the peers had also forgotten it. It will be propagated again. Doing this change means that we can simplify the pool internals, quite a lot. ### The semantics of `local` Historically, there has been two features, or usecases, that has been combined into the concept of `locals`. 1. "I want my local node to remember this transaction indefinitely, and resubmit to the network occasionally" 2. "I want this (valid) transaction included to be top-prio for my miner" This PR splits these features up, let's call it `1: local` and `2: prio`. The `prio` is not actually individual transaction, but rather a set of `address`es to prioritize. The attribute `local` means it will be tracked, and `prio` means it will be prioritized by miner. For `local`: anything transaction received via the RPC is marked as `local`, and tracked by the tracker. For `prio`: any transactions from this sender is included first, when building a block. The existing commandline-flag `--txpool.locals` sets the set of `prio` addresses. --------- Co-authored-by: Gary Rong --- core/txpool/blobpool/blobpool.go | 9 +- core/txpool/legacypool/legacypool.go | 385 +++--------- core/txpool/legacypool/legacypool_test.go | 575 +++--------------- core/txpool/legacypool/list.go | 63 +- core/txpool/{legacypool => locals}/journal.go | 2 +- core/txpool/locals/tx_tracker.go | 212 +++++++ core/txpool/subpool.go | 5 +- core/txpool/txpool.go | 21 +- eth/api_backend.go | 5 +- eth/backend.go | 20 +- eth/catalyst/api_test.go | 18 +- eth/catalyst/simulated_beacon_test.go | 52 +- eth/handler.go | 4 +- eth/handler_eth_test.go | 6 +- eth/handler_test.go | 2 +- miner/miner.go | 8 + miner/payload_building_test.go | 2 +- miner/worker.go | 31 +- 18 files changed, 493 insertions(+), 927 deletions(-) rename core/txpool/{legacypool => locals}/journal.go (99%) create mode 100644 core/txpool/locals/tx_tracker.go diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 78cf2c05af..51b8b67c61 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1269,7 +1269,7 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844. // Add inserts a set of blob transactions into the pool if they pass validation (both // consensus validity and pool restrictions). -func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error { +func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error { var ( adds = make([]*types.Transaction, 0, len(txs)) errs = make([]error, len(txs)) @@ -1701,13 +1701,6 @@ func (p *BlobPool) ContentFrom(addr common.Address) ([]*types.Transaction, []*ty return []*types.Transaction{}, []*types.Transaction{} } -// Locals retrieves the accounts currently considered local by the pool. -// -// There is no notion of local accounts in the blob pool. -func (p *BlobPool) Locals() []common.Address { - return []common.Address{} -} - // Status returns the known status (unknown/pending/queued) of a transaction // identified by their hashes. func (p *BlobPool) Status(hash common.Hash) txpool.TxStatus { diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 3d780ad373..1440af5440 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -99,7 +99,6 @@ var ( pendingGauge = metrics.NewRegisteredGauge("txpool/pending", nil) queuedGauge = metrics.NewRegisteredGauge("txpool/queued", nil) - localGauge = metrics.NewRegisteredGauge("txpool/local", nil) slotsGauge = metrics.NewRegisteredGauge("txpool/slots", nil) reheapTimer = metrics.NewRegisteredTimer("txpool/reheap", nil) @@ -159,10 +158,6 @@ var DefaultConfig = Config{ // unreasonable or unworkable. func (config *Config) sanitize() Config { conf := *config - if conf.Rejournal < time.Second { - log.Warn("Sanitizing invalid txpool journal time", "provided", conf.Rejournal, "updated", time.Second) - conf.Rejournal = time.Second - } if conf.PriceLimit < 1 { log.Warn("Sanitizing invalid txpool price limit", "provided", conf.PriceLimit, "updated", DefaultConfig.PriceLimit) conf.PriceLimit = DefaultConfig.PriceLimit @@ -214,9 +209,6 @@ type LegacyPool struct { currentState *state.StateDB // Current state in the blockchain head pendingNonces *noncer // Pending state tracking virtual nonces - locals *accountSet // Set of local transaction to exempt from eviction rules - journal *journal // Journal of local transaction to back up to disk - reserve txpool.AddressReserver // Address reserver to ensure exclusivity across subpools pending map[common.Address]*list // All currently processable transactions queue map[common.Address]*list // Queued but non-processable transactions @@ -262,16 +254,8 @@ func New(config Config, chain BlockChain) *LegacyPool { reorgShutdownCh: make(chan struct{}), initDoneCh: make(chan struct{}), } - pool.locals = newAccountSet(pool.signer) - for _, addr := range config.Locals { - log.Info("Setting new local account", "address", addr) - pool.locals.add(addr) - } pool.priced = newPricedList(pool.all) - if !config.NoLocals && config.Journal != "" { - pool.journal = newTxJournal(config.Journal) - } return pool } @@ -287,8 +271,7 @@ func (pool *LegacyPool) Filter(tx *types.Transaction) bool { } // Init sets the gas price needed to keep a transaction in the pool and the chain -// head to allow balance / nonce checks. The transaction journal will be loaded -// from disk and filtered based on the provided starting settings. The internal +// head to allow balance / nonce checks. The internal // goroutines will be spun up and the pool deemed operational afterwards. func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserve txpool.AddressReserver) error { // Set the address reserver to request exclusive access to pooled accounts @@ -311,20 +294,9 @@ func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserve txpool.A pool.currentState = statedb pool.pendingNonces = newNoncer(statedb) - // Start the reorg loop early, so it can handle requests generated during - // journal loading. pool.wg.Add(1) go pool.scheduleReorgLoop() - // If local transactions and journaling is enabled, load from disk - if pool.journal != nil { - if err := pool.journal.load(pool.addLocals); err != nil { - log.Warn("Failed to load transaction journal", "err", err) - } - if err := pool.journal.rotate(pool.local()); err != nil { - log.Warn("Failed to rotate transaction journal", "err", err) - } - } pool.wg.Add(1) go pool.loop() return nil @@ -340,13 +312,11 @@ func (pool *LegacyPool) loop() { prevPending, prevQueued, prevStales int // Start the stats reporting and transaction eviction tickers - report = time.NewTicker(statsReportInterval) - evict = time.NewTicker(evictionInterval) - journal = time.NewTicker(pool.config.Rejournal) + report = time.NewTicker(statsReportInterval) + evict = time.NewTicker(evictionInterval) ) defer report.Stop() defer evict.Stop() - defer journal.Stop() // Notify tests that the init phase is done close(pool.initDoneCh) @@ -372,11 +342,7 @@ func (pool *LegacyPool) loop() { case <-evict.C: pool.mu.Lock() for addr := range pool.queue { - // Skip local transactions from the eviction mechanism - if pool.locals.contains(addr) { - continue - } - // Any non-locals old enough should be removed + // Any old enough should be removed if time.Since(pool.beats[addr]) > pool.config.Lifetime { list := pool.queue[addr].Flatten() for _, tx := range list { @@ -386,16 +352,6 @@ func (pool *LegacyPool) loop() { } } pool.mu.Unlock() - - // Handle local transaction journal rotation - case <-journal.C: - if pool.journal != nil { - pool.mu.Lock() - if err := pool.journal.rotate(pool.local()); err != nil { - log.Warn("Failed to rotate local tx journal", "err", err) - } - pool.mu.Unlock() - } } } } @@ -406,9 +362,6 @@ func (pool *LegacyPool) Close() error { close(pool.reorgShutdownCh) pool.wg.Wait() - if pool.journal != nil { - pool.journal.close() - } log.Info("Transaction pool stopped") return nil } @@ -444,7 +397,7 @@ func (pool *LegacyPool) SetGasTip(tip *big.Int) { // If the min miner fee increased, remove transactions below the new threshold if newTip.Cmp(old) > 0 { // pool.priced is sorted by GasFeeCap, so we have to iterate through pool.all instead - drop := pool.all.RemotesBelowTip(tip) + drop := pool.all.TxsBelowTip(tip) for _, tx := range drop { pool.removeTx(tx.Hash(), false, true) } @@ -549,7 +502,7 @@ func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address] txs := list.Flatten() // If the miner requests tip enforcement, cap the lists now - if minTipBig != nil && !pool.locals.contains(addr) { + if minTipBig != nil { for i, tx := range txs { if tx.EffectiveGasTipIntCmp(minTipBig, baseFeeBig) < 0 { txs = txs[:i] @@ -577,35 +530,11 @@ func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address] return pending } -// Locals retrieves the accounts currently considered local by the pool. -func (pool *LegacyPool) Locals() []common.Address { - pool.mu.Lock() - defer pool.mu.Unlock() - - return pool.locals.flatten() -} - -// local retrieves all currently known local transactions, grouped by origin -// account and sorted by nonce. The returned transaction set is a copy and can be -// freely modified by calling code. -func (pool *LegacyPool) local() map[common.Address]types.Transactions { - txs := make(map[common.Address]types.Transactions) - for addr := range pool.locals.accounts { - if pending := pool.pending[addr]; pending != nil { - txs[addr] = append(txs[addr], pending.Flatten()...) - } - if queued := pool.queue[addr]; queued != nil { - txs[addr] = append(txs[addr], queued.Flatten()...) - } - } - return txs -} - // validateTxBasics checks whether a transaction is valid according to the consensus // rules, but does not check state-dependent validation such as sufficient balance. // This check is meant as an early check which only needs to be performed once, // and does not require the pool mutex to be held. -func (pool *LegacyPool) validateTxBasics(tx *types.Transaction, local bool) error { +func (pool *LegacyPool) validateTxBasics(tx *types.Transaction) error { opts := &txpool.ValidationOptions{ Config: pool.chainconfig, Accept: 0 | @@ -615,9 +544,6 @@ func (pool *LegacyPool) validateTxBasics(tx *types.Transaction, local bool) erro MaxSize: txMaxSize, MinTip: pool.gasTip.Load().ToBig(), } - if local { - opts.MinTip = new(big.Int) - } if err := txpool.ValidateTransaction(tx, pool.currentHead.Load(), pool.signer, opts); err != nil { return err } @@ -665,11 +591,7 @@ func (pool *LegacyPool) validateTx(tx *types.Transaction) error { // add validates a transaction and inserts it into the non-executable queue for later // pending promotion and execution. If the transaction is a replacement for an already // pending or queued one, it overwrites the previous transaction if its price is higher. -// -// If a newly added transaction is marked as local, its sending account will be -// added to the allowlist, preventing any associated transaction from being dropped -// out of the pool due to pricing constraints. -func (pool *LegacyPool) add(tx *types.Transaction, local bool) (replaced bool, err error) { +func (pool *LegacyPool) add(tx *types.Transaction) (replaced bool, err error) { // If the transaction is already known, discard it hash := tx.Hash() if pool.all.Get(hash) != nil { @@ -677,9 +599,6 @@ func (pool *LegacyPool) add(tx *types.Transaction, local bool) (replaced bool, e knownTxMeter.Mark(1) return false, txpool.ErrAlreadyKnown } - // Make the local flag. If it's from local source or it's from the network but - // the sender is marked as local previously, treat it as the local transaction. - isLocal := local || pool.locals.containsTx(tx) // If the transaction fails basic validation, discard it if err := pool.validateTx(tx); err != nil { @@ -715,7 +634,7 @@ func (pool *LegacyPool) add(tx *types.Transaction, local bool) (replaced bool, e // If the transaction pool is full, discard underpriced transactions if uint64(pool.all.Slots()+numSlots(tx)) > pool.config.GlobalSlots+pool.config.GlobalQueue { // If the new transaction is underpriced, don't accept it - if !isLocal && pool.priced.Underpriced(tx) { + if pool.priced.Underpriced(tx) { log.Trace("Discarding underpriced transaction", "hash", hash, "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap()) underpricedTxMeter.Mark(1) return false, txpool.ErrUnderpriced @@ -731,19 +650,18 @@ func (pool *LegacyPool) add(tx *types.Transaction, local bool) (replaced bool, e } // New transaction is better than our worse ones, make room for it. - // If it's a local transaction, forcibly discard all available transactions. - // Otherwise if we can't make enough room for new one, abort the operation. - drop, success := pool.priced.Discard(pool.all.Slots()-int(pool.config.GlobalSlots+pool.config.GlobalQueue)+numSlots(tx), isLocal) + // If we can't make enough room for new one, abort the operation. + drop, success := pool.priced.Discard(pool.all.Slots() - int(pool.config.GlobalSlots+pool.config.GlobalQueue) + numSlots(tx)) // Special case, we still can't make the room for the new remote one. - if !isLocal && !success { + if !success { log.Trace("Discarding overflown transaction", "hash", hash) overflowedTxMeter.Mark(1) return false, ErrTxPoolOverflow } // If the new transaction is a future transaction it should never churn pending transactions - if !isLocal && pool.isGapped(from, tx) { + if pool.isGapped(from, tx) { var replacesPending bool for _, dropTx := range drop { dropSender, _ := types.Sender(pool.signer, dropTx) @@ -755,7 +673,7 @@ func (pool *LegacyPool) add(tx *types.Transaction, local bool) (replaced bool, e // Add all transactions back to the priced queue if replacesPending { for _, dropTx := range drop { - pool.priced.Put(dropTx, false) + pool.priced.Put(dropTx) } log.Trace("Discarding future transaction replacing pending tx", "hash", hash) return false, txpool.ErrFutureReplacePending @@ -788,9 +706,8 @@ func (pool *LegacyPool) add(tx *types.Transaction, local bool) (replaced bool, e pool.priced.Removed(1) pendingReplaceMeter.Mark(1) } - pool.all.Add(tx, isLocal) - pool.priced.Put(tx, isLocal) - pool.journalTx(from, tx) + pool.all.Add(tx) + pool.priced.Put(tx) pool.queueTxEvent(tx) log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) @@ -799,20 +716,10 @@ func (pool *LegacyPool) add(tx *types.Transaction, local bool) (replaced bool, e return old != nil, nil } // New transaction isn't replacing a pending one, push into queue - replaced, err = pool.enqueueTx(hash, tx, isLocal, true) + replaced, err = pool.enqueueTx(hash, tx, true) if err != nil { return false, err } - // Mark local addresses and journal local transactions - if local && !pool.locals.contains(from) { - log.Info("Setting new local account", "address", from) - pool.locals.add(from) - pool.priced.Removed(pool.all.RemoteToLocals(pool.locals)) // Migrate the remotes if it's marked as local first time. - } - if isLocal { - localGauge.Inc(1) - } - pool.journalTx(from, tx) log.Trace("Pooled new future transaction", "hash", hash, "from", from, "to", tx.To()) return replaced, nil @@ -845,7 +752,7 @@ func (pool *LegacyPool) isGapped(from common.Address, tx *types.Transaction) boo // enqueueTx inserts a new transaction into the non-executable transaction queue. // // Note, this method assumes the pool lock is held! -func (pool *LegacyPool) enqueueTx(hash common.Hash, tx *types.Transaction, local bool, addAll bool) (bool, error) { +func (pool *LegacyPool) enqueueTx(hash common.Hash, tx *types.Transaction, addAll bool) (bool, error) { // Try to insert the transaction into the future queue from, _ := types.Sender(pool.signer, tx) // already validated if pool.queue[from] == nil { @@ -872,8 +779,8 @@ func (pool *LegacyPool) enqueueTx(hash common.Hash, tx *types.Transaction, local log.Error("Missing transaction in lookup set, please report the issue", "hash", hash) } if addAll { - pool.all.Add(tx, local) - pool.priced.Put(tx, local) + pool.all.Add(tx) + pool.priced.Put(tx) } // If we never record the heartbeat, do it right now. if _, exist := pool.beats[from]; !exist { @@ -882,18 +789,6 @@ func (pool *LegacyPool) enqueueTx(hash common.Hash, tx *types.Transaction, local return old != nil, nil } -// journalTx adds the specified transaction to the local disk journal if it is -// deemed to have been sent from a local account. -func (pool *LegacyPool) journalTx(from common.Address, tx *types.Transaction) { - // Only journal if it's enabled and the transaction is local - if pool.journal == nil || !pool.locals.contains(from) { - return - } - if err := pool.journal.insert(tx); err != nil { - log.Warn("Failed to journal local transaction", "err", err) - } -} - // promoteTx adds a transaction to the pending (processable) list of transactions // and returns whether it was inserted or an older was better. // @@ -930,28 +825,13 @@ func (pool *LegacyPool) promoteTx(addr common.Address, hash common.Hash, tx *typ return true } -// addLocals enqueues a batch of transactions into the pool if they are valid, marking the -// senders as local ones, ensuring they go around the local pricing constraints. -// -// This method is used to add transactions from the RPC API and performs synchronous pool -// reorganization and event propagation. -func (pool *LegacyPool) addLocals(txs []*types.Transaction) []error { - return pool.Add(txs, !pool.config.NoLocals, true) -} - -// addLocal enqueues a single local transaction into the pool if it is valid. This is -// a convenience wrapper around addLocals. -func (pool *LegacyPool) addLocal(tx *types.Transaction) error { - return pool.addLocals([]*types.Transaction{tx})[0] -} - -// addRemotes enqueues a batch of transactions into the pool if they are valid. If the -// senders are not among the locally tracked ones, full pricing constraints will apply. +// addRemotes enqueues a batch of transactions into the pool if they are valid. +// Full pricing constraints will apply. // // This method is used to add transactions from the p2p network and does not wait for pool // reorganization and internal event propagation. func (pool *LegacyPool) addRemotes(txs []*types.Transaction) []error { - return pool.Add(txs, false, false) + return pool.Add(txs, false) } // addRemote enqueues a single transaction into the pool if it is valid. This is a convenience @@ -962,23 +842,19 @@ func (pool *LegacyPool) addRemote(tx *types.Transaction) error { // addRemotesSync is like addRemotes, but waits for pool reorganization. Tests use this method. func (pool *LegacyPool) addRemotesSync(txs []*types.Transaction) []error { - return pool.Add(txs, false, true) + return pool.Add(txs, true) } // This is like addRemotes with a single transaction, but waits for pool reorganization. Tests use this method. func (pool *LegacyPool) addRemoteSync(tx *types.Transaction) error { - return pool.Add([]*types.Transaction{tx}, false, true)[0] + return pool.Add([]*types.Transaction{tx}, true)[0] } -// Add enqueues a batch of transactions into the pool if they are valid. Depending -// on the local flag, full pricing constraints will or will not be applied. +// Add enqueues a batch of transactions into the pool if they are valid. // // If sync is set, the method will block until all internal maintenance related // to the add is finished. Only use this during tests for determinism! -func (pool *LegacyPool) Add(txs []*types.Transaction, local, sync bool) []error { - // Do not treat as local if local transactions have been disabled - local = local && !pool.config.NoLocals - +func (pool *LegacyPool) Add(txs []*types.Transaction, sync bool) []error { // Filter out known ones without obtaining the pool lock or recovering signatures var ( errs = make([]error, len(txs)) @@ -994,7 +870,7 @@ func (pool *LegacyPool) Add(txs []*types.Transaction, local, sync bool) []error // Exclude transactions with basic errors, e.g invalid signatures and // insufficient intrinsic gas as soon as possible and cache senders // in transactions before obtaining lock - if err := pool.validateTxBasics(tx, local); err != nil { + if err := pool.validateTxBasics(tx); err != nil { errs[i] = err log.Trace("Discarding invalid transaction", "hash", tx.Hash(), "err", err) invalidTxMeter.Mark(1) @@ -1009,7 +885,7 @@ func (pool *LegacyPool) Add(txs []*types.Transaction, local, sync bool) []error // Process all the new transaction and merge any errors into the original slice pool.mu.Lock() - newErrs, dirtyAddrs := pool.addTxsLocked(news, local) + newErrs, dirtyAddrs := pool.addTxsLocked(news) pool.mu.Unlock() var nilSlot = 0 @@ -1030,11 +906,11 @@ func (pool *LegacyPool) Add(txs []*types.Transaction, local, sync bool) []error // addTxsLocked attempts to queue a batch of transactions if they are valid. // The transaction pool lock must be held. -func (pool *LegacyPool) addTxsLocked(txs []*types.Transaction, local bool) ([]error, *accountSet) { +func (pool *LegacyPool) addTxsLocked(txs []*types.Transaction) ([]error, *accountSet) { dirty := newAccountSet(pool.signer) errs := make([]error, len(txs)) for i, tx := range txs { - replaced, err := pool.add(tx, local) + replaced, err := pool.add(tx) errs[i] = err if err == nil && !replaced { dirty.addTx(tx) @@ -1126,9 +1002,6 @@ func (pool *LegacyPool) removeTx(hash common.Hash, outofbound bool, unreserve bo if outofbound { pool.priced.Removed(1) } - if pool.locals.contains(addr) { - localGauge.Dec(1) - } // Remove the transaction from the pending lists and reset the account nonce if pending := pool.pending[addr]; pending != nil { if removed, invalids := pending.Remove(tx); removed { @@ -1139,7 +1012,7 @@ func (pool *LegacyPool) removeTx(hash common.Hash, outofbound bool, unreserve bo // Postpone any invalidated transactions for _, tx := range invalids { // Internal shuffle shouldn't touch the lookup set. - pool.enqueueTx(tx.Hash(), tx, false, false) + pool.enqueueTx(tx.Hash(), tx, false) } // Update the account nonce if needed pool.pendingNonces.setIfLower(addr, tx.Nonce()) @@ -1204,7 +1077,7 @@ func (pool *LegacyPool) scheduleReorgLoop() { launchNextRun bool reset *txpoolResetRequest dirtyAccounts *accountSet - queuedEvents = make(map[common.Address]*sortedMap) + queuedEvents = make(map[common.Address]*SortedMap) ) for { // Launch next background reorg if needed @@ -1217,7 +1090,7 @@ func (pool *LegacyPool) scheduleReorgLoop() { launchNextRun = false reset, dirtyAccounts = nil, nil - queuedEvents = make(map[common.Address]*sortedMap) + queuedEvents = make(map[common.Address]*SortedMap) } select { @@ -1246,7 +1119,7 @@ func (pool *LegacyPool) scheduleReorgLoop() { // request one later if they want the events sent. addr, _ := types.Sender(pool.signer, tx) if _, ok := queuedEvents[addr]; !ok { - queuedEvents[addr] = newSortedMap() + queuedEvents[addr] = NewSortedMap() } queuedEvents[addr].Put(tx) @@ -1265,7 +1138,7 @@ func (pool *LegacyPool) scheduleReorgLoop() { } // runReorg runs reset and promoteExecutables on behalf of scheduleReorgLoop. -func (pool *LegacyPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirtyAccounts *accountSet, events map[common.Address]*sortedMap) { +func (pool *LegacyPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirtyAccounts *accountSet, events map[common.Address]*SortedMap) { defer func(t0 time.Time) { reorgDurationTimer.Update(time.Since(t0)) }(time.Now()) @@ -1332,7 +1205,7 @@ func (pool *LegacyPool) runReorg(done chan struct{}, reset *txpoolResetRequest, for _, tx := range promoted { addr, _ := types.Sender(pool.signer, tx) if _, ok := events[addr]; !ok { - events[addr] = newSortedMap() + events[addr] = NewSortedMap() } events[addr].Put(tx) } @@ -1441,7 +1314,7 @@ func (pool *LegacyPool) reset(oldHead, newHead *types.Header) { // Inject any transactions discarded due to reorgs log.Debug("Reinjecting stale transactions", "count", len(reinject)) core.SenderCacher().Recover(pool.signer, reinject) - pool.addTxsLocked(reinject, false) + pool.addTxsLocked(reinject) } // promoteExecutables moves transactions that have become processable from the @@ -1486,22 +1359,17 @@ func (pool *LegacyPool) promoteExecutables(accounts []common.Address) []*types.T queuedGauge.Dec(int64(len(readies))) // Drop all transactions over the allowed limit - var caps types.Transactions - if !pool.locals.contains(addr) { - caps = list.Cap(int(pool.config.AccountQueue)) - for _, tx := range caps { - hash := tx.Hash() - pool.all.Remove(hash) - log.Trace("Removed cap-exceeding queued transaction", "hash", hash) - } - queuedRateLimitMeter.Mark(int64(len(caps))) + var caps = list.Cap(int(pool.config.AccountQueue)) + for _, tx := range caps { + hash := tx.Hash() + pool.all.Remove(hash) + log.Trace("Removed cap-exceeding queued transaction", "hash", hash) } + queuedRateLimitMeter.Mark(int64(len(caps))) // Mark all the items dropped as removed pool.priced.Removed(len(forwards) + len(drops) + len(caps)) queuedGauge.Dec(int64(len(forwards) + len(drops) + len(caps))) - if pool.locals.contains(addr) { - localGauge.Dec(int64(len(forwards) + len(drops) + len(caps))) - } + // Delete the entire queue entry if it became empty. if list.Empty() { delete(pool.queue, addr) @@ -1531,14 +1399,14 @@ func (pool *LegacyPool) truncatePending() { spammers := prque.New[int64, common.Address](nil) for addr, list := range pool.pending { // Only evict transactions from high rollers - if !pool.locals.contains(addr) && uint64(list.Len()) > pool.config.AccountSlots { + if uint64(list.Len()) > pool.config.AccountSlots { spammers.Push(addr, int64(list.Len())) } } // Gradually drop transactions from offenders offenders := []common.Address{} for pending > pool.config.GlobalSlots && !spammers.Empty() { - // Retrieve the next offender if not local address + // Retrieve the next offender offender, _ := spammers.Pop() offenders = append(offenders, offender) @@ -1564,9 +1432,7 @@ func (pool *LegacyPool) truncatePending() { } pool.priced.Removed(len(caps)) pendingGauge.Dec(int64(len(caps))) - if pool.locals.contains(offenders[i]) { - localGauge.Dec(int64(len(caps))) - } + pending-- } } @@ -1591,9 +1457,6 @@ func (pool *LegacyPool) truncatePending() { } pool.priced.Removed(len(caps)) pendingGauge.Dec(int64(len(caps))) - if pool.locals.contains(addr) { - localGauge.Dec(int64(len(caps))) - } pending-- } } @@ -1614,13 +1477,11 @@ func (pool *LegacyPool) truncateQueue() { // Sort all accounts with queued transactions by heartbeat addresses := make(addressesByHeartbeat, 0, len(pool.queue)) for addr := range pool.queue { - if !pool.locals.contains(addr) { // don't drop locals - addresses = append(addresses, addressByHeartbeat{addr, pool.beats[addr]}) - } + addresses = append(addresses, addressByHeartbeat{addr, pool.beats[addr]}) } sort.Sort(sort.Reverse(addresses)) - // Drop transactions until the total is below the limit or only locals remain + // Drop transactions until the total is below the limit for drop := queued - pool.config.GlobalQueue; drop > 0 && len(addresses) > 0; { addr := addresses[len(addresses)-1] list := pool.queue[addr.address] @@ -1680,12 +1541,10 @@ func (pool *LegacyPool) demoteUnexecutables() { log.Trace("Demoting pending transaction", "hash", hash) // Internal shuffle shouldn't touch the lookup set. - pool.enqueueTx(hash, tx, false, false) + pool.enqueueTx(hash, tx, false) } pendingGauge.Dec(int64(len(olds) + len(drops) + len(invalids))) - if pool.locals.contains(addr) { - localGauge.Dec(int64(len(olds) + len(drops) + len(invalids))) - } + // If there's a gap in front, alert (should never happen) and postpone all transactions if list.Len() > 0 && list.txs.Get(nonce) == nil { gapped := list.Cap(0) @@ -1694,7 +1553,7 @@ func (pool *LegacyPool) demoteUnexecutables() { log.Error("Demoting invalidated transaction", "hash", hash) // Internal shuffle shouldn't touch the lookup set. - pool.enqueueTx(hash, tx, false, false) + pool.enqueueTx(hash, tx, false) } pendingGauge.Dec(int64(len(gapped))) } @@ -1741,21 +1600,6 @@ func newAccountSet(signer types.Signer, addrs ...common.Address) *accountSet { return as } -// contains checks if a given address is contained within the set. -func (as *accountSet) contains(addr common.Address) bool { - _, exist := as.accounts[addr] - return exist -} - -// containsTx checks if the sender of a given tx is within the set. If the sender -// cannot be derived, this method returns false. -func (as *accountSet) containsTx(tx *types.Transaction) bool { - if addr, err := types.Sender(as.signer, tx); err == nil { - return as.contains(addr) - } - return false -} - // add inserts a new address into the set to track. func (as *accountSet) add(addr common.Address) { as.accounts[addr] = struct{}{} @@ -1793,43 +1637,29 @@ func (as *accountSet) merge(other *accountSet) { // internal mechanisms. The sole purpose of the type is to permit out-of-bound // peeking into the pool in LegacyPool.Get without having to acquire the widely scoped // LegacyPool.mu mutex. -// -// This lookup set combines the notion of "local transactions", which is useful -// to build upper-level structure. type lookup struct { - slots int - lock sync.RWMutex - locals map[common.Hash]*types.Transaction - remotes map[common.Hash]*types.Transaction + slots int + lock sync.RWMutex + txs map[common.Hash]*types.Transaction } // newLookup returns a new lookup structure. func newLookup() *lookup { return &lookup{ - locals: make(map[common.Hash]*types.Transaction), - remotes: make(map[common.Hash]*types.Transaction), + txs: make(map[common.Hash]*types.Transaction), } } // Range calls f on each key and value present in the map. The callback passed // should return the indicator whether the iteration needs to be continued. // Callers need to specify which set (or both) to be iterated. -func (t *lookup) Range(f func(hash common.Hash, tx *types.Transaction, local bool) bool, local bool, remote bool) { +func (t *lookup) Range(f func(hash common.Hash, tx *types.Transaction) bool) { t.lock.RLock() defer t.lock.RUnlock() - if local { - for key, value := range t.locals { - if !f(key, value, true) { - return - } - } - } - if remote { - for key, value := range t.remotes { - if !f(key, value, false) { - return - } + for key, value := range t.txs { + if !f(key, value) { + return } } } @@ -1839,26 +1669,7 @@ func (t *lookup) Get(hash common.Hash) *types.Transaction { t.lock.RLock() defer t.lock.RUnlock() - if tx := t.locals[hash]; tx != nil { - return tx - } - return t.remotes[hash] -} - -// GetLocal returns a transaction if it exists in the lookup, or nil if not found. -func (t *lookup) GetLocal(hash common.Hash) *types.Transaction { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.locals[hash] -} - -// GetRemote returns a transaction if it exists in the lookup, or nil if not found. -func (t *lookup) GetRemote(hash common.Hash) *types.Transaction { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.remotes[hash] + return t.txs[hash] } // Count returns the current number of transactions in the lookup. @@ -1866,23 +1677,7 @@ func (t *lookup) Count() int { t.lock.RLock() defer t.lock.RUnlock() - return len(t.locals) + len(t.remotes) -} - -// LocalCount returns the current number of local transactions in the lookup. -func (t *lookup) LocalCount() int { - t.lock.RLock() - defer t.lock.RUnlock() - - return len(t.locals) -} - -// RemoteCount returns the current number of remote transactions in the lookup. -func (t *lookup) RemoteCount() int { - t.lock.RLock() - defer t.lock.RUnlock() - - return len(t.remotes) + return len(t.txs) } // Slots returns the current number of slots used in the lookup. @@ -1894,18 +1689,14 @@ func (t *lookup) Slots() int { } // Add adds a transaction to the lookup. -func (t *lookup) Add(tx *types.Transaction, local bool) { +func (t *lookup) Add(tx *types.Transaction) { t.lock.Lock() defer t.lock.Unlock() t.slots += numSlots(tx) slotsGauge.Update(int64(t.slots)) - if local { - t.locals[tx.Hash()] = tx - } else { - t.remotes[tx.Hash()] = tx - } + t.txs[tx.Hash()] = tx } // Remove removes a transaction from the lookup. @@ -1913,10 +1704,7 @@ func (t *lookup) Remove(hash common.Hash) { t.lock.Lock() defer t.lock.Unlock() - tx, ok := t.locals[hash] - if !ok { - tx, ok = t.remotes[hash] - } + tx, ok := t.txs[hash] if !ok { log.Error("No transaction found to be deleted", "hash", hash) return @@ -1924,36 +1712,18 @@ func (t *lookup) Remove(hash common.Hash) { t.slots -= numSlots(tx) slotsGauge.Update(int64(t.slots)) - delete(t.locals, hash) - delete(t.remotes, hash) + delete(t.txs, hash) } -// RemoteToLocals migrates the transactions belongs to the given locals to locals -// set. The assumption is held the locals set is thread-safe to be used. -func (t *lookup) RemoteToLocals(locals *accountSet) int { - t.lock.Lock() - defer t.lock.Unlock() - - var migrated int - for hash, tx := range t.remotes { - if locals.containsTx(tx) { - t.locals[hash] = tx - delete(t.remotes, hash) - migrated += 1 - } - } - return migrated -} - -// RemotesBelowTip finds all remote transactions below the given tip threshold. -func (t *lookup) RemotesBelowTip(threshold *big.Int) types.Transactions { +// TxsBelowTip finds all remote transactions below the given tip threshold. +func (t *lookup) TxsBelowTip(threshold *big.Int) types.Transactions { found := make(types.Transactions, 0, 128) - t.Range(func(hash common.Hash, tx *types.Transaction, local bool) bool { + t.Range(func(hash common.Hash, tx *types.Transaction) bool { if tx.GasTipCapIntCmp(threshold) < 0 { found = append(found, tx) } return true - }, false, true) // Only iterate remotes + }) return found } @@ -1982,24 +1752,13 @@ func (pool *LegacyPool) Clear() { // The transaction addition may attempt to reserve the sender addr which // can't happen until Clear releases the reservation lock. Clear cannot // acquire the subpool lock until the transaction addition is completed. - for _, tx := range pool.all.remotes { + for _, tx := range pool.all.txs { senderAddr, _ := types.Sender(pool.signer, tx) pool.reserve(senderAddr, false) } - for localSender := range pool.locals.accounts { - pool.reserve(localSender, false) - } - pool.all = newLookup() pool.priced = newPricedList(pool.all) pool.pending = make(map[common.Address]*list) pool.queue = make(map[common.Address]*list) pool.pendingNonces = newNoncer(pool.currentState) - - if !pool.config.NoLocals && pool.config.Journal != "" { - pool.journal = newTxJournal(pool.config.Journal) - if err := pool.journal.rotate(pool.local()); err != nil { - log.Warn("Failed to rotate transaction journal", "err", err) - } - } } diff --git a/core/txpool/legacypool/legacypool_test.go b/core/txpool/legacypool/legacypool_test.go index abbde8cae3..55699e93ee 100644 --- a/core/txpool/legacypool/legacypool_test.go +++ b/core/txpool/legacypool/legacypool_test.go @@ -23,7 +23,6 @@ import ( "fmt" "math/big" "math/rand" - "os" "sync" "sync/atomic" "testing" @@ -183,7 +182,7 @@ func validatePoolInternals(pool *LegacyPool) error { return fmt.Errorf("total transaction count %d != %d pending + %d queued", total, pending, queued) } pool.priced.Reheap() - priced, remote := pool.priced.urgent.Len()+pool.priced.floating.Len(), pool.all.RemoteCount() + priced, remote := pool.priced.urgent.Len()+pool.priced.floating.Len(), pool.all.Count() if priced != remote { return fmt.Errorf("total priced transaction count %d != %d", priced, remote) } @@ -350,9 +349,6 @@ func TestInvalidTransactions(t *testing.T) { if err, want := pool.addRemote(tx), txpool.ErrUnderpriced; !errors.Is(err, want) { t.Errorf("want %v have %v", want, err) } - if err := pool.addLocal(tx); err != nil { - t.Error("expected", nil, "got", err) - } } func TestQueue(t *testing.T) { @@ -366,7 +362,7 @@ func TestQueue(t *testing.T) { testAddBalance(pool, from, big.NewInt(1000)) <-pool.requestReset(nil, nil) - pool.enqueueTx(tx.Hash(), tx, false, true) + pool.enqueueTx(tx.Hash(), tx, true) <-pool.requestPromoteExecutables(newAccountSet(pool.signer, from)) if len(pool.pending) != 1 { t.Error("expected valid txs to be 1 is", len(pool.pending)) @@ -375,7 +371,7 @@ func TestQueue(t *testing.T) { tx = transaction(1, 100, key) from, _ = deriveSender(tx) testSetNonce(pool, from, 2) - pool.enqueueTx(tx.Hash(), tx, false, true) + pool.enqueueTx(tx.Hash(), tx, true) <-pool.requestPromoteExecutables(newAccountSet(pool.signer, from)) if _, ok := pool.pending[from].txs.items[tx.Nonce()]; ok { @@ -399,9 +395,9 @@ func TestQueue2(t *testing.T) { testAddBalance(pool, from, big.NewInt(1000)) pool.reset(nil, nil) - pool.enqueueTx(tx1.Hash(), tx1, false, true) - pool.enqueueTx(tx2.Hash(), tx2, false, true) - pool.enqueueTx(tx3.Hash(), tx3, false, true) + pool.enqueueTx(tx1.Hash(), tx1, true) + pool.enqueueTx(tx2.Hash(), tx2, true) + pool.enqueueTx(tx3.Hash(), tx3, true) pool.promoteExecutables([]common.Address{from}) if len(pool.pending) != 1 { @@ -476,14 +472,14 @@ func TestChainFork(t *testing.T) { resetState() tx := transaction(0, 100000, key) - if _, err := pool.add(tx, false); err != nil { + if _, err := pool.add(tx); err != nil { t.Error("didn't expect error", err) } pool.removeTx(tx.Hash(), true, true) // reset the pool's internal state resetState() - if _, err := pool.add(tx, false); err != nil { + if _, err := pool.add(tx); err != nil { t.Error("didn't expect error", err) } } @@ -510,10 +506,10 @@ func TestDoubleNonce(t *testing.T) { tx3, _ := types.SignTx(types.NewTransaction(0, common.Address{}, big.NewInt(100), 1000000, big.NewInt(1), nil), signer, key) // Add the first two transaction, ensure higher priced stays only - if replace, err := pool.add(tx1, false); err != nil || replace { + if replace, err := pool.add(tx1); err != nil || replace { t.Errorf("first transaction insert failed (%v) or reported replacement (%v)", err, replace) } - if replace, err := pool.add(tx2, false); err != nil || !replace { + if replace, err := pool.add(tx2); err != nil || !replace { t.Errorf("second transaction insert failed (%v) or not reported replacement (%v)", err, replace) } <-pool.requestPromoteExecutables(newAccountSet(signer, addr)) @@ -525,7 +521,7 @@ func TestDoubleNonce(t *testing.T) { } // Add the third transaction and ensure it's not saved (smaller price) - pool.add(tx3, false) + pool.add(tx3) <-pool.requestPromoteExecutables(newAccountSet(signer, addr)) if pool.pending[addr].Len() != 1 { t.Error("expected 1 pending transactions, got", pool.pending[addr].Len()) @@ -548,7 +544,7 @@ func TestMissingNonce(t *testing.T) { addr := crypto.PubkeyToAddress(key.PublicKey) testAddBalance(pool, addr, big.NewInt(100000000000000)) tx := transaction(1, 100000, key) - if _, err := pool.add(tx, false); err != nil { + if _, err := pool.add(tx); err != nil { t.Error("didn't expect error", err) } if len(pool.pending) != 0 { @@ -607,21 +603,21 @@ func TestDropping(t *testing.T) { tx11 = transaction(11, 200, key) tx12 = transaction(12, 300, key) ) - pool.all.Add(tx0, false) - pool.priced.Put(tx0, false) + pool.all.Add(tx0) + pool.priced.Put(tx0) pool.promoteTx(account, tx0.Hash(), tx0) - pool.all.Add(tx1, false) - pool.priced.Put(tx1, false) + pool.all.Add(tx1) + pool.priced.Put(tx1) pool.promoteTx(account, tx1.Hash(), tx1) - pool.all.Add(tx2, false) - pool.priced.Put(tx2, false) + pool.all.Add(tx2) + pool.priced.Put(tx2) pool.promoteTx(account, tx2.Hash(), tx2) - pool.enqueueTx(tx10.Hash(), tx10, false, true) - pool.enqueueTx(tx11.Hash(), tx11, false, true) - pool.enqueueTx(tx12.Hash(), tx12, false, true) + pool.enqueueTx(tx10.Hash(), tx10, true) + pool.enqueueTx(tx11.Hash(), tx11, true) + pool.enqueueTx(tx12.Hash(), tx12, true) // Check that pre and post validations leave the pool as is if pool.pending[account].Len() != 3 { @@ -899,13 +895,6 @@ func TestQueueAccountLimiting(t *testing.T) { // This logic should not hold for local transactions, unless the local tracking // mechanism is disabled. func TestQueueGlobalLimiting(t *testing.T) { - testQueueGlobalLimiting(t, false) -} -func TestQueueGlobalLimitingNoLocals(t *testing.T) { - testQueueGlobalLimiting(t, true) -} - -func testQueueGlobalLimiting(t *testing.T, nolocals bool) { t.Parallel() // Create the pool to test the limit enforcement with @@ -913,7 +902,7 @@ func testQueueGlobalLimiting(t *testing.T, nolocals bool) { blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig - config.NoLocals = nolocals + config.NoLocals = true config.GlobalQueue = config.AccountQueue*3 - 1 // reduce the queue limits to shorten test time (-1 to make it non divisible) pool := New(config, blockchain) @@ -926,7 +915,6 @@ func testQueueGlobalLimiting(t *testing.T, nolocals bool) { keys[i], _ = crypto.GenerateKey() testAddBalance(pool, crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) } - local := keys[len(keys)-1] // Generate and queue a batch of transactions nonces := make(map[common.Address]uint64) @@ -952,51 +940,12 @@ func testQueueGlobalLimiting(t *testing.T, nolocals bool) { if queued > int(config.GlobalQueue) { t.Fatalf("total transactions overflow allowance: %d > %d", queued, config.GlobalQueue) } - // Generate a batch of transactions from the local account and import them - txs = txs[:0] - for i := uint64(0); i < 3*config.GlobalQueue; i++ { - txs = append(txs, transaction(i+1, 100000, local)) - } - pool.addLocals(txs) - - // If locals are disabled, the previous eviction algorithm should apply here too - if nolocals { - queued := 0 - for addr, list := range pool.queue { - if list.Len() > int(config.AccountQueue) { - t.Errorf("addr %x: queued accounts overflown allowance: %d > %d", addr, list.Len(), config.AccountQueue) - } - queued += list.Len() - } - if queued > int(config.GlobalQueue) { - t.Fatalf("total transactions overflow allowance: %d > %d", queued, config.GlobalQueue) - } - } else { - // Local exemptions are enabled, make sure the local account owned the queue - if len(pool.queue) != 1 { - t.Errorf("multiple accounts in queue: have %v, want %v", len(pool.queue), 1) - } - // Also ensure no local transactions are ever dropped, even if above global limits - if queued := pool.queue[crypto.PubkeyToAddress(local.PublicKey)].Len(); uint64(queued) != 3*config.GlobalQueue { - t.Fatalf("local account queued transaction count mismatch: have %v, want %v", queued, 3*config.GlobalQueue) - } - } } // Tests that if an account remains idle for a prolonged amount of time, any // non-executable transactions queued up are dropped to prevent wasting resources // on shuffling them around. -// -// This logic should not hold for local transactions, unless the local tracking -// mechanism is disabled. func TestQueueTimeLimiting(t *testing.T) { - testQueueTimeLimiting(t, false) -} -func TestQueueTimeLimitingNoLocals(t *testing.T) { - testQueueTimeLimiting(t, true) -} - -func testQueueTimeLimiting(t *testing.T, nolocals bool) { // Reduce the eviction interval to a testable amount defer func(old time.Duration) { evictionInterval = old }(evictionInterval) evictionInterval = time.Millisecond * 100 @@ -1007,23 +956,17 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { config := testTxPoolConfig config.Lifetime = time.Second - config.NoLocals = nolocals pool := New(config, blockchain) pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) defer pool.Close() - // Create two test accounts to ensure remotes expire but locals do not - local, _ := crypto.GenerateKey() + // Create a test account to ensure remotes expire remote, _ := crypto.GenerateKey() - testAddBalance(pool, crypto.PubkeyToAddress(local.PublicKey), big.NewInt(1000000000)) testAddBalance(pool, crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) - // Add the two transactions and ensure they both are queued up - if err := pool.addLocal(pricedTransaction(1, 100000, big.NewInt(1), local)); err != nil { - t.Fatalf("failed to add local transaction: %v", err) - } + // Add the transaction and ensure it is queued up if err := pool.addRemote(pricedTransaction(1, 100000, big.NewInt(1), remote)); err != nil { t.Fatalf("failed to add remote transaction: %v", err) } @@ -1031,7 +974,7 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { if pending != 0 { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 0) } - if queued != 2 { + if queued != 1 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2) } if err := validatePoolInternals(pool); err != nil { @@ -1046,7 +989,7 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { if pending != 0 { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 0) } - if queued != 2 { + if queued != 1 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2) } if err := validatePoolInternals(pool); err != nil { @@ -1060,14 +1003,8 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { if pending != 0 { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 0) } - if nolocals { - if queued != 0 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) - } - } else { - if queued != 1 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1) - } + if queued != 0 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) } if err := validatePoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) @@ -1075,7 +1012,6 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { // remove current transactions and increase nonce to prepare for a reset and cleanup statedb.SetNonce(crypto.PubkeyToAddress(remote.PublicKey), 2) - statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 2) <-pool.requestReset(nil, nil) // make sure queue, pending are cleared @@ -1091,18 +1027,12 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { } // Queue gapped transactions - if err := pool.addLocal(pricedTransaction(4, 100000, big.NewInt(1), local)); err != nil { - t.Fatalf("failed to add remote transaction: %v", err) - } if err := pool.addRemoteSync(pricedTransaction(4, 100000, big.NewInt(1), remote)); err != nil { t.Fatalf("failed to add remote transaction: %v", err) } time.Sleep(5 * evictionInterval) // A half lifetime pass // Queue executable transactions, the life cycle should be restarted. - if err := pool.addLocal(pricedTransaction(2, 100000, big.NewInt(1), local)); err != nil { - t.Fatalf("failed to add remote transaction: %v", err) - } if err := pool.addRemoteSync(pricedTransaction(2, 100000, big.NewInt(1), remote)); err != nil { t.Fatalf("failed to add remote transaction: %v", err) } @@ -1110,11 +1040,11 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { // All gapped transactions shouldn't be kicked out pending, queued = pool.Stats() - if pending != 2 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + if pending != 1 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 1) } - if queued != 2 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2) + if queued != 1 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1) } if err := validatePoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) @@ -1123,17 +1053,11 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { // The whole life time pass after last promotion, kick out stale transactions time.Sleep(2 * config.Lifetime) pending, queued = pool.Stats() - if pending != 2 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + if pending != 1 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 1) } - if nolocals { - if queued != 0 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) - } - } else { - if queued != 1 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1) - } + if queued != 0 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) } if err := validatePoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) @@ -1363,8 +1287,6 @@ func TestPendingMinimumAllowance(t *testing.T) { // Tests that setting the transaction pool gas price to a higher value correctly // discards everything cheaper than that and moves any gapped transactions back // from the pending pool to the queue. -// -// Note, local transactions are never allowed to be dropped. func TestRepricing(t *testing.T) { t.Parallel() @@ -1382,7 +1304,7 @@ func TestRepricing(t *testing.T) { defer sub.Unsubscribe() // Create a number of test accounts and fund them - keys := make([]*ecdsa.PrivateKey, 4) + keys := make([]*ecdsa.PrivateKey, 3) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() testAddBalance(pool, crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) @@ -1402,20 +1324,17 @@ func TestRepricing(t *testing.T) { txs = append(txs, pricedTransaction(2, 100000, big.NewInt(1), keys[2])) txs = append(txs, pricedTransaction(3, 100000, big.NewInt(2), keys[2])) - ltx := pricedTransaction(0, 100000, big.NewInt(1), keys[3]) - // Import the batch and that both pending and queued transactions match up pool.addRemotesSync(txs) - pool.addLocal(ltx) pending, queued := pool.Stats() - if pending != 7 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 7) + if pending != 6 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 6) } if queued != 3 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3) } - if err := validateEvents(events, 7); err != nil { + if err := validateEvents(events, 6); err != nil { t.Fatalf("original event firing failed: %v", err) } if err := validatePoolInternals(pool); err != nil { @@ -1425,8 +1344,8 @@ func TestRepricing(t *testing.T) { pool.SetGasTip(big.NewInt(2)) pending, queued = pool.Stats() - if pending != 2 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + if pending != 1 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 1) } if queued != 5 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 5) @@ -1453,21 +1372,7 @@ func TestRepricing(t *testing.T) { if err := validatePoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } - // However we can add local underpriced transactions - tx := pricedTransaction(1, 100000, big.NewInt(1), keys[3]) - if err := pool.addLocal(tx); err != nil { - t.Fatalf("failed to add underpriced local transaction: %v", err) - } - if pending, _ = pool.Stats(); pending != 3 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3) - } - if err := validateEvents(events, 1); err != nil { - t.Fatalf("post-reprice local event firing failed: %v", err) - } - if err := validatePoolInternals(pool); err != nil { - t.Fatalf("pool internal state corrupted: %v", err) - } - // And we can fill gaps with properly priced transactions + // we can fill gaps with properly priced transactions if err := pool.addRemote(pricedTransaction(1, 100000, big.NewInt(2), keys[0])); err != nil { t.Fatalf("failed to add pending transaction: %v", err) } @@ -1504,29 +1409,16 @@ func TestMinGasPriceEnforced(t *testing.T) { tx := pricedTransaction(0, 100000, big.NewInt(2), key) pool.SetGasTip(big.NewInt(tx.GasPrice().Int64() + 1)) - if err := pool.addLocal(tx); !errors.Is(err, txpool.ErrUnderpriced) { - t.Fatalf("Min tip not enforced") - } - - if err := pool.Add([]*types.Transaction{tx}, true, false)[0]; !errors.Is(err, txpool.ErrUnderpriced) { + if err := pool.Add([]*types.Transaction{tx}, true)[0]; !errors.Is(err, txpool.ErrUnderpriced) { t.Fatalf("Min tip not enforced") } tx = dynamicFeeTx(0, 100000, big.NewInt(3), big.NewInt(2), key) pool.SetGasTip(big.NewInt(tx.GasTipCap().Int64() + 1)) - if err := pool.addLocal(tx); !errors.Is(err, txpool.ErrUnderpriced) { + if err := pool.Add([]*types.Transaction{tx}, true)[0]; !errors.Is(err, txpool.ErrUnderpriced) { t.Fatalf("Min tip not enforced") } - - if err := pool.Add([]*types.Transaction{tx}, true, false)[0]; !errors.Is(err, txpool.ErrUnderpriced) { - t.Fatalf("Min tip not enforced") - } - // Make sure the tx is accepted if locals are enabled - pool.config.NoLocals = false - if err := pool.Add([]*types.Transaction{tx}, true, false)[0]; err != nil { - t.Fatalf("Min tip enforced with locals enabled, error: %v", err) - } } // Tests that setting the transaction pool gas price to a higher value correctly @@ -1567,20 +1459,17 @@ func TestRepricingDynamicFee(t *testing.T) { txs = append(txs, dynamicFeeTx(2, 100000, big.NewInt(1), big.NewInt(1), keys[2])) txs = append(txs, dynamicFeeTx(3, 100000, big.NewInt(2), big.NewInt(2), keys[2])) - ltx := dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[3]) - // Import the batch and that both pending and queued transactions match up pool.addRemotesSync(txs) - pool.addLocal(ltx) pending, queued := pool.Stats() - if pending != 7 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 7) + if pending != 6 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 6) } if queued != 3 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3) } - if err := validateEvents(events, 7); err != nil { + if err := validateEvents(events, 6); err != nil { t.Fatalf("original event firing failed: %v", err) } if err := validatePoolInternals(pool); err != nil { @@ -1590,8 +1479,8 @@ func TestRepricingDynamicFee(t *testing.T) { pool.SetGasTip(big.NewInt(2)) pending, queued = pool.Stats() - if pending != 2 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + if pending != 1 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 1) } if queued != 5 { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 5) @@ -1621,20 +1510,7 @@ func TestRepricingDynamicFee(t *testing.T) { if err := validatePoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } - // However we can add local underpriced transactions - tx = dynamicFeeTx(1, 100000, big.NewInt(1), big.NewInt(1), keys[3]) - if err := pool.addLocal(tx); err != nil { - t.Fatalf("failed to add underpriced local transaction: %v", err) - } - if pending, _ = pool.Stats(); pending != 3 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3) - } - if err := validateEvents(events, 1); err != nil { - t.Fatalf("post-reprice local event firing failed: %v", err) - } - if err := validatePoolInternals(pool); err != nil { - t.Fatalf("pool internal state corrupted: %v", err) - } + // And we can fill gaps with properly priced transactions tx = pricedTransaction(1, 100000, big.NewInt(2), keys[0]) if err := pool.addRemote(tx); err != nil { @@ -1656,77 +1532,6 @@ func TestRepricingDynamicFee(t *testing.T) { } } -// Tests that setting the transaction pool gas price to a higher value does not -// remove local transactions (legacy & dynamic fee). -func TestRepricingKeepsLocals(t *testing.T) { - t.Parallel() - - // Create the pool to test the pricing enforcement with - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) - - pool := New(testTxPoolConfig, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) - defer pool.Close() - - // Create a number of test accounts and fund them - keys := make([]*ecdsa.PrivateKey, 3) - for i := 0; i < len(keys); i++ { - keys[i], _ = crypto.GenerateKey() - testAddBalance(pool, crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(100000*1000000)) - } - // Create transaction (both pending and queued) with a linearly growing gasprice - for i := uint64(0); i < 500; i++ { - // Add pending transaction. - pendingTx := pricedTransaction(i, 100000, big.NewInt(int64(i)), keys[2]) - if err := pool.addLocal(pendingTx); err != nil { - t.Fatal(err) - } - // Add queued transaction. - queuedTx := pricedTransaction(i+501, 100000, big.NewInt(int64(i)), keys[2]) - if err := pool.addLocal(queuedTx); err != nil { - t.Fatal(err) - } - - // Add pending dynamic fee transaction. - pendingTx = dynamicFeeTx(i, 100000, big.NewInt(int64(i)+1), big.NewInt(int64(i)), keys[1]) - if err := pool.addLocal(pendingTx); err != nil { - t.Fatal(err) - } - // Add queued dynamic fee transaction. - queuedTx = dynamicFeeTx(i+501, 100000, big.NewInt(int64(i)+1), big.NewInt(int64(i)), keys[1]) - if err := pool.addLocal(queuedTx); err != nil { - t.Fatal(err) - } - } - pending, queued := pool.Stats() - expPending, expQueued := 1000, 1000 - validate := func() { - pending, queued = pool.Stats() - if pending != expPending { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, expPending) - } - if queued != expQueued { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, expQueued) - } - - if err := validatePoolInternals(pool); err != nil { - t.Fatalf("pool internal state corrupted: %v", err) - } - } - validate() - - // Reprice the pool and check that nothing is dropped - pool.SetGasTip(big.NewInt(2)) - validate() - - pool.SetGasTip(big.NewInt(2)) - pool.SetGasTip(big.NewInt(4)) - pool.SetGasTip(big.NewInt(8)) - pool.SetGasTip(big.NewInt(100)) - validate() -} - // Tests that when the pool reaches its global transaction limit, underpriced // transactions are gradually shifted out for more expensive ones and any gapped // pending transactions are moved into the queue. @@ -1756,21 +1561,18 @@ func TestUnderpricing(t *testing.T) { keys := make([]*ecdsa.PrivateKey, 5) for i := 0; i < len(keys); i++ { keys[i], _ = crypto.GenerateKey() - testAddBalance(pool, crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000)) + testAddBalance(pool, crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(10000000)) } // Generate and queue a batch of transactions, both pending and queued txs := types.Transactions{} - txs = append(txs, pricedTransaction(0, 100000, big.NewInt(1), keys[0])) - txs = append(txs, pricedTransaction(1, 100000, big.NewInt(2), keys[0])) - - txs = append(txs, pricedTransaction(1, 100000, big.NewInt(1), keys[1])) - - ltx := pricedTransaction(0, 100000, big.NewInt(1), keys[2]) + txs = append(txs, pricedTransaction(0, 100000, big.NewInt(1), keys[0])) // pending + txs = append(txs, pricedTransaction(1, 100000, big.NewInt(2), keys[0])) // pending + txs = append(txs, pricedTransaction(0, 100000, big.NewInt(1), keys[2])) // pending + txs = append(txs, pricedTransaction(1, 100000, big.NewInt(1), keys[1])) // queued // Import the batch and that both pending and queued transactions match up - pool.addRemotes(txs) - pool.addLocal(ltx) + pool.addRemotesSync(txs) pending, queued := pool.Stats() if pending != 3 { @@ -1790,7 +1592,7 @@ func TestUnderpricing(t *testing.T) { t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) } // Replace a future transaction with a future transaction - if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(2), keys[1])); err != nil { // +K1:1 => -K1:1 => Pend K0:0, K0:1, K2:0; Que K1:1 + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(5), keys[1])); err != nil { // +K1:1 => -K1:1 => Pend K0:0, K0:1, K2:0; Que K1:1 t.Fatalf("failed to add well priced transaction: %v", err) } // Ensure that adding high priced transactions drops cheap ones, but not own @@ -1800,48 +1602,26 @@ func TestUnderpricing(t *testing.T) { if err := pool.addRemoteSync(pricedTransaction(2, 100000, big.NewInt(4), keys[1])); err != nil { // +K1:2 => -K0:0 => Pend K1:0, K2:0; Que K0:1 K1:2 t.Fatalf("failed to add well priced transaction: %v", err) } - if err := pool.addRemote(pricedTransaction(3, 100000, big.NewInt(5), keys[1])); err != nil { // +K1:3 => -K0:1 => Pend K1:0, K2:0; Que K1:2 K1:3 + if err := pool.addRemoteSync(pricedTransaction(3, 100000, big.NewInt(5), keys[1])); err != nil { // +K1:3 => -K0:1 => Pend K1:0, K2:0; Que K1:2 K1:3 t.Fatalf("failed to add well priced transaction: %v", err) } // Ensure that replacing a pending transaction with a future transaction fails - if err := pool.addRemote(pricedTransaction(5, 100000, big.NewInt(6), keys[1])); err != txpool.ErrFutureReplacePending { + if err := pool.addRemoteSync(pricedTransaction(5, 100000, big.NewInt(6), keys[1])); err != txpool.ErrFutureReplacePending { t.Fatalf("adding future replace transaction error mismatch: have %v, want %v", err, txpool.ErrFutureReplacePending) } pending, queued = pool.Stats() - if pending != 2 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + if pending != 4 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 4) } - if queued != 2 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2) + if queued != 0 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) } - if err := validateEvents(events, 2); err != nil { + if err := validateEvents(events, 4); err != nil { t.Fatalf("additional event firing failed: %v", err) } if err := validatePoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } - // Ensure that adding local transactions can push out even higher priced ones - ltx = pricedTransaction(1, 100000, big.NewInt(0), keys[2]) - if err := pool.addLocal(ltx); err != nil { - t.Fatalf("failed to append underpriced local transaction: %v", err) - } - ltx = pricedTransaction(0, 100000, big.NewInt(0), keys[3]) - if err := pool.addLocal(ltx); err != nil { - t.Fatalf("failed to add new underpriced local transaction: %v", err) - } - pending, queued = pool.Stats() - if pending != 3 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3) - } - if queued != 1 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1) - } - if err := validateEvents(events, 2); err != nil { - t.Fatalf("local event firing failed: %v", err) - } - if err := validatePoolInternals(pool); err != nil { - t.Fatalf("pool internal state corrupted: %v", err) - } } // Tests that more expensive transactions push out cheap ones from the pool, but @@ -1915,8 +1695,6 @@ func TestStableUnderpricing(t *testing.T) { // Tests that when the pool reaches its global transaction limit, underpriced // transactions (legacy & dynamic fee) are gradually shifted out for more // expensive ones and any gapped pending transactions are moved into the queue. -// -// Note, local transactions are never allowed to be dropped. func TestUnderpricingDynamicFee(t *testing.T) { t.Parallel() @@ -1941,15 +1719,13 @@ func TestUnderpricingDynamicFee(t *testing.T) { // Generate and queue a batch of transactions, both pending and queued txs := types.Transactions{} - txs = append(txs, dynamicFeeTx(0, 100000, big.NewInt(3), big.NewInt(2), keys[0])) - txs = append(txs, pricedTransaction(1, 100000, big.NewInt(2), keys[0])) - txs = append(txs, dynamicFeeTx(1, 100000, big.NewInt(2), big.NewInt(1), keys[1])) + txs = append(txs, dynamicFeeTx(0, 100000, big.NewInt(3), big.NewInt(2), keys[0])) // pending + txs = append(txs, pricedTransaction(1, 100000, big.NewInt(2), keys[0])) // pending + txs = append(txs, dynamicFeeTx(1, 100000, big.NewInt(2), big.NewInt(1), keys[1])) // queued + txs = append(txs, dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[2])) // pending - ltx := dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[2]) - - // Import the batch and that both pending and queued transactions match up - pool.addRemotes(txs) // Pend K0:0, K0:1; Que K1:1 - pool.addLocal(ltx) // +K2:0 => Pend K0:0, K0:1, K2:0; Que K1:1 + // Import the batch and check that both pending and queued transactions match up + pool.addRemotesSync(txs) // Pend K0:0, K0:1; Que K1:1 pending, queued := pool.Stats() if pending != 3 { @@ -1967,13 +1743,13 @@ func TestUnderpricingDynamicFee(t *testing.T) { // Ensure that adding an underpriced transaction fails tx := dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[1]) - if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrUnderpriced) { // Pend K0:0, K0:1, K2:0; Que K1:1 + if err := pool.addRemoteSync(tx); !errors.Is(err, txpool.ErrUnderpriced) { // Pend K0:0, K0:1, K2:0; Que K1:1 t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) } // Ensure that adding high priced transactions drops cheap ones, but not own tx = pricedTransaction(0, 100000, big.NewInt(2), keys[1]) - if err := pool.addRemote(tx); err != nil { // +K1:0, -K1:1 => Pend K0:0, K0:1, K1:0, K2:0; Que - + if err := pool.addRemoteSync(tx); err != nil { // +K1:0, -K1:1 => Pend K0:0, K0:1, K1:0, K2:0; Que - t.Fatalf("failed to add well priced transaction: %v", err) } @@ -1986,40 +1762,18 @@ func TestUnderpricingDynamicFee(t *testing.T) { t.Fatalf("failed to add well priced transaction: %v", err) } pending, queued = pool.Stats() - if pending != 2 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + if pending != 4 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 4) } - if queued != 2 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2) + if queued != 0 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) } - if err := validateEvents(events, 2); err != nil { + if err := validateEvents(events, 3); err != nil { t.Fatalf("additional event firing failed: %v", err) } if err := validatePoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } - // Ensure that adding local transactions can push out even higher priced ones - ltx = dynamicFeeTx(1, 100000, big.NewInt(0), big.NewInt(0), keys[2]) - if err := pool.addLocal(ltx); err != nil { - t.Fatalf("failed to append underpriced local transaction: %v", err) - } - ltx = dynamicFeeTx(0, 100000, big.NewInt(0), big.NewInt(0), keys[3]) - if err := pool.addLocal(ltx); err != nil { - t.Fatalf("failed to add new underpriced local transaction: %v", err) - } - pending, queued = pool.Stats() - if pending != 3 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3) - } - if queued != 1 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1) - } - if err := validateEvents(events, 2); err != nil { - t.Fatalf("local event firing failed: %v", err) - } - if err := validatePoolInternals(pool); err != nil { - t.Fatalf("pool internal state corrupted: %v", err) - } } // Tests whether highest fee cap transaction is retained after a batch of high effective @@ -2039,7 +1793,7 @@ func TestDualHeapEviction(t *testing.T) { ) check := func(tx *types.Transaction, name string) { - if pool.all.GetRemote(tx.Hash()) == nil { + if pool.all.Get(tx.Hash()) == nil { t.Fatalf("highest %s transaction evicted from the pool", name) } } @@ -2336,122 +2090,6 @@ func TestReplacementDynamicFee(t *testing.T) { } } -// Tests that local transactions are journaled to disk, but remote transactions -// get discarded between restarts. -func TestJournaling(t *testing.T) { testJournaling(t, false) } -func TestJournalingNoLocals(t *testing.T) { testJournaling(t, true) } - -func testJournaling(t *testing.T, nolocals bool) { - t.Parallel() - - // Create a temporary file for the journal - file, err := os.CreateTemp("", "") - if err != nil { - t.Fatalf("failed to create temporary journal: %v", err) - } - journal := file.Name() - defer os.Remove(journal) - - // Clean up the temporary file, we only need the path for now - file.Close() - os.Remove(journal) - - // Create the original pool to inject transaction into the journal - statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) - blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) - - config := testTxPoolConfig - config.NoLocals = nolocals - config.Journal = journal - config.Rejournal = time.Second - - pool := New(config, blockchain) - pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) - - // Create two test accounts to ensure remotes expire but locals do not - local, _ := crypto.GenerateKey() - remote, _ := crypto.GenerateKey() - - testAddBalance(pool, crypto.PubkeyToAddress(local.PublicKey), big.NewInt(1000000000)) - testAddBalance(pool, crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) - - // Add three local and a remote transactions and ensure they are queued up - if err := pool.addLocal(pricedTransaction(0, 100000, big.NewInt(1), local)); err != nil { - t.Fatalf("failed to add local transaction: %v", err) - } - if err := pool.addLocal(pricedTransaction(1, 100000, big.NewInt(1), local)); err != nil { - t.Fatalf("failed to add local transaction: %v", err) - } - if err := pool.addLocal(pricedTransaction(2, 100000, big.NewInt(1), local)); err != nil { - t.Fatalf("failed to add local transaction: %v", err) - } - if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), remote)); err != nil { - t.Fatalf("failed to add remote transaction: %v", err) - } - pending, queued := pool.Stats() - if pending != 4 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 4) - } - if queued != 0 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) - } - if err := validatePoolInternals(pool); err != nil { - t.Fatalf("pool internal state corrupted: %v", err) - } - // Terminate the old pool, bump the local nonce, create a new pool and ensure relevant transaction survive - pool.Close() - statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1) - blockchain = newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) - - pool = New(config, blockchain) - pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) - - pending, queued = pool.Stats() - if queued != 0 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) - } - if nolocals { - if pending != 0 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 0) - } - } else { - if pending != 2 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) - } - } - if err := validatePoolInternals(pool); err != nil { - t.Fatalf("pool internal state corrupted: %v", err) - } - // Bump the nonce temporarily and ensure the newly invalidated transaction is removed - statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 2) - <-pool.requestReset(nil, nil) - time.Sleep(2 * config.Rejournal) - pool.Close() - - statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1) - blockchain = newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) - pool = New(config, blockchain) - pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) - - pending, queued = pool.Stats() - if pending != 0 { - t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 0) - } - if nolocals { - if queued != 0 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) - } - } else { - if queued != 1 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1) - } - } - if err := validatePoolInternals(pool); err != nil { - t.Fatalf("pool internal state corrupted: %v", err) - } - pool.Close() -} - // TestStatusCheck tests that the pool can correctly retrieve the // pending status of individual transactions. func TestStatusCheck(t *testing.T) { @@ -2566,7 +2204,7 @@ func benchmarkFuturePromotion(b *testing.B, size int) { for i := 0; i < size; i++ { tx := transaction(uint64(1+i), 100000, key) - pool.enqueueTx(tx.Hash(), tx, false, true) + pool.enqueueTx(tx.Hash(), tx, true) } // Benchmark the speed of pool validation b.ResetTimer() @@ -2576,15 +2214,11 @@ func benchmarkFuturePromotion(b *testing.B, size int) { } // Benchmarks the speed of batched transaction insertion. -func BenchmarkBatchInsert100(b *testing.B) { benchmarkBatchInsert(b, 100, false) } -func BenchmarkBatchInsert1000(b *testing.B) { benchmarkBatchInsert(b, 1000, false) } -func BenchmarkBatchInsert10000(b *testing.B) { benchmarkBatchInsert(b, 10000, false) } +func BenchmarkBatchInsert100(b *testing.B) { benchmarkBatchInsert(b, 100) } +func BenchmarkBatchInsert1000(b *testing.B) { benchmarkBatchInsert(b, 1000) } +func BenchmarkBatchInsert10000(b *testing.B) { benchmarkBatchInsert(b, 10000) } -func BenchmarkBatchLocalInsert100(b *testing.B) { benchmarkBatchInsert(b, 100, true) } -func BenchmarkBatchLocalInsert1000(b *testing.B) { benchmarkBatchInsert(b, 1000, true) } -func BenchmarkBatchLocalInsert10000(b *testing.B) { benchmarkBatchInsert(b, 10000, true) } - -func benchmarkBatchInsert(b *testing.B, size int, local bool) { +func benchmarkBatchInsert(b *testing.B, size int) { // Generate a batch of transactions to enqueue into the pool pool, key := setupPool() defer pool.Close() @@ -2602,46 +2236,7 @@ func benchmarkBatchInsert(b *testing.B, size int, local bool) { // Benchmark importing the transactions into the queue b.ResetTimer() for _, batch := range batches { - if local { - pool.addLocals(batch) - } else { - pool.addRemotes(batch) - } - } -} - -func BenchmarkInsertRemoteWithAllLocals(b *testing.B) { - // Allocate keys for testing - key, _ := crypto.GenerateKey() - account := crypto.PubkeyToAddress(key.PublicKey) - - remoteKey, _ := crypto.GenerateKey() - remoteAddr := crypto.PubkeyToAddress(remoteKey.PublicKey) - - locals := make([]*types.Transaction, 4096+1024) // Occupy all slots - for i := 0; i < len(locals); i++ { - locals[i] = transaction(uint64(i), 100000, key) - } - remotes := make([]*types.Transaction, 1000) - for i := 0; i < len(remotes); i++ { - remotes[i] = pricedTransaction(uint64(i), 100000, big.NewInt(2), remoteKey) // Higher gasprice - } - // Benchmark importing the transactions into the queue - b.ResetTimer() - for i := 0; i < b.N; i++ { - b.StopTimer() - pool, _ := setupPool() - testAddBalance(pool, account, big.NewInt(100000000)) - for _, local := range locals { - pool.addLocal(local) - } - b.StartTimer() - // Assign a high enough balance for testing - testAddBalance(pool, remoteAddr, big.NewInt(100000000)) - for i := 0; i < len(remotes); i++ { - pool.addRemotes([]*types.Transaction{remotes[i]}) - } - pool.Close() + pool.addRemotes(batch) } } diff --git a/core/txpool/legacypool/list.go b/core/txpool/legacypool/list.go index b749db44d4..736c28ec4a 100644 --- a/core/txpool/legacypool/list.go +++ b/core/txpool/legacypool/list.go @@ -52,31 +52,31 @@ func (h *nonceHeap) Pop() interface{} { return x } -// sortedMap is a nonce->transaction hash map with a heap based index to allow +// SortedMap is a nonce->transaction hash map with a heap based index to allow // iterating over the contents in a nonce-incrementing way. -type sortedMap struct { +type SortedMap struct { items map[uint64]*types.Transaction // Hash map storing the transaction data index *nonceHeap // Heap of nonces of all the stored transactions (non-strict mode) cache types.Transactions // Cache of the transactions already sorted cacheMu sync.Mutex // Mutex covering the cache } -// newSortedMap creates a new nonce-sorted transaction map. -func newSortedMap() *sortedMap { - return &sortedMap{ +// NewSortedMap creates a new nonce-sorted transaction map. +func NewSortedMap() *SortedMap { + return &SortedMap{ items: make(map[uint64]*types.Transaction), index: new(nonceHeap), } } // Get retrieves the current transactions associated with the given nonce. -func (m *sortedMap) Get(nonce uint64) *types.Transaction { +func (m *SortedMap) Get(nonce uint64) *types.Transaction { return m.items[nonce] } // Put inserts a new transaction into the map, also updating the map's nonce // index. If a transaction already exists with the same nonce, it's overwritten. -func (m *sortedMap) Put(tx *types.Transaction) { +func (m *SortedMap) Put(tx *types.Transaction) { nonce := tx.Nonce() if m.items[nonce] == nil { heap.Push(m.index, nonce) @@ -89,7 +89,7 @@ func (m *sortedMap) Put(tx *types.Transaction) { // Forward removes all transactions from the map with a nonce lower than the // provided threshold. Every removed transaction is returned for any post-removal // maintenance. -func (m *sortedMap) Forward(threshold uint64) types.Transactions { +func (m *SortedMap) Forward(threshold uint64) types.Transactions { var removed types.Transactions // Pop off heap items until the threshold is reached @@ -112,7 +112,7 @@ func (m *sortedMap) Forward(threshold uint64) types.Transactions { // Filter, as opposed to 'filter', re-initialises the heap after the operation is done. // If you want to do several consecutive filterings, it's therefore better to first // do a .filter(func1) followed by .Filter(func2) or reheap() -func (m *sortedMap) Filter(filter func(*types.Transaction) bool) types.Transactions { +func (m *SortedMap) Filter(filter func(*types.Transaction) bool) types.Transactions { removed := m.filter(filter) // If transactions were removed, the heap and cache are ruined if len(removed) > 0 { @@ -121,7 +121,7 @@ func (m *sortedMap) Filter(filter func(*types.Transaction) bool) types.Transacti return removed } -func (m *sortedMap) reheap() { +func (m *SortedMap) reheap() { *m.index = make([]uint64, 0, len(m.items)) for nonce := range m.items { *m.index = append(*m.index, nonce) @@ -134,7 +134,7 @@ func (m *sortedMap) reheap() { // filter is identical to Filter, but **does not** regenerate the heap. This method // should only be used if followed immediately by a call to Filter or reheap() -func (m *sortedMap) filter(filter func(*types.Transaction) bool) types.Transactions { +func (m *SortedMap) filter(filter func(*types.Transaction) bool) types.Transactions { var removed types.Transactions // Collect all the transactions to filter out @@ -154,7 +154,7 @@ func (m *sortedMap) filter(filter func(*types.Transaction) bool) types.Transacti // Cap places a hard limit on the number of items, returning all transactions // exceeding that limit. -func (m *sortedMap) Cap(threshold int) types.Transactions { +func (m *SortedMap) Cap(threshold int) types.Transactions { // Short circuit if the number of items is under the limit if len(m.items) <= threshold { return nil @@ -181,7 +181,7 @@ func (m *sortedMap) Cap(threshold int) types.Transactions { // Remove deletes a transaction from the maintained map, returning whether the // transaction was found. -func (m *sortedMap) Remove(nonce uint64) bool { +func (m *SortedMap) Remove(nonce uint64) bool { // Short circuit if no transaction is present _, ok := m.items[nonce] if !ok { @@ -209,7 +209,7 @@ func (m *sortedMap) Remove(nonce uint64) bool { // Note, all transactions with nonces lower than start will also be returned to // prevent getting into an invalid state. This is not something that should ever // happen but better to be self correcting than failing! -func (m *sortedMap) Ready(start uint64) types.Transactions { +func (m *SortedMap) Ready(start uint64) types.Transactions { // Short circuit if no transactions are available if m.index.Len() == 0 || (*m.index)[0] > start { return nil @@ -229,11 +229,11 @@ func (m *sortedMap) Ready(start uint64) types.Transactions { } // Len returns the length of the transaction map. -func (m *sortedMap) Len() int { +func (m *SortedMap) Len() int { return len(m.items) } -func (m *sortedMap) flatten() types.Transactions { +func (m *SortedMap) flatten() types.Transactions { m.cacheMu.Lock() defer m.cacheMu.Unlock() // If the sorting was not cached yet, create and cache it @@ -250,7 +250,7 @@ func (m *sortedMap) flatten() types.Transactions { // Flatten creates a nonce-sorted slice of transactions based on the loosely // sorted internal representation. The result of the sorting is cached in case // it's requested again before any modifications are made to the contents. -func (m *sortedMap) Flatten() types.Transactions { +func (m *SortedMap) Flatten() types.Transactions { cache := m.flatten() // Copy the cache to prevent accidental modification txs := make(types.Transactions, len(cache)) @@ -260,7 +260,7 @@ func (m *sortedMap) Flatten() types.Transactions { // LastElement returns the last element of a flattened list, thus, the // transaction with the highest nonce -func (m *sortedMap) LastElement() *types.Transaction { +func (m *SortedMap) LastElement() *types.Transaction { cache := m.flatten() return cache[len(cache)-1] } @@ -271,7 +271,7 @@ func (m *sortedMap) LastElement() *types.Transaction { // executable/future queue, with minor behavioral changes. type list struct { strict bool // Whether nonces are strictly continuous or not - txs *sortedMap // Heap indexed sorted hash map of the transactions + txs *SortedMap // Heap indexed sorted hash map of the transactions costcap *uint256.Int // Price of the highest costing transaction (reset only if exceeds balance) gascap uint64 // Gas limit of the highest spending transaction (reset only if exceeds block limit) @@ -283,7 +283,7 @@ type list struct { func newList(strict bool) *list { return &list{ strict: strict, - txs: newSortedMap(), + txs: NewSortedMap(), costcap: new(uint256.Int), totalcost: new(uint256.Int), } @@ -556,10 +556,7 @@ func newPricedList(all *lookup) *pricedList { } // Put inserts a new transaction into the heap. -func (l *pricedList) Put(tx *types.Transaction, local bool) { - if local { - return - } +func (l *pricedList) Put(tx *types.Transaction) { // Insert every new transaction to the urgent heap first; Discard will balance the heaps heap.Push(&l.urgent, tx) } @@ -593,7 +590,7 @@ func (l *pricedList) underpricedFor(h *priceHeap, tx *types.Transaction) bool { // Discard stale price points if found at the heap start for len(h.list) > 0 { head := h.list[0] - if l.all.GetRemote(head.Hash()) == nil { // Removed or migrated + if l.all.Get(head.Hash()) == nil { // Removed or migrated l.stales.Add(-1) heap.Pop(h) continue @@ -612,15 +609,13 @@ func (l *pricedList) underpricedFor(h *priceHeap, tx *types.Transaction) bool { // Discard finds a number of most underpriced transactions, removes them from the // priced list and returns them for further removal from the entire pool. // If noPending is set to true, we will only consider the floating list -// -// Note local transaction won't be considered for eviction. -func (l *pricedList) Discard(slots int, force bool) (types.Transactions, bool) { +func (l *pricedList) Discard(slots int) (types.Transactions, bool) { drop := make(types.Transactions, 0, slots) // Remote underpriced transactions to drop for slots > 0 { if len(l.urgent.list)*floatingRatio > len(l.floating.list)*urgentRatio { // Discard stale transactions if found during cleanup tx := heap.Pop(&l.urgent).(*types.Transaction) - if l.all.GetRemote(tx.Hash()) == nil { // Removed or migrated + if l.all.Get(tx.Hash()) == nil { // Removed or migrated l.stales.Add(-1) continue } @@ -633,7 +628,7 @@ func (l *pricedList) Discard(slots int, force bool) (types.Transactions, bool) { } // Discard stale transactions if found during cleanup tx := heap.Pop(&l.floating).(*types.Transaction) - if l.all.GetRemote(tx.Hash()) == nil { // Removed or migrated + if l.all.Get(tx.Hash()) == nil { // Removed or migrated l.stales.Add(-1) continue } @@ -643,7 +638,7 @@ func (l *pricedList) Discard(slots int, force bool) (types.Transactions, bool) { } } // If we still can't make enough room for the new transaction - if slots > 0 && !force { + if slots > 0 { for _, tx := range drop { heap.Push(&l.urgent, tx) } @@ -658,11 +653,11 @@ func (l *pricedList) Reheap() { defer l.reheapMu.Unlock() start := time.Now() l.stales.Store(0) - l.urgent.list = make([]*types.Transaction, 0, l.all.RemoteCount()) - l.all.Range(func(hash common.Hash, tx *types.Transaction, local bool) bool { + l.urgent.list = make([]*types.Transaction, 0, l.all.Count()) + l.all.Range(func(hash common.Hash, tx *types.Transaction) bool { l.urgent.list = append(l.urgent.list, tx) return true - }, false, true) // Only iterate remotes + }) heap.Init(&l.urgent) // balance out the two heaps by moving the worse half of transactions into the diff --git a/core/txpool/legacypool/journal.go b/core/txpool/locals/journal.go similarity index 99% rename from core/txpool/legacypool/journal.go rename to core/txpool/locals/journal.go index 899ed00bcc..46fd6de346 100644 --- a/core/txpool/legacypool/journal.go +++ b/core/txpool/locals/journal.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package legacypool +package locals import ( "errors" diff --git a/core/txpool/locals/tx_tracker.go b/core/txpool/locals/tx_tracker.go new file mode 100644 index 0000000000..a24fcb1f4e --- /dev/null +++ b/core/txpool/locals/tx_tracker.go @@ -0,0 +1,212 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package locals implements tracking for "local" transactions +package locals + +import ( + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/params" + "golang.org/x/exp/slices" +) + +var ( + recheckInterval = time.Minute + localGauge = metrics.GetOrRegisterGauge("txpool/local", nil) +) + +// TxTracker is a struct used to track priority transactions; it will check from +// time to time if the main pool has forgotten about any of the transaction +// it is tracking, and if so, submit it again. +// This is used to track 'locals'. +// This struct does not care about transaction validity, price-bumps or account limits, +// but optimistically accepts transactions. +type TxTracker struct { + all map[common.Hash]*types.Transaction // All tracked transactions + byAddr map[common.Address]*legacypool.SortedMap // Transactions by address + + journal *journal // Journal of local transaction to back up to disk + rejournal time.Duration // How often to rotate journal + pool *txpool.TxPool // The tx pool to interact with + signer types.Signer + + shutdownCh chan struct{} + mu sync.Mutex + wg sync.WaitGroup +} + +// New creates a new TxTracker +func New(journalPath string, journalTime time.Duration, chainConfig *params.ChainConfig, next *txpool.TxPool) *TxTracker { + pool := &TxTracker{ + all: make(map[common.Hash]*types.Transaction), + byAddr: make(map[common.Address]*legacypool.SortedMap), + signer: types.LatestSigner(chainConfig), + shutdownCh: make(chan struct{}), + pool: next, + } + if journalPath != "" { + pool.journal = newTxJournal(journalPath) + pool.rejournal = journalTime + } + return pool +} + +// Track adds a transaction to the tracked set. +// Note: blob-type transactions are ignored. +func (tracker *TxTracker) Track(tx *types.Transaction) { + tracker.TrackAll([]*types.Transaction{tx}) +} + +// TrackAll adds a list of transactions to the tracked set. +// Note: blob-type transactions are ignored. +func (tracker *TxTracker) TrackAll(txs []*types.Transaction) { + tracker.mu.Lock() + defer tracker.mu.Unlock() + + for _, tx := range txs { + if tx.Type() == types.BlobTxType { + continue + } + // If we're already tracking it, it's a no-op + if _, ok := tracker.all[tx.Hash()]; ok { + continue + } + addr, err := types.Sender(tracker.signer, tx) + if err != nil { // Ignore this tx + continue + } + tracker.all[tx.Hash()] = tx + if tracker.byAddr[addr] == nil { + tracker.byAddr[addr] = legacypool.NewSortedMap() + } + tracker.byAddr[addr].Put(tx) + + if tracker.journal != nil { + _ = tracker.journal.insert(tx) + } + } + localGauge.Update(int64(len(tracker.all))) +} + +// recheck checks and returns any transactions that needs to be resubmitted. +func (tracker *TxTracker) recheck(journalCheck bool) (resubmits []*types.Transaction, rejournal map[common.Address]types.Transactions) { + tracker.mu.Lock() + defer tracker.mu.Unlock() + + var ( + numStales = 0 + numOk = 0 + ) + for sender, txs := range tracker.byAddr { + // Wipe the stales + stales := txs.Forward(tracker.pool.Nonce(sender)) + for _, tx := range stales { + delete(tracker.all, tx.Hash()) + } + numStales += len(stales) + + // Check the non-stale + for _, tx := range txs.Flatten() { + if tracker.pool.Has(tx.Hash()) { + numOk++ + continue + } + resubmits = append(resubmits, tx) + } + } + + if journalCheck { // rejournal + rejournal = make(map[common.Address]types.Transactions) + for _, tx := range tracker.all { + addr, _ := types.Sender(tracker.signer, tx) + rejournal[addr] = append(rejournal[addr], tx) + } + // Sort them + for _, list := range rejournal { + // cmp(a, b) should return a negative number when a < b, + slices.SortFunc(list, func(a, b *types.Transaction) int { + return int(a.Nonce() - b.Nonce()) + }) + } + } + localGauge.Update(int64(len(tracker.all))) + log.Debug("Tx tracker status", "need-resubmit", len(resubmits), "stale", numStales, "ok", numOk) + return resubmits, rejournal +} + +// Start implements node.Lifecycle interface +// Start is called after all services have been constructed and the networking +// layer was also initialized to spawn any goroutines required by the service. +func (tracker *TxTracker) Start() error { + tracker.wg.Add(1) + go tracker.loop() + return nil +} + +// Stop implements node.Lifecycle interface +// Stop terminates all goroutines belonging to the service, blocking until they +// are all terminated. +func (tracker *TxTracker) Stop() error { + close(tracker.shutdownCh) + tracker.wg.Wait() + return nil +} + +func (tracker *TxTracker) loop() { + defer tracker.wg.Done() + + if tracker.journal != nil { + tracker.journal.load(func(transactions []*types.Transaction) []error { + tracker.TrackAll(transactions) + return nil + }) + defer tracker.journal.close() + } + var ( + lastJournal = time.Now() + timer = time.NewTimer(10 * time.Second) // Do initial check after 10 seconds, do rechecks more seldom. + ) + for { + select { + case <-tracker.shutdownCh: + return + case <-timer.C: + checkJournal := tracker.journal != nil && time.Since(lastJournal) > tracker.rejournal + resubmits, rejournal := tracker.recheck(checkJournal) + if len(resubmits) > 0 { + tracker.pool.Add(resubmits, false) + } + if checkJournal { + // Lock to prevent journal.rotate <-> journal.insert (via TrackAll) conflicts + tracker.mu.Lock() + lastJournal = time.Now() + if err := tracker.journal.rotate(rejournal); err != nil { + log.Warn("Transaction journal rotation failed", "err", err) + } + tracker.mu.Unlock() + } + timer.Reset(recheckInterval) + } + } +} diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index 9ee0a69c0b..5ad0f5b0e0 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -132,7 +132,7 @@ type SubPool interface { // Add enqueues a batch of transactions into the pool if they are valid. Due // to the large transaction churn, add may postpone fully integrating the tx // to a later point to batch multiple ones together. - Add(txs []*types.Transaction, local bool, sync bool) []error + Add(txs []*types.Transaction, sync bool) []error // Pending retrieves all currently processable transactions, grouped by origin // account and sorted by nonce. @@ -162,9 +162,6 @@ type SubPool interface { // pending as well as queued transactions of this address, grouped by nonce. ContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) - // Locals retrieves the accounts currently considered local by the pool. - Locals() []common.Address - // Status returns the known status (unknown/pending/queued) of a transaction // identified by their hashes. Status(hash common.Hash) TxStatus diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 182706d63c..361dbe8b38 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -328,7 +328,7 @@ func (p *TxPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Pr // Add enqueues a batch of transactions into the pool if they are valid. Due // to the large transaction churn, add may postpone fully integrating the tx // to a later point to batch multiple ones together. -func (p *TxPool) Add(txs []*types.Transaction, local bool, sync bool) []error { +func (p *TxPool) Add(txs []*types.Transaction, sync bool) []error { // Split the input transactions between the subpools. It shouldn't really // happen that we receive merged batches, but better graceful than strange // errors. @@ -355,7 +355,7 @@ func (p *TxPool) Add(txs []*types.Transaction, local bool, sync bool) []error { // back the errors into the original sort order. errsets := make([][]error, len(p.subpools)) for i := 0; i < len(p.subpools); i++ { - errsets[i] = p.subpools[i].Add(txsets[i], local, sync) + errsets[i] = p.subpools[i].Add(txsets[i], sync) } errs := make([]error, len(txs)) for i, split := range splits { @@ -456,23 +456,6 @@ func (p *TxPool) ContentFrom(addr common.Address) ([]*types.Transaction, []*type return []*types.Transaction{}, []*types.Transaction{} } -// Locals retrieves the accounts currently considered local by the pool. -func (p *TxPool) Locals() []common.Address { - // Retrieve the locals from each subpool and deduplicate them - locals := make(map[common.Address]struct{}) - for _, subpool := range p.subpools { - for _, local := range subpool.Locals() { - locals[local] = struct{}{} - } - } - // Flatten and return the deduplicated local set - flat := make([]common.Address, 0, len(locals)) - for local := range locals { - flat = append(flat, local) - } - return flat -} - // Status returns the known status (unknown/pending/queued) of a transaction // identified by its hash. func (p *TxPool) Status(hash common.Hash) TxStatus { diff --git a/eth/api_backend.go b/eth/api_backend.go index 52ecd91789..66621190dd 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -272,7 +272,10 @@ func (b *EthAPIBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscri } func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { - return b.eth.txPool.Add([]*types.Transaction{signedTx}, true, false)[0] + if locals := b.eth.localTxTracker; locals != nil { + locals.Track(signedTx) + } + return b.eth.txPool.Add([]*types.Transaction{signedTx}, false)[0] } func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { diff --git a/eth/backend.go b/eth/backend.go index a3aa0a7b9b..fea7e4e1fe 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -23,6 +23,7 @@ import ( "math/big" "runtime" "sync" + "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" @@ -35,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" + "github.com/ethereum/go-ethereum/core/txpool/locals" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" @@ -67,9 +69,10 @@ type Config = ethconfig.Config // Ethereum implements the Ethereum full node service. type Ethereum struct { // core protocol objects - config *ethconfig.Config - txPool *txpool.TxPool - blockchain *core.BlockChain + config *ethconfig.Config + txPool *txpool.TxPool + localTxTracker *locals.TxTracker + blockchain *core.BlockChain handler *handler discmix *enode.FairMix @@ -237,6 +240,16 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } + + if !config.TxPool.NoLocals { + rejournal := config.TxPool.Rejournal + if rejournal < time.Second { + log.Warn("Sanitizing invalid txpool journal time", "provided", rejournal, "updated", time.Second) + rejournal = time.Second + } + eth.localTxTracker = locals.New(config.TxPool.Journal, rejournal, eth.blockchain.Config(), eth.txPool) + stack.RegisterLifecycle(eth.localTxTracker) + } // Permit the downloader to use the trie cache allowance during fast sync cacheLimit := cacheConfig.TrieCleanLimit + cacheConfig.TrieDirtyLimit + cacheConfig.SnapshotLimit if eth.handler, err = newHandler(&handlerConfig{ @@ -255,6 +268,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { eth.miner = miner.New(eth, config.Miner, eth.engine) eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) + eth.miner.SetPrioAddresses(config.TxPool.Locals) eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil} if eth.APIBackend.allowUnprotectedTxs { diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 9840d9c6ad..8fc3361192 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -115,7 +115,7 @@ func TestEth2AssembleBlock(t *testing.T) { if err != nil { t.Fatalf("error signing transaction, err=%v", err) } - ethservice.TxPool().Add([]*types.Transaction{tx}, true, true) + ethservice.TxPool().Add([]*types.Transaction{tx}, true) blockParams := engine.PayloadAttributes{ Timestamp: blocks[9].Time() + 5, } @@ -152,7 +152,7 @@ func TestEth2AssembleBlockWithAnotherBlocksTxs(t *testing.T) { // Put the 10th block's tx in the pool and produce a new block txs := blocks[9].Transactions() - api.eth.TxPool().Add(txs, false, true) + api.eth.TxPool().Add(txs, true) blockParams := engine.PayloadAttributes{ Timestamp: blocks[8].Time() + 5, } @@ -174,7 +174,7 @@ func TestEth2PrepareAndGetPayload(t *testing.T) { // Put the 10th block's tx in the pool and produce a new block txs := blocks[9].Transactions() - ethservice.TxPool().Add(txs, true, true) + ethservice.TxPool().Add(txs, true) blockParams := engine.PayloadAttributes{ Timestamp: blocks[8].Time() + 5, } @@ -294,7 +294,7 @@ func TestEth2NewBlock(t *testing.T) { statedb, _ := ethservice.BlockChain().StateAt(parent.Root()) nonce := statedb.GetNonce(testAddr) tx, _ := types.SignTx(types.NewContractCreation(nonce, new(big.Int), 1000000, big.NewInt(2*params.InitialBaseFee), logCode), types.LatestSigner(ethservice.BlockChain().Config()), testKey) - ethservice.TxPool().Add([]*types.Transaction{tx}, true, true) + ethservice.TxPool().Add([]*types.Transaction{tx}, true) execData, err := assembleWithTransactions(api, parent.Hash(), &engine.PayloadAttributes{ Timestamp: parent.Time() + 5, @@ -463,7 +463,7 @@ func TestFullAPI(t *testing.T) { statedb, _ := ethservice.BlockChain().StateAt(parent.Root) nonce := statedb.GetNonce(testAddr) tx, _ := types.SignTx(types.NewContractCreation(nonce, new(big.Int), 1000000, big.NewInt(2*params.InitialBaseFee), logCode), types.LatestSigner(ethservice.BlockChain().Config()), testKey) - ethservice.TxPool().Add([]*types.Transaction{tx}, true, false) + ethservice.TxPool().Add([]*types.Transaction{tx}, false) } setupBlocks(t, ethservice, 10, parent, callback, nil, nil) @@ -594,7 +594,7 @@ func TestNewPayloadOnInvalidChain(t *testing.T) { GasPrice: big.NewInt(2 * params.InitialBaseFee), Data: logCode, }) - ethservice.TxPool().Add([]*types.Transaction{tx}, false, true) + ethservice.TxPool().Add([]*types.Transaction{tx}, true) var ( params = engine.PayloadAttributes{ Timestamp: parent.Time + 1, @@ -1246,8 +1246,8 @@ func setupBodies(t *testing.T) (*node.Node, *eth.Ethereum, []*types.Block) { // Create tx to trigger deposit generator. tx2, _ = types.SignTx(types.NewTransaction(statedb.GetNonce(testAddr)+1, ethservice.APIBackend.ChainConfig().DepositContractAddress, new(big.Int), 500000, big.NewInt(2*params.InitialBaseFee), nil), types.LatestSigner(ethservice.BlockChain().Config()), testKey) ) - ethservice.TxPool().Add([]*types.Transaction{tx1}, false, false) - ethservice.TxPool().Add([]*types.Transaction{tx2}, false, false) + ethservice.TxPool().Add([]*types.Transaction{tx1}, false) + ethservice.TxPool().Add([]*types.Transaction{tx2}, false) } // Make some withdrawals to include. @@ -1637,7 +1637,7 @@ func TestWitnessCreationAndConsumption(t *testing.T) { // Put the 10th block's tx in the pool and produce a new block txs := blocks[9].Transactions() - ethservice.TxPool().Add(txs, true, true) + ethservice.TxPool().Add(txs, true) blockParams := engine.PayloadAttributes{ Timestamp: blocks[8].Time() + 5, Withdrawals: make([]*types.Withdrawal, 0), diff --git a/eth/catalyst/simulated_beacon_test.go b/eth/catalyst/simulated_beacon_test.go index 79d9ba738e..ea35482896 100644 --- a/eth/catalyst/simulated_beacon_test.go +++ b/eth/catalyst/simulated_beacon_test.go @@ -18,6 +18,7 @@ package catalyst import ( "context" + "fmt" "math/big" "testing" "time" @@ -143,9 +144,14 @@ func TestSimulatedBeaconSendWithdrawals(t *testing.T) { // Tests that zero-period dev mode can handle a lot of simultaneous // transactions/withdrawals func TestOnDemandSpam(t *testing.T) { + // This test is flaky, due to various causes, and the root cause is synchronicity. + // We have optimistic timeouts here and there in the simulated becaon and the worker. + // This test typically fails on 32-bit windows appveyor. + t.Skip("flaky test") var ( withdrawals []types.Withdrawal - txs = make(map[common.Hash]*types.Transaction) + txCount = 20000 + wxCount = 20 testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddr = crypto.PubkeyToAddress(testKey.PublicKey) gasLimit uint64 = 10_000_000 @@ -160,7 +166,7 @@ func TestOnDemandSpam(t *testing.T) { defer sub.Unsubscribe() // generate some withdrawals - for i := 0; i < 20; i++ { + for i := 0; i < wxCount; i++ { withdrawals = append(withdrawals, types.Withdrawal{Index: uint64(i)}) if err := mock.withdrawals.add(&withdrawals[i]); err != nil { t.Fatal("addWithdrawal failed", err) @@ -168,37 +174,37 @@ func TestOnDemandSpam(t *testing.T) { } // generate a bunch of transactions - for i := 0; i < 20000; i++ { - tx, err := types.SignTx(types.NewTransaction(uint64(i), common.Address{byte(i), byte(1)}, big.NewInt(1000), params.TxGas, big.NewInt(params.InitialBaseFee*2), nil), signer, testKey) - if err != nil { - t.Fatal("error signing transaction", err) + go func() { + for i := 0; i < txCount; i++ { + tx, err := types.SignTx(types.NewTransaction(uint64(i), common.Address{byte(i), byte(1)}, big.NewInt(1000), params.TxGas, big.NewInt(params.InitialBaseFee*2), nil), signer, testKey) + if err != nil { + panic(fmt.Sprintf("error signing transaction: %v", err)) + } + if err := eth.TxPool().Add([]*types.Transaction{tx}, false)[0]; err != nil { + panic(fmt.Sprintf("error adding txs to pool: %v", err)) + } } - txs[tx.Hash()] = tx - if err := eth.APIBackend.SendTx(context.Background(), tx); err != nil { - t.Fatal("error adding txs to pool", err) - } - } - + }() var ( - includedTxs = make(map[common.Hash]struct{}) - includedWxs []uint64 + includedTxs int + includedWxs int + abort = time.NewTimer(10 * time.Second) ) + defer abort.Stop() for { select { case ev := <-chainHeadCh: block := eth.BlockChain().GetBlock(ev.Header.Hash(), ev.Header.Number.Uint64()) - for _, itx := range block.Transactions() { - includedTxs[itx.Hash()] = struct{}{} - } - for _, iwx := range block.Withdrawals() { - includedWxs = append(includedWxs, iwx.Index) - } + includedTxs += len(block.Transactions()) + includedWxs += len(block.Withdrawals()) // ensure all withdrawals/txs included. this will take two blocks b/c number of withdrawals > 10 - if len(includedTxs) == len(txs) && len(includedWxs) == len(withdrawals) { + if includedTxs == txCount && includedWxs == wxCount { return } - case <-time.After(10 * time.Second): - t.Fatalf("timed out without including all withdrawals/txs: have txs %d, want %d, have wxs %d, want %d", len(includedTxs), len(txs), len(includedWxs), len(withdrawals)) + abort.Reset(10 * time.Second) + case <-abort.C: + t.Fatalf("timed out without including all withdrawals/txs: have txs %d, want %d, have wxs %d, want %d", + includedTxs, txCount, includedWxs, wxCount) } } } diff --git a/eth/handler.go b/eth/handler.go index 8893920497..6ac890902b 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -68,7 +68,7 @@ type txPool interface { Get(hash common.Hash) *types.Transaction // Add should add the given transactions to the pool. - Add(txs []*types.Transaction, local bool, sync bool) []error + Add(txs []*types.Transaction, sync bool) []error // Pending should return pending transactions. // The slice should be modifiable by the caller. @@ -189,7 +189,7 @@ func newHandler(config *handlerConfig) (*handler, error) { return p.RequestTxs(hashes) } addTxs := func(txs []*types.Transaction) []error { - return h.txpool.Add(txs, false, false) + return h.txpool.Add(txs, false) } h.txFetcher = fetcher.NewTxFetcher(h.txpool.Has, addTxs, fetchTx, h.removePeer) return h, nil diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 622880b097..8d572ca966 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -299,8 +299,8 @@ func testSendTransactions(t *testing.T, protocol uint) { tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) insert[nonce] = tx } - go handler.txpool.Add(insert, false, false) // Need goroutine to not block on feed - time.Sleep(250 * time.Millisecond) // Wait until tx events get out of the system (can't use events, tx broadcaster races with peer join) + go handler.txpool.Add(insert, false) // Need goroutine to not block on feed + time.Sleep(250 * time.Millisecond) // Wait until tx events get out of the system (can't use events, tx broadcaster races with peer join) // Create a source handler to send messages through and a sink peer to receive them p2pSrc, p2pSink := p2p.MsgPipe() @@ -419,7 +419,7 @@ func testTransactionPropagation(t *testing.T, protocol uint) { tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) txs[nonce] = tx } - source.txpool.Add(txs, false, false) + source.txpool.Add(txs, false) // Iterate through all the sinks and ensure they all got the transactions for i := range sinks { diff --git a/eth/handler_test.go b/eth/handler_test.go index b63d3e8592..d5d46a3c65 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -80,7 +80,7 @@ func (p *testTxPool) Get(hash common.Hash) *types.Transaction { // Add appends a batch of transactions to the pool, and notifies any // listeners if the addition channel is non nil -func (p *testTxPool) Add(txs []*types.Transaction, local bool, sync bool) []error { +func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error { p.lock.Lock() defer p.lock.Unlock() diff --git a/miner/miner.go b/miner/miner.go index 9892c08ed6..595ef8081c 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -70,6 +70,7 @@ type Miner struct { chainConfig *params.ChainConfig engine consensus.Engine txpool *txpool.TxPool + prio []common.Address // A list of senders to prioritize chain *core.BlockChain pending *pending pendingMu sync.Mutex // Lock protects the pending block @@ -109,6 +110,13 @@ func (miner *Miner) SetExtra(extra []byte) error { return nil } +// SetPrioAddresses sets a list of addresses to prioritize for transaction inclusion. +func (miner *Miner) SetPrioAddresses(prio []common.Address) { + miner.confMu.Lock() + miner.prio = prio + miner.confMu.Unlock() +} + // SetGasCeil sets the gaslimit to strive for when mining blocks post 1559. // For pre-1559 blocks, it sets the ceiling. func (miner *Miner) SetGasCeil(ceil uint64) { diff --git a/miner/payload_building_test.go b/miner/payload_building_test.go index e5eb0297a1..307024c6bb 100644 --- a/miner/payload_building_test.go +++ b/miner/payload_building_test.go @@ -138,7 +138,7 @@ func (b *testWorkerBackend) TxPool() *txpool.TxPool { return b.txPool } func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*Miner, *testWorkerBackend) { backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) - backend.txPool.Add(pendingTxs, true, true) + backend.txPool.Add(pendingTxs, true) w := New(backend, testConfig, engine) return w, backend } diff --git a/miner/worker.go b/miner/worker.go index f8f4bae833..16ac7de9a9 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -427,6 +427,7 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) error { miner.confMu.RLock() tip := miner.config.GasPrice + prio := miner.prio miner.confMu.RUnlock() // Retrieve the pending transactions pre-filtered by the 1559/4844 dynamic fees @@ -446,31 +447,31 @@ func (miner *Miner) fillTransactions(interrupt *atomic.Int32, env *environment) pendingBlobTxs := miner.txpool.Pending(filter) // Split the pending transactions into locals and remotes. - localPlainTxs, remotePlainTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingPlainTxs - localBlobTxs, remoteBlobTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingBlobTxs + prioPlainTxs, normalPlainTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingPlainTxs + prioBlobTxs, normalBlobTxs := make(map[common.Address][]*txpool.LazyTransaction), pendingBlobTxs - for _, account := range miner.txpool.Locals() { - if txs := remotePlainTxs[account]; len(txs) > 0 { - delete(remotePlainTxs, account) - localPlainTxs[account] = txs + for _, account := range prio { + if txs := normalPlainTxs[account]; len(txs) > 0 { + delete(normalPlainTxs, account) + prioPlainTxs[account] = txs } - if txs := remoteBlobTxs[account]; len(txs) > 0 { - delete(remoteBlobTxs, account) - localBlobTxs[account] = txs + if txs := normalBlobTxs[account]; len(txs) > 0 { + delete(normalBlobTxs, account) + prioBlobTxs[account] = txs } } // Fill the block with all available pending transactions. - if len(localPlainTxs) > 0 || len(localBlobTxs) > 0 { - plainTxs := newTransactionsByPriceAndNonce(env.signer, localPlainTxs, env.header.BaseFee) - blobTxs := newTransactionsByPriceAndNonce(env.signer, localBlobTxs, env.header.BaseFee) + if len(prioPlainTxs) > 0 || len(prioBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, prioPlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, prioBlobTxs, env.header.BaseFee) if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err } } - if len(remotePlainTxs) > 0 || len(remoteBlobTxs) > 0 { - plainTxs := newTransactionsByPriceAndNonce(env.signer, remotePlainTxs, env.header.BaseFee) - blobTxs := newTransactionsByPriceAndNonce(env.signer, remoteBlobTxs, env.header.BaseFee) + if len(normalPlainTxs) > 0 || len(normalBlobTxs) > 0 { + plainTxs := newTransactionsByPriceAndNonce(env.signer, normalPlainTxs, env.header.BaseFee) + blobTxs := newTransactionsByPriceAndNonce(env.signer, normalBlobTxs, env.header.BaseFee) if err := miner.commitTransactions(env, plainTxs, blobTxs, interrupt); err != nil { return err From 59d2eec9fc5177953eb7798b8fa5c9da5ebf95b5 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 4 Feb 2025 18:10:40 +0100 Subject: [PATCH 22/24] core/txpool/blobpool: fix incorrect arguments in test (#31127) Fixes the linter on master which was broken by https://github.com/ethereum/go-ethereum/pull/30559 --- core/txpool/blobpool/blobpool_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 0187e8d9a0..88d68eb307 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -1108,7 +1108,7 @@ func TestChangingSlotterSize(t *testing.T) { // Try to add the big blob tx. In the initial iteration it should overflow // the pool. On the subsequent iteration it should be accepted. - errs := pool.Add([]*types.Transaction{tx3}, false, true) + errs := pool.Add([]*types.Transaction{tx3}, true) if _, ok := pool.index[addr3]; ok && maxBlobs == 6 { t.Errorf("expected insert of oversized blob tx to fail: blobs=24, maxBlobs=%d, err=%v", maxBlobs, errs[0]) } else if !ok && maxBlobs == 10 { From c4ad459bd2d85137e26d91144f733bbb938de12d Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 4 Feb 2025 13:43:18 -0700 Subject: [PATCH 23/24] consensus/misc/eip4844: use head's target blobs, not parent (#31101) A clarification was made to EIP-7691 stating that at the fork boundary it is required to use the target blob count associated with the head block, rather than the parent as implemented here. See for more: https://github.com/ethereum/EIPs/pull/9249 --- cmd/evm/internal/t8ntool/execution.go | 2 +- consensus/misc/eip4844/eip4844.go | 6 +++--- consensus/misc/eip4844/eip4844_test.go | 5 ++++- core/chain_makers.go | 2 +- core/state_processor_test.go | 2 +- eth/gasprice/feehistory.go | 2 +- eth/tracers/internal/tracetest/util.go | 5 +++-- internal/ethapi/simulate.go | 2 +- miner/worker.go | 2 +- 9 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 9332f4901b..1613521b0d 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -194,11 +194,11 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, ExcessBlobGas: pre.Env.ParentExcessBlobGas, BlobGasUsed: pre.Env.ParentBlobGasUsed, } - excessBlobGas = eip4844.CalcExcessBlobGas(chainConfig, parent) header := &types.Header{ Time: pre.Env.Timestamp, ExcessBlobGas: &excessBlobGas, } + excessBlobGas = eip4844.CalcExcessBlobGas(chainConfig, parent, header) vmContext.BlobBaseFee = eip4844.CalcBlobFee(chainConfig, header) } } diff --git a/consensus/misc/eip4844/eip4844.go b/consensus/misc/eip4844/eip4844.go index 1d76b21b30..148ea60274 100644 --- a/consensus/misc/eip4844/eip4844.go +++ b/consensus/misc/eip4844/eip4844.go @@ -50,7 +50,7 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", header.BlobGasUsed, params.BlobTxBlobGasPerBlob) } // Verify the excessBlobGas is correct based on the parent header - expectedExcessBlobGas := CalcExcessBlobGas(config, parent) + expectedExcessBlobGas := CalcExcessBlobGas(config, parent, header) if *header.ExcessBlobGas != expectedExcessBlobGas { return fmt.Errorf("invalid excessBlobGas: have %d, want %d", *header.ExcessBlobGas, expectedExcessBlobGas) } @@ -59,9 +59,9 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade // CalcExcessBlobGas calculates the excess blob gas after applying the set of // blobs on top of the excess blob gas. -func CalcExcessBlobGas(config *params.ChainConfig, parent *types.Header) uint64 { +func CalcExcessBlobGas(config *params.ChainConfig, parent, header *types.Header) uint64 { var ( - targetGas = uint64(targetBlobsPerBlock(config, parent.Time)) * params.BlobTxBlobGasPerBlob + targetGas = uint64(targetBlobsPerBlock(config, header.Time)) * params.BlobTxBlobGasPerBlob parentExcessBlobGas uint64 parentBlobGasUsed uint64 ) diff --git a/consensus/misc/eip4844/eip4844_test.go b/consensus/misc/eip4844/eip4844_test.go index 839ea8df22..7d221aa96e 100644 --- a/consensus/misc/eip4844/eip4844_test.go +++ b/consensus/misc/eip4844/eip4844_test.go @@ -57,12 +57,15 @@ func TestCalcExcessBlobGas(t *testing.T) { } for i, tt := range tests { blobGasUsed := uint64(tt.blobs) * params.BlobTxBlobGasPerBlob + head := &types.Header{ + Time: *config.CancunTime, + } parent := &types.Header{ Time: *config.CancunTime, ExcessBlobGas: &tt.excess, BlobGasUsed: &blobGasUsed, } - result := CalcExcessBlobGas(config, parent) + result := CalcExcessBlobGas(config, parent, head) if result != tt.want { t.Errorf("test %d: excess blob gas mismatch: have %v, want %v", i, result, tt.want) } diff --git a/core/chain_makers.go b/core/chain_makers.go index 19f433d7b0..2ade70f175 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -600,7 +600,7 @@ func (cm *chainMaker) makeHeader(parent *types.Block, state *state.StateDB, engi } } if cm.config.IsCancun(header.Number, header.Time) { - excessBlobGas := eip4844.CalcExcessBlobGas(cm.config, parent.Header()) + excessBlobGas := eip4844.CalcExcessBlobGas(cm.config, parent.Header(), header) header.ExcessBlobGas = &excessBlobGas header.BlobGasUsed = new(uint64) header.ParentBeaconRoot = new(common.Hash) diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 7d62e90ad4..b0258ae37b 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -407,7 +407,7 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr } header.Root = common.BytesToHash(hasher.Sum(nil)) if config.IsCancun(header.Number, header.Time) { - excess := eip4844.CalcExcessBlobGas(config, parent.Header()) + excess := eip4844.CalcExcessBlobGas(config, parent.Header(), header) used := uint64(nBlobs * params.BlobTxBlobGasPerBlob) header.ExcessBlobGas = &excess header.BlobGasUsed = &used diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 697263f20b..94fc5b35f9 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -97,7 +97,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { // Fill in blob base fee and next blob base fee. if excessBlobGas := bf.header.ExcessBlobGas; excessBlobGas != nil { bf.results.blobBaseFee = eip4844.CalcBlobFee(config, bf.header) - excess := eip4844.CalcExcessBlobGas(config, bf.header) + excess := eip4844.CalcExcessBlobGas(config, bf.header, bf.header) next := &types.Header{Number: bf.header.Number, Time: bf.header.Time, ExcessBlobGas: &excess} bf.results.nextBlobBaseFee = eip4844.CalcBlobFee(config, next) } else { diff --git a/eth/tracers/internal/tracetest/util.go b/eth/tracers/internal/tracetest/util.go index e29144e04e..97896213b4 100644 --- a/eth/tracers/internal/tracetest/util.go +++ b/eth/tracers/internal/tracetest/util.go @@ -54,8 +54,9 @@ func (c *callContext) toBlockContext(genesis *core.Genesis) vm.BlockContext { } if genesis.ExcessBlobGas != nil && genesis.BlobGasUsed != nil { - excess := eip4844.CalcExcessBlobGas(genesis.Config, genesis.ToBlock().Header()) - header := &types.Header{ExcessBlobGas: &excess, Number: genesis.Config.LondonBlock, Time: *genesis.Config.CancunTime} + header := &types.Header{Number: genesis.Config.LondonBlock, Time: *genesis.Config.CancunTime} + excess := eip4844.CalcExcessBlobGas(genesis.Config, header, genesis.ToBlock().Header()) + header.ExcessBlobGas = &excess context.BlobBaseFee = eip4844.CalcBlobFee(genesis.Config, header) } return context diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index d364b80485..0262f69c02 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -159,7 +159,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, if sim.chainConfig.IsCancun(header.Number, header.Time) { var excess uint64 if sim.chainConfig.IsCancun(parent.Number, parent.Time) { - excess = eip4844.CalcExcessBlobGas(sim.chainConfig, parent) + excess = eip4844.CalcExcessBlobGas(sim.chainConfig, parent, header) } header.ExcessBlobGas = &excess } diff --git a/miner/worker.go b/miner/worker.go index 16ac7de9a9..6ec23ec624 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -210,7 +210,7 @@ func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*envir if miner.chainConfig.IsCancun(header.Number, header.Time) { var excessBlobGas uint64 if miner.chainConfig.IsCancun(parent.Number, parent.Time) { - excessBlobGas = eip4844.CalcExcessBlobGas(miner.chainConfig, parent) + excessBlobGas = eip4844.CalcExcessBlobGas(miner.chainConfig, parent, header) } header.BlobGasUsed = new(uint64) header.ExcessBlobGas = &excessBlobGas From ed1d46b3d31f250e5385a4b8b8f97293acf8adee Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 5 Feb 2025 10:35:03 +0100 Subject: [PATCH 24/24] consensus/misc/eip4844: more changes for blob gas calculation (#31128) This PR changes the signature of `CalcExcessBlobGas` to take in just the header timestamp instead of the whole object. It also adds a sanity check for the parent->child block order to `VerifyEIP4844Header`. --- cmd/evm/internal/t8ntool/execution.go | 2 +- consensus/misc/eip4844/eip4844.go | 9 ++++++--- consensus/misc/eip4844/eip4844_test.go | 8 ++------ core/chain_makers.go | 7 ++++--- core/state_processor_test.go | 2 +- eth/gasprice/feehistory.go | 2 +- eth/tracers/internal/tracetest/util.go | 2 +- internal/ethapi/simulate.go | 2 +- miner/worker.go | 2 +- 9 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 1613521b0d..793b2d425b 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -198,7 +198,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, Time: pre.Env.Timestamp, ExcessBlobGas: &excessBlobGas, } - excessBlobGas = eip4844.CalcExcessBlobGas(chainConfig, parent, header) + excessBlobGas = eip4844.CalcExcessBlobGas(chainConfig, parent, header.Time) vmContext.BlobBaseFee = eip4844.CalcBlobFee(chainConfig, header) } } diff --git a/consensus/misc/eip4844/eip4844.go b/consensus/misc/eip4844/eip4844.go index 148ea60274..4a2754b55c 100644 --- a/consensus/misc/eip4844/eip4844.go +++ b/consensus/misc/eip4844/eip4844.go @@ -34,6 +34,9 @@ var ( // if the current block contains no transactions, the excessBlobGas is updated // accordingly. func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Header) error { + if header.Number.Uint64() != parent.Number.Uint64()+1 { + panic("bad header pair") + } // Verify the header is not malformed if header.ExcessBlobGas == nil { return errors.New("header is missing excessBlobGas") @@ -50,7 +53,7 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", header.BlobGasUsed, params.BlobTxBlobGasPerBlob) } // Verify the excessBlobGas is correct based on the parent header - expectedExcessBlobGas := CalcExcessBlobGas(config, parent, header) + expectedExcessBlobGas := CalcExcessBlobGas(config, parent, header.Time) if *header.ExcessBlobGas != expectedExcessBlobGas { return fmt.Errorf("invalid excessBlobGas: have %d, want %d", *header.ExcessBlobGas, expectedExcessBlobGas) } @@ -59,9 +62,8 @@ func VerifyEIP4844Header(config *params.ChainConfig, parent, header *types.Heade // CalcExcessBlobGas calculates the excess blob gas after applying the set of // blobs on top of the excess blob gas. -func CalcExcessBlobGas(config *params.ChainConfig, parent, header *types.Header) uint64 { +func CalcExcessBlobGas(config *params.ChainConfig, parent *types.Header, headTimestamp uint64) uint64 { var ( - targetGas = uint64(targetBlobsPerBlock(config, header.Time)) * params.BlobTxBlobGasPerBlob parentExcessBlobGas uint64 parentBlobGasUsed uint64 ) @@ -70,6 +72,7 @@ func CalcExcessBlobGas(config *params.ChainConfig, parent, header *types.Header) parentBlobGasUsed = *parent.BlobGasUsed } excessBlobGas := parentExcessBlobGas + parentBlobGasUsed + targetGas := uint64(targetBlobsPerBlock(config, headTimestamp)) * params.BlobTxBlobGasPerBlob if excessBlobGas < targetGas { return 0 } diff --git a/consensus/misc/eip4844/eip4844_test.go b/consensus/misc/eip4844/eip4844_test.go index 7d221aa96e..f4e3cb3d9a 100644 --- a/consensus/misc/eip4844/eip4844_test.go +++ b/consensus/misc/eip4844/eip4844_test.go @@ -57,15 +57,11 @@ func TestCalcExcessBlobGas(t *testing.T) { } for i, tt := range tests { blobGasUsed := uint64(tt.blobs) * params.BlobTxBlobGasPerBlob - head := &types.Header{ - Time: *config.CancunTime, - } - parent := &types.Header{ - Time: *config.CancunTime, + header := &types.Header{ ExcessBlobGas: &tt.excess, BlobGasUsed: &blobGasUsed, } - result := CalcExcessBlobGas(config, parent, head) + result := CalcExcessBlobGas(config, header, *config.CancunTime) if result != tt.want { t.Errorf("test %d: excess blob gas mismatch: have %v, want %v", i, result, tt.want) } diff --git a/core/chain_makers.go b/core/chain_makers.go index 2ade70f175..8d09390b72 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -582,25 +582,26 @@ func GenerateVerkleChainWithGenesis(genesis *Genesis, engine consensus.Engine, n func (cm *chainMaker) makeHeader(parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header { time := parent.Time() + 10 // block time is fixed at 10 seconds + parentHeader := parent.Header() header := &types.Header{ Root: state.IntermediateRoot(cm.config.IsEIP158(parent.Number())), ParentHash: parent.Hash(), Coinbase: parent.Coinbase(), - Difficulty: engine.CalcDifficulty(cm, time, parent.Header()), + Difficulty: engine.CalcDifficulty(cm, time, parentHeader), GasLimit: parent.GasLimit(), Number: new(big.Int).Add(parent.Number(), common.Big1), Time: time, } if cm.config.IsLondon(header.Number) { - header.BaseFee = eip1559.CalcBaseFee(cm.config, parent.Header()) + header.BaseFee = eip1559.CalcBaseFee(cm.config, parentHeader) if !cm.config.IsLondon(parent.Number()) { parentGasLimit := parent.GasLimit() * cm.config.ElasticityMultiplier() header.GasLimit = CalcGasLimit(parentGasLimit, parentGasLimit) } } if cm.config.IsCancun(header.Number, header.Time) { - excessBlobGas := eip4844.CalcExcessBlobGas(cm.config, parent.Header(), header) + excessBlobGas := eip4844.CalcExcessBlobGas(cm.config, parentHeader, time) header.ExcessBlobGas = &excessBlobGas header.BlobGasUsed = new(uint64) header.ParentBeaconRoot = new(common.Hash) diff --git a/core/state_processor_test.go b/core/state_processor_test.go index b0258ae37b..a6ca2781f8 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -407,7 +407,7 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr } header.Root = common.BytesToHash(hasher.Sum(nil)) if config.IsCancun(header.Number, header.Time) { - excess := eip4844.CalcExcessBlobGas(config, parent.Header(), header) + excess := eip4844.CalcExcessBlobGas(config, parent.Header(), header.Time) used := uint64(nBlobs * params.BlobTxBlobGasPerBlob) header.ExcessBlobGas = &excess header.BlobGasUsed = &used diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 94fc5b35f9..fe84950c50 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -97,7 +97,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { // Fill in blob base fee and next blob base fee. if excessBlobGas := bf.header.ExcessBlobGas; excessBlobGas != nil { bf.results.blobBaseFee = eip4844.CalcBlobFee(config, bf.header) - excess := eip4844.CalcExcessBlobGas(config, bf.header, bf.header) + excess := eip4844.CalcExcessBlobGas(config, bf.header, bf.header.Time) next := &types.Header{Number: bf.header.Number, Time: bf.header.Time, ExcessBlobGas: &excess} bf.results.nextBlobBaseFee = eip4844.CalcBlobFee(config, next) } else { diff --git a/eth/tracers/internal/tracetest/util.go b/eth/tracers/internal/tracetest/util.go index 97896213b4..524a396d5c 100644 --- a/eth/tracers/internal/tracetest/util.go +++ b/eth/tracers/internal/tracetest/util.go @@ -55,7 +55,7 @@ func (c *callContext) toBlockContext(genesis *core.Genesis) vm.BlockContext { if genesis.ExcessBlobGas != nil && genesis.BlobGasUsed != nil { header := &types.Header{Number: genesis.Config.LondonBlock, Time: *genesis.Config.CancunTime} - excess := eip4844.CalcExcessBlobGas(genesis.Config, header, genesis.ToBlock().Header()) + excess := eip4844.CalcExcessBlobGas(genesis.Config, header, genesis.Timestamp) header.ExcessBlobGas = &excess context.BlobBaseFee = eip4844.CalcBlobFee(genesis.Config, header) } diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 0262f69c02..c461b1f0a1 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -159,7 +159,7 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, if sim.chainConfig.IsCancun(header.Number, header.Time) { var excess uint64 if sim.chainConfig.IsCancun(parent.Number, parent.Time) { - excess = eip4844.CalcExcessBlobGas(sim.chainConfig, parent, header) + excess = eip4844.CalcExcessBlobGas(sim.chainConfig, parent, header.Time) } header.ExcessBlobGas = &excess } diff --git a/miner/worker.go b/miner/worker.go index 6ec23ec624..2c54421196 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -210,7 +210,7 @@ func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*envir if miner.chainConfig.IsCancun(header.Number, header.Time) { var excessBlobGas uint64 if miner.chainConfig.IsCancun(parent.Number, parent.Time) { - excessBlobGas = eip4844.CalcExcessBlobGas(miner.chainConfig, parent, header) + excessBlobGas = eip4844.CalcExcessBlobGas(miner.chainConfig, parent, timestamp) } header.BlobGasUsed = new(uint64) header.ExcessBlobGas = &excessBlobGas