diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b569180852..681b5e7c66 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,12 +9,11 @@ beacon/light/ @zsfelfoldi beacon/merkle/ @zsfelfoldi beacon/types/ @zsfelfoldi @fjl beacon/params/ @zsfelfoldi @fjl -cmd/clef/ @holiman -cmd/evm/ @holiman @MariusVanDerWijden @lightclient -core/state/ @rjl493456442 @holiman -crypto/ @gballet @jwasinger @holiman @fjl -core/ @holiman @rjl493456442 -eth/ @holiman @rjl493456442 +cmd/evm/ @MariusVanDerWijden @lightclient +core/state/ @rjl493456442 +crypto/ @gballet @jwasinger @fjl +core/ @rjl493456442 +eth/ @rjl493456442 eth/catalyst/ @MariusVanDerWijden @lightclient @fjl @jwasinger eth/tracers/ @s1na ethclient/ @fjl @@ -26,11 +25,9 @@ core/tracing/ @s1na graphql/ @s1na internal/ethapi/ @fjl @s1na @lightclient internal/era/ @lightclient -metrics/ @holiman -miner/ @MariusVanDerWijden @holiman @fjl @rjl493456442 +miner/ @MariusVanDerWijden @fjl @rjl493456442 node/ @fjl p2p/ @fjl @zsfelfoldi rlp/ @fjl -params/ @fjl @holiman @karalabe @gballet @rjl493456442 @zsfelfoldi -rpc/ @fjl @holiman -signer/ @holiman +params/ @fjl @karalabe @gballet @rjl493456442 @zsfelfoldi +rpc/ @fjl diff --git a/accounts/abi/abigen/bind_test.go b/accounts/abi/abigen/bind_test.go index 195064fb7a..b3c52e81e5 100644 --- a/accounts/abi/abigen/bind_test.go +++ b/accounts/abi/abigen/bind_test.go @@ -939,6 +939,7 @@ var bindTests = []struct { if _, err := eventer.RaiseSimpleEvent(auth, common.Address{byte(j)}, [32]byte{byte(j)}, true, big.NewInt(int64(10*i+j))); err != nil { t.Fatalf("block %d, event %d: raise failed: %v", i, j, err) } + time.Sleep(time.Millisecond * 200) } sim.Commit() } @@ -1495,7 +1496,7 @@ var bindTests = []struct { if n != 3 { t.Fatalf("Invalid bar0 event") } - case <-time.NewTimer(3 * time.Second).C: + case <-time.NewTimer(10 * time.Second).C: t.Fatalf("Wait bar0 event timeout") } @@ -1506,7 +1507,7 @@ var bindTests = []struct { if n != 1 { t.Fatalf("Invalid bar event") } - case <-time.NewTimer(3 * time.Second).C: + case <-time.NewTimer(10 * time.Second).C: t.Fatalf("Wait bar event timeout") } close(stopCh) diff --git a/beacon/params/checkpoint_holesky.hex b/beacon/params/checkpoint_holesky.hex index f1e27b48f6..740d7aba21 100644 --- a/beacon/params/checkpoint_holesky.hex +++ b/beacon/params/checkpoint_holesky.hex @@ -1 +1 @@ -0xf5606a019f0f1006e9ec2070695045f4334450362a48da4c5965314510e0b4c3 \ No newline at end of file +0xd60e5310c5d52ced44cfb13be4e9f22a1e6a6dc56964c3cccd429182d26d72d0 \ No newline at end of file diff --git a/beacon/params/checkpoint_mainnet.hex b/beacon/params/checkpoint_mainnet.hex index a11c3394bb..45f065ca15 100644 --- a/beacon/params/checkpoint_mainnet.hex +++ b/beacon/params/checkpoint_mainnet.hex @@ -1 +1 @@ -0x3c0cb4aa83beded1803d262664ba4392b1023f334d9cca02dc3a6925987ebe91 \ No newline at end of file +0x02f0bb348b0d45f95a9b7e2bb5705768ad06548876cee03d880a2c9dabb1ff88 \ No newline at end of file diff --git a/beacon/params/checkpoint_sepolia.hex b/beacon/params/checkpoint_sepolia.hex index e4f2f96733..3d1b2885b3 100644 --- a/beacon/params/checkpoint_sepolia.hex +++ b/beacon/params/checkpoint_sepolia.hex @@ -1 +1 @@ -0xa8d56457aa414523d93659aef1f7409bbfb72ad75e94d917c8c0b1baa38153ef \ No newline at end of file +0xa0dad451a230c01be6f2492980ec5bb412d8cf33351a75e8b172b5b84a5fd03a \ No newline at end of file diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 7de1eb6949..b2e5f70714 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -363,9 +363,13 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not parse requests logs: %v", err)) } // EIP-7002 - core.ProcessWithdrawalQueue(&requests, evm) + if err := core.ProcessWithdrawalQueue(&requests, evm); err != nil { + return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not process withdrawal requests: %v", err)) + } // EIP-7251 - core.ProcessConsolidationQueue(&requests, evm) + if err := core.ProcessConsolidationQueue(&requests, evm); err != nil { + return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not process consolidation requests: %v", err)) + } } // Commit block diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 2279509542..c57a9a947d 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -31,11 +31,11 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/era" @@ -625,7 +625,7 @@ func pruneHistory(ctx *cli.Context) error { defer chain.Stop() // Determine the prune point. This will be the first PoS block. - prunePoint, ok := ethconfig.HistoryPrunePoints[chain.Genesis().Hash()] + prunePoint, ok := history.PrunePoints[chain.Genesis().Hash()] if !ok || prunePoint == nil { return errors.New("prune point not found") } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index ab46e059f3..289030ae65 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -100,7 +100,6 @@ var ( utils.LightNoSyncServeFlag, // deprecated utils.EthRequiredBlocksFlag, utils.LegacyWhitelistFlag, // deprecated - utils.BloomFilterSizeFlag, utils.CacheFlag, utils.CacheDatabaseFlag, utils.CacheTrieFlag, diff --git a/cmd/workload/testsuite.go b/cmd/workload/testsuite.go index e7019e2055..e8e25e7731 100644 --- a/cmd/workload/testsuite.go +++ b/cmd/workload/testsuite.go @@ -23,7 +23,7 @@ import ( "os" "slices" - "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/log" @@ -124,13 +124,13 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { cfg.filterQueryFile = "queries/filter_queries_mainnet.json" cfg.historyTestFile = "queries/history_mainnet.json" cfg.historyPruneBlock = new(uint64) - *cfg.historyPruneBlock = ethconfig.HistoryPrunePoints[params.MainnetGenesisHash].BlockNumber + *cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber case ctx.Bool(testSepoliaFlag.Name): cfg.fsys = builtinTestFiles cfg.filterQueryFile = "queries/filter_queries_sepolia.json" cfg.historyTestFile = "queries/history_sepolia.json" cfg.historyPruneBlock = new(uint64) - *cfg.historyPruneBlock = ethconfig.HistoryPrunePoints[params.SepoliaGenesisHash].BlockNumber + *cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber default: cfg.fsys = os.DirFS(".") cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name) diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 8af4057693..5217979236 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -50,8 +50,11 @@ func testHeaderVerification(t *testing.T, scheme string) { headers[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) defer chain.Stop() + if err != nil { + t.Fatal(err) + } for i := 0; i < len(blocks); i++ { for j, valid := range []bool{true, false} { @@ -163,8 +166,11 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { postHeaders[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil) defer chain.Stop() + if err != nil { + t.Fatal(err) + } // Verify the blocks before the merging for i := 0; i < len(preBlocks); i++ { diff --git a/core/blockchain.go b/core/blockchain.go index d56996dadb..6667f64911 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "math" "math/big" "runtime" "slices" @@ -36,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -99,6 +101,10 @@ var ( errInvalidNewChain = errors.New("invalid new chain") ) +var ( + forkReadyInterval = 3 * time.Minute +) + const ( bodyCacheLimit = 256 blockCacheLimit = 256 @@ -158,8 +164,7 @@ type CacheConfig struct { // This defines the cutoff block for history expiry. // Blocks before this number may be unavailable in the chain database. - HistoryPruningCutoffNumber uint64 - HistoryPruningCutoffHash common.Hash + ChainHistoryMode history.HistoryMode } // triedbConfig derives the configures for trie database. @@ -255,6 +260,7 @@ type BlockChain struct { currentSnapBlock atomic.Pointer[types.Header] // Current head of snap-sync currentFinalBlock atomic.Pointer[types.Header] // Latest (consensus) finalized block currentSafeBlock atomic.Pointer[types.Header] // Latest (consensus) safe block + historyPrunePoint atomic.Pointer[history.PrunePoint] bodyCache *lru.Cache[common.Hash, *types.Body] bodyRLPCache *lru.Cache[common.Hash, rlp.RawValue] @@ -274,6 +280,8 @@ type BlockChain struct { processor Processor // Block transaction processor interface vmConfig vm.Config logger *tracing.Hooks + + lastForkReadyAlert time.Time // Last time there was a fork readiness print out } // NewBlockChain returns a fully initialised block chain using information @@ -513,19 +521,33 @@ func (bc *BlockChain) loadLastState() error { log.Warn("Empty database, resetting chain") return bc.Reset() } - // Make sure the entire head block is available - headBlock := bc.GetBlockByHash(head) + headHeader := bc.GetHeaderByHash(head) + if headHeader == nil { + // Corrupt or empty database, init from scratch + log.Warn("Head header missing, resetting chain", "hash", head) + return bc.Reset() + } + + var headBlock *types.Block + if cmp := headHeader.Number.Cmp(new(big.Int)); cmp == 1 { + // Make sure the entire head block is available. + headBlock = bc.GetBlockByHash(head) + } else if cmp == 0 { + // On a pruned node the block body might not be available. But a pruned + // block should never be the head block. The only exception is when, as + // a last resort, chain is reset to genesis. + headBlock = bc.genesisBlock + } if headBlock == nil { // Corrupt or empty database, init from scratch log.Warn("Head block missing, resetting chain", "hash", head) return bc.Reset() } // Everything seems to be fine, set as the head block - bc.currentBlock.Store(headBlock.Header()) + bc.currentBlock.Store(headHeader) headBlockGauge.Update(int64(headBlock.NumberU64())) // Restore the last known head header - headHeader := headBlock.Header() if head := rawdb.ReadHeadHeaderHash(bc.db); head != (common.Hash{}) { if header := bc.GetHeaderByHash(head); header != nil { headHeader = header @@ -533,6 +555,12 @@ func (bc *BlockChain) loadLastState() error { } bc.hc.SetCurrentHeader(headHeader) + // Initialize history pruning. + latest := max(headBlock.NumberU64(), headHeader.Number.Uint64()) + if err := bc.initializeHistoryPruning(latest); err != nil { + return err + } + // Restore the last known head snap block bc.currentSnapBlock.Store(headBlock.Header()) headFastBlockGauge.Update(int64(headBlock.NumberU64())) @@ -555,6 +583,7 @@ func (bc *BlockChain) loadLastState() error { headSafeBlockGauge.Update(int64(block.NumberU64())) } } + // Issue a status log for the user var ( currentSnapBlock = bc.CurrentSnapBlock() @@ -573,9 +602,57 @@ func (bc *BlockChain) loadLastState() error { if pivot := rawdb.ReadLastPivotNumber(bc.db); pivot != nil { log.Info("Loaded last snap-sync pivot marker", "number", *pivot) } + if pruning := bc.historyPrunePoint.Load(); pruning != nil { + log.Info("Chain history is pruned", "earliest", pruning.BlockNumber, "hash", pruning.BlockHash) + } return nil } +// initializeHistoryPruning sets bc.historyPrunePoint. +func (bc *BlockChain) initializeHistoryPruning(latest uint64) error { + freezerTail, _ := bc.db.Tail() + + switch bc.cacheConfig.ChainHistoryMode { + case history.KeepAll: + if freezerTail == 0 { + return nil + } + // The database was pruned somehow, so we need to figure out if it's a known + // configuration or an error. + predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] + if predefinedPoint == nil || freezerTail != predefinedPoint.BlockNumber { + log.Error("Chain history database is pruned with unknown configuration", "tail", freezerTail) + return fmt.Errorf("unexpected database tail") + } + bc.historyPrunePoint.Store(predefinedPoint) + return nil + + case history.KeepPostMerge: + if freezerTail == 0 && latest != 0 { + // This is the case where a user is trying to run with --history.chain + // postmerge directly on an existing DB. We could just trigger the pruning + // here, but it'd be a bit dangerous since they may not have intended this + // action to happen. So just tell them how to do it. + log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cacheConfig.ChainHistoryMode.String())) + log.Error(fmt.Sprintf("Run 'geth prune-history' to prune pre-merge history.")) + return fmt.Errorf("history pruning requested via configuration") + } + predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] + if predefinedPoint == nil { + log.Error("Chain history pruning is not supported for this network", "genesis", bc.genesisBlock.Hash()) + return fmt.Errorf("history pruning requested for unknown network") + } else if freezerTail > 0 && freezerTail != predefinedPoint.BlockNumber { + log.Error("Chain history database is pruned to unknown block", "tail", freezerTail) + return fmt.Errorf("unexpected database tail") + } + bc.historyPrunePoint.Store(predefinedPoint) + return nil + + default: + return fmt.Errorf("invalid history mode: %d", bc.cacheConfig.ChainHistoryMode) + } +} + // SetHead rewinds the local chain to a new head. Depending on whether the node // was snap synced or full synced and in which state, the method will try to // delete minimal data from disk whilst retaining chain consistency. @@ -586,11 +663,15 @@ func (bc *BlockChain) SetHead(head uint64) error { // Send chain head event to update the transaction pool header := bc.CurrentBlock() if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil { - // This should never happen. In practice, previously currentBlock - // contained the entire block whereas now only a "marker", so there - // is an ever so slight chance for a race we should handle. - log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) - return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + // In a pruned node the genesis block will not exist in the freezer. + // It should not happen that we set head to any other pruned block. + if header.Number.Uint64() > 0 { + // This should never happen. In practice, previously currentBlock + // contained the entire block whereas now only a "marker", so there + // is an ever so slight chance for a race we should handle. + log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) + return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + } } bc.chainHeadFeed.Send(ChainHeadEvent{Header: header}) return nil @@ -607,11 +688,15 @@ func (bc *BlockChain) SetHeadWithTimestamp(timestamp uint64) error { // Send chain head event to update the transaction pool header := bc.CurrentBlock() if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil { - // This should never happen. In practice, previously currentBlock - // contained the entire block whereas now only a "marker", so there - // is an ever so slight chance for a race we should handle. - log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) - return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + // In a pruned node the genesis block will not exist in the freezer. + // It should not happen that we set head to any other pruned block. + if header.Number.Uint64() > 0 { + // This should never happen. In practice, previously currentBlock + // contained the entire block whereas now only a "marker", so there + // is an ever so slight chance for a race we should handle. + log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) + return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + } } bc.chainHeadFeed.Send(ChainHeadEvent{Header: header}) return nil @@ -1014,7 +1099,9 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { bc.hc.SetCurrentHeader(bc.genesisBlock.Header()) bc.currentSnapBlock.Store(bc.genesisBlock.Header()) headFastBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) - return nil + + // Reset history pruning status. + return bc.initializeHistoryPruning(0) } // Export writes the active chain to the given writer. @@ -1804,6 +1891,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness trieDiffNodes, trieBufNodes, _ := bc.triedb.Size() stats.report(chain, it.index, snapDiffItems, snapBufItems, trieDiffNodes, trieBufNodes, setHead) + // Print confirmation that a future fork is scheduled, but not yet active. + bc.logForkReadiness(block) + if !setHead { // After merge we expect few side chains. Simply count // all blocks the CL gives us for GC processing time @@ -1837,6 +1927,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness "root", block.Root()) } } + stats.ignored += it.remaining() return witness, it.index, err } @@ -2434,6 +2525,23 @@ func (bc *BlockChain) reportBlock(block *types.Block, res *ProcessResult, err er log.Error(summarizeBadBlock(block, receipts, bc.Config(), err)) } +// logForkReadiness will write a log when a future fork is scheduled, but not +// active. This is useful so operators know their client is ready for the fork. +func (bc *BlockChain) logForkReadiness(block *types.Block) { + c := bc.Config() + current, last := c.LatestFork(block.Time()), c.LatestFork(math.MaxUint64) + t := c.Timestamp(last) + if t == nil { + return + } + at := time.Unix(int64(*t), 0) + if current < last && time.Now().After(bc.lastForkReadyAlert.Add(forkReadyInterval)) { + log.Info("Ready for fork activation", "fork", last, "date", at.Format(time.RFC822), + "remaining", time.Until(at).Round(time.Second), "timestamp", at.Unix()) + bc.lastForkReadyAlert = time.Now() + } +} + // summarizeBadBlock returns a string summarizing the bad block and other // relevant information. func summarizeBadBlock(block *types.Block, receipts []*types.Receipt, config *params.ChainConfig, err error) string { diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 4114723469..a8c2e26d18 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -410,7 +410,11 @@ func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) { // HistoryPruningCutoff returns the configured history pruning point. // Blocks before this might not be available in the database. func (bc *BlockChain) HistoryPruningCutoff() (uint64, common.Hash) { - return bc.cacheConfig.HistoryPruningCutoffNumber, bc.cacheConfig.HistoryPruningCutoffHash + pt := bc.historyPrunePoint.Load() + if pt == nil { + return 0, bc.genesisBlock.Hash() + } + return pt.BlockNumber, pt.BlockHash } // TrieDB retrieves the low level trie database used for data storage. diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 3f7c03b93c..289eff0a8f 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -4257,13 +4258,7 @@ func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) { // be persisted without the receipts and bodies; chain after should be persisted // normally. func TestInsertChainWithCutoff(t *testing.T) { - testInsertChainWithCutoff(t, 32, 32) // cutoff = 32, ancientLimit = 32 - testInsertChainWithCutoff(t, 32, 64) // cutoff = 32, ancientLimit = 64 (entire chain in ancient) - testInsertChainWithCutoff(t, 32, 65) // cutoff = 32, ancientLimit = 65 (64 blocks in ancient, 1 block in live) -} - -func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64) { - // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) + const chainLength = 64 // Configure and generate a sample block chain var ( @@ -4278,24 +4273,51 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64) signer = types.LatestSigner(gspec.Config) engine = beacon.New(ethash.NewFaker()) ) - _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, int(2*cutoff), func(i int, block *BlockGen) { + _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, chainLength, func(i int, block *BlockGen) { block.SetCoinbase(common.Address{0x00}) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) if err != nil { panic(err) } block.AddTx(tx) }) + + // Run the actual tests. + t.Run("cutoff-32/ancientLimit-32", func(t *testing.T) { + // cutoff = 32, ancientLimit = 32 + testInsertChainWithCutoff(t, 32, 32, gspec, blocks, receipts) + }) + t.Run("cutoff-32/ancientLimit-64", func(t *testing.T) { + // cutoff = 32, ancientLimit = 64 (entire chain in ancient) + testInsertChainWithCutoff(t, 32, 64, gspec, blocks, receipts) + }) + t.Run("cutoff-32/ancientLimit-64", func(t *testing.T) { + // cutoff = 32, ancientLimit = 65 (64 blocks in ancient, 1 block in live) + testInsertChainWithCutoff(t, 32, 65, gspec, blocks, receipts) + }) +} + +func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64, genesis *Genesis, blocks []*types.Block, receipts []types.Receipts) { + // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) + + // Add a known pruning point for the duration of the test. + ghash := genesis.ToBlock().Hash() + cutoffBlock := blocks[cutoff-1] + history.PrunePoints[ghash] = &history.PrunePoint{ + BlockNumber: cutoffBlock.NumberU64(), + BlockHash: cutoffBlock.Hash(), + } + defer func() { + delete(history.PrunePoints, ghash) + }() + + // Enable pruning in cache config. + config := DefaultCacheConfigWithScheme(rawdb.PathScheme) + config.ChainHistoryMode = history.KeepPostMerge + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false) defer db.Close() - - cutoffBlock := blocks[cutoff-1] - config := DefaultCacheConfigWithScheme(rawdb.PathScheme) - config.HistoryPruningCutoffNumber = cutoffBlock.NumberU64() - config.HistoryPruningCutoffHash = cutoffBlock.Hash() - - chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil) + chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), genesis, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil) defer chain.Stop() var ( @@ -4326,8 +4348,8 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64) t.Errorf("head header #%d: header mismatch: want: %v, got: %v", headHeader.Number, blocks[len(blocks)-1].Hash(), headHeader.Hash()) } headBlock := chain.CurrentBlock() - if headBlock.Hash() != gspec.ToBlock().Hash() { - t.Errorf("head block #%d: header mismatch: want: %v, got: %v", headBlock.Number, gspec.ToBlock().Hash(), headBlock.Hash()) + if headBlock.Hash() != ghash { + t.Errorf("head block #%d: header mismatch: want: %v, got: %v", headBlock.Number, ghash, headBlock.Hash()) } // Iterate over all chain data components, and cross reference diff --git a/core/chain_makers.go b/core/chain_makers.go index 7a258dc05f..37bddcfda5 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -328,9 +328,13 @@ func (b *BlockGen) collectRequests(readonly bool) (requests [][]byte) { blockContext := NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase) evm := vm.NewEVM(blockContext, statedb, b.cm.config, vm.Config{}) // EIP-7002 - ProcessWithdrawalQueue(&requests, evm) + if err := ProcessWithdrawalQueue(&requests, evm); err != nil { + panic(fmt.Sprintf("could not process withdrawal requests: %v", err)) + } // EIP-7251 - ProcessConsolidationQueue(&requests, evm) + if err := ProcessConsolidationQueue(&requests, evm); err != nil { + panic(fmt.Sprintf("could not process consolidation requests: %v", err)) + } } return requests } diff --git a/core/filtermaps/chain_view.go b/core/filtermaps/chain_view.go index a8cf53b1c0..aa74f3901a 100644 --- a/core/filtermaps/chain_view.go +++ b/core/filtermaps/chain_view.go @@ -17,6 +17,8 @@ package filtermaps import ( + "sync" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -39,6 +41,7 @@ type blockchain interface { // of the underlying blockchain, it should only possess the block headers // and receipts up until the expected chain view head. type ChainView struct { + lock sync.Mutex chain blockchain headNumber uint64 hashes []common.Hash // block hashes starting backwards from headNumber until first canonical hash @@ -55,47 +58,75 @@ func NewChainView(chain blockchain, number uint64, hash common.Hash) *ChainView return cv } -// getBlockHash returns the block hash belonging to the given block number. +// HeadNumber returns the head block number of the chain view. +func (cv *ChainView) HeadNumber() uint64 { + return cv.headNumber +} + +// BlockHash returns the block hash belonging to the given block number. // Note that the hash of the head block is not returned because ChainView might // represent a view where the head block is currently being created. -func (cv *ChainView) getBlockHash(number uint64) common.Hash { - if number >= cv.headNumber { +func (cv *ChainView) BlockHash(number uint64) common.Hash { + cv.lock.Lock() + defer cv.lock.Unlock() + + if number > cv.headNumber { panic("invalid block number") } return cv.blockHash(number) } -// getBlockId returns the unique block id belonging to the given block number. +// BlockId returns the unique block id belonging to the given block number. // Note that it is currently equal to the block hash. In the future it might // be a different id for future blocks if the log index root becomes part of // consensus and therefore rendering the index with the new head will happen // before the hash of that new head is available. -func (cv *ChainView) getBlockId(number uint64) common.Hash { +func (cv *ChainView) BlockId(number uint64) common.Hash { + cv.lock.Lock() + defer cv.lock.Unlock() + if number > cv.headNumber { panic("invalid block number") } return cv.blockHash(number) } -// getReceipts returns the set of receipts belonging to the block at the given +// Header returns the block header at the given block number. +func (cv *ChainView) Header(number uint64) *types.Header { + return cv.chain.GetHeader(cv.BlockHash(number), number) +} + +// Receipts returns the set of receipts belonging to the block at the given // block number. -func (cv *ChainView) getReceipts(number uint64) types.Receipts { - if number > cv.headNumber { - panic("invalid block number") - } - blockHash := cv.blockHash(number) +func (cv *ChainView) Receipts(number uint64) types.Receipts { + blockHash := cv.BlockHash(number) if blockHash == (common.Hash{}) { log.Error("Chain view: block hash unavailable", "number", number, "head", cv.headNumber) } return cv.chain.GetReceiptsByHash(blockHash) } +// SharedRange returns the block range shared by two chain views. +func (cv *ChainView) SharedRange(cv2 *ChainView) common.Range[uint64] { + cv.lock.Lock() + defer cv.lock.Unlock() + + if cv == nil || cv2 == nil || !cv.extendNonCanonical() || !cv2.extendNonCanonical() { + return common.Range[uint64]{} + } + var sharedLen uint64 + for n := min(cv.headNumber+1-uint64(len(cv.hashes)), cv2.headNumber+1-uint64(len(cv2.hashes))); n <= cv.headNumber && n <= cv2.headNumber && cv.blockHash(n) == cv2.blockHash(n); n++ { + sharedLen = n + 1 + } + return common.NewRange(0, sharedLen) +} + // limitedView returns a new chain view that is a truncated version of the parent view. func (cv *ChainView) limitedView(newHead uint64) *ChainView { if newHead >= cv.headNumber { return cv } - return NewChainView(cv.chain, newHead, cv.blockHash(newHead)) + return NewChainView(cv.chain, newHead, cv.BlockHash(newHead)) } // equalViews returns true if the two chain views are equivalent. @@ -103,7 +134,7 @@ func equalViews(cv1, cv2 *ChainView) bool { if cv1 == nil || cv2 == nil { return false } - return cv1.headNumber == cv2.headNumber && cv1.getBlockId(cv1.headNumber) == cv2.getBlockId(cv2.headNumber) + return cv1.headNumber == cv2.headNumber && cv1.BlockId(cv1.headNumber) == cv2.BlockId(cv2.headNumber) } // matchViews returns true if the two chain views are equivalent up until the @@ -117,9 +148,9 @@ func matchViews(cv1, cv2 *ChainView, number uint64) bool { return false } if number == cv1.headNumber || number == cv2.headNumber { - return cv1.getBlockId(number) == cv2.getBlockId(number) + return cv1.BlockId(number) == cv2.BlockId(number) } - return cv1.getBlockHash(number) == cv2.getBlockHash(number) + return cv1.BlockHash(number) == cv2.BlockHash(number) } // extendNonCanonical checks whether the previously known reverse list of head diff --git a/core/filtermaps/checkpoints_holesky.json b/core/filtermaps/checkpoints_holesky.json index b8f6be9caf..a56611cc8e 100644 --- a/core/filtermaps/checkpoints_holesky.json +++ b/core/filtermaps/checkpoints_holesky.json @@ -17,5 +17,6 @@ {"blockNumber": 3101669, "blockId": "0xa6e66a18fed64d33178c7088f273c404be597662c34f6e6884cf2e24f3ea4ace", "firstIndex": 1073741353}, {"blockNumber": 3221725, "blockId": "0xe771f897dece48b1583cc1d1d10de8015da57407eb1fdf239fdbe46eaab85143", "firstIndex": 1140850137}, {"blockNumber": 3357164, "blockId": "0x6252d0aa54c79623b0680069c88d7b5c47983f0d5c4845b6c811b8d9b5e8ff3c", "firstIndex": 1207959453}, -{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542} +{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542}, +{"blockNumber": 3546397, "blockId": "0xdabdef7defa4281180a57c5af121877b82274f15ccf074ea0096146f4c246df2", "firstIndex": 1342176778} ] diff --git a/core/filtermaps/checkpoints_mainnet.json b/core/filtermaps/checkpoints_mainnet.json index ca30280e03..70a08b1aaf 100644 --- a/core/filtermaps/checkpoints_mainnet.json +++ b/core/filtermaps/checkpoints_mainnet.json @@ -260,5 +260,12 @@ {"blockNumber": 21959188, "blockId": "0x6b9c482cc4822af2ad62a359b923062556ab6a046d1a39a8243b764848690601", "firstIndex": 17381193886}, {"blockNumber": 21994911, "blockId": "0x0d99ef9dd42dd1c62fc5249eb4f618e7672ab93fa0ba7545c77360371cf972e5", "firstIndex": 17448302795}, {"blockNumber": 22026007, "blockId": "0xf7cdc7f6694f2c2ed31fa8a607f65cfa59d0dd7d7424ab5c017f959ae2982c71", "firstIndex": 17515413036}, -{"blockNumber": 22059890, "blockId": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768} +{"blockNumber": 22059890, "blockId": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768}, +{"blockNumber": 22090784, "blockId": "0xf97c2eaf9a550360ac24000c0ff17ffa388a2bdd6f73f2f36718e332edfa107a", "firstIndex": 17649630983}, +{"blockNumber": 22121157, "blockId": "0xa790025235db782e899f23d8b09663ec2d74ec149e4125d62989f98829b08e2d", "firstIndex": 17716724973}, +{"blockNumber": 22148056, "blockId": "0xbe25ac4f1bdd89a7db5782bf1157e6c4378d80220d67a029397853ef16cd1a4c", "firstIndex": 17783838366}, +{"blockNumber": 22168652, "blockId": "0x6ae43618c915e636794e2cc2d75dde9992766881c7405fe6479c045ed4bee57e", "firstIndex": 17850956277}, +{"blockNumber": 22190975, "blockId": "0x9437121647899a4b7b84d67fbea7cc6ff967481c2eab4328ccd86e2cefe19420", "firstIndex": 17918066140}, +{"blockNumber": 22234357, "blockId": "0x036030830134f9224160d5a0b62da35ec7813dc8855d554bd22e9d38545243ed", "firstIndex": 17985175075}, +{"blockNumber": 22276736, "blockId": "0x5ceb96d98aa1b4c1c2f2fa253ae9cdb1b04e0420c11bf795400e8762c0a1635c", "firstIndex": 18052284344} ] diff --git a/core/filtermaps/checkpoints_sepolia.json b/core/filtermaps/checkpoints_sepolia.json index c6d3cd9086..8d799daefd 100644 --- a/core/filtermaps/checkpoints_sepolia.json +++ b/core/filtermaps/checkpoints_sepolia.json @@ -64,5 +64,9 @@ {"blockNumber": 7730079, "blockId": "0x08e0d511a93e2a9c004b3456d77d687ae86247417ada1cdd1f85332a62dfed1d", "firstIndex": 4227846795}, {"blockNumber": 7777515, "blockId": "0xeba8faed2e12bb9ed5e7fad19dd82662f15f6f4f512a0d232eedf1e676712428", "firstIndex": 4294966082}, {"blockNumber": 7828860, "blockId": "0x62c054bd24be351dc4777460ee922a75fdcea5c9830455cbac6fa29552719c85", "firstIndex": 4362076087}, -{"blockNumber": 7869890, "blockId": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267} +{"blockNumber": 7869890, "blockId": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267}, +{"blockNumber": 7911722, "blockId": "0x9a85e48e3135c97c51fc1786f2af0596c802e021b6c53cfca65a129cafcd23ed", "firstIndex": 4496287265}, +{"blockNumber": 7960147, "blockId": "0xc9359cc76d7090e1c8a031108f0ab7a8935d971efd4325fe53612a1d99562f6f", "firstIndex": 4563402388}, +{"blockNumber": 8030418, "blockId": "0x21867e68cd8327aed2da2601399d60f7f9e41dca4a4f2f9be982e5a2b9304a88", "firstIndex": 4630511616}, +{"blockNumber": 8087701, "blockId": "0x0fa8c8d7549cc9a8d308262706fe248efe759f8b63511efb1e7f3926e9af2dcb", "firstIndex": 4697614758} ] diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index 18b1c7dc79..920167ca8d 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -50,6 +50,7 @@ var ( ) const ( + databaseVersion = 1 // reindexed if database version does not match cachedLastBlocks = 1000 // last block of map pointers cachedLvPointers = 1000 // first log value pointer of block pointers cachedBaseRows = 100 // groups of base layer filter row data @@ -127,6 +128,7 @@ type FilterMaps struct { // test hooks testDisableSnapshots, testSnapshotUsed bool + testProcessEventsHook func() } // filterMap is a full or partial in-memory representation of a filter map where @@ -138,13 +140,25 @@ type FilterMaps struct { // as transparent (uncached/unchanged). type filterMap []FilterRow -// copy returns a copy of the given filter map. Note that the row slices are -// copied but their contents are not. This permits extending the rows further +// fastCopy returns a copy of the given filter map. Note that the row slices are +// copied but their contents are not. This permits appending to the rows further // (which happens during map rendering) without affecting the validity of // copies made for snapshots during rendering. -func (fm filterMap) copy() filterMap { +// Appending to the rows of both the original map and the fast copy, or two fast +// copies of the same map would result in data corruption, therefore a fast copy +// should always be used in a read only way. +func (fm filterMap) fastCopy() filterMap { + return slices.Clone(fm) +} + +// fullCopy returns a copy of the given filter map, also making a copy of each +// individual filter row, ensuring that a modification to either one will never +// affect the other. +func (fm filterMap) fullCopy() filterMap { c := make(filterMap, len(fm)) - copy(c, fm) + for i, row := range fm { + c[i] = slices.Clone(row) + } return c } @@ -207,8 +221,9 @@ type Config struct { // NewFilterMaps creates a new FilterMaps and starts the indexer. func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, finalBlock uint64, params Params, config Config) *FilterMaps { rs, initialized, err := rawdb.ReadFilterMapsRange(db) - if err != nil { - log.Error("Error reading log index range", "error", err) + if err != nil || rs.Version != databaseVersion { + rs, initialized = rawdb.FilterMapsRange{}, false + log.Warn("Invalid log index database version; resetting log index") } params.deriveFields() f := &FilterMaps{ @@ -248,7 +263,7 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f f.targetView = initView if f.indexedRange.initialized { f.indexedView = f.initChainView(f.targetView) - f.indexedRange.headIndexed = f.indexedRange.blocks.AfterLast() == f.indexedView.headNumber+1 + f.indexedRange.headIndexed = f.indexedRange.blocks.AfterLast() == f.indexedView.HeadNumber()+1 if !f.indexedRange.headIndexed { f.indexedRange.headDelimiter = 0 } @@ -299,7 +314,7 @@ func (f *FilterMaps) initChainView(chainView *ChainView) *ChainView { log.Error("Could not initialize indexed chain view", "error", err) break } - if lastBlockNumber <= chainView.headNumber && chainView.getBlockId(lastBlockNumber) == lastBlockId { + if lastBlockNumber <= chainView.HeadNumber() && chainView.BlockId(lastBlockNumber) == lastBlockId { return chainView.limitedView(lastBlockNumber) } } @@ -356,7 +371,7 @@ func (f *FilterMaps) init() error { for min < max { mid := (min + max + 1) / 2 cp := checkpointList[mid-1] - if cp.BlockNumber <= f.targetView.headNumber && f.targetView.getBlockId(cp.BlockNumber) == cp.BlockId { + if cp.BlockNumber <= f.targetView.HeadNumber() && f.targetView.BlockId(cp.BlockNumber) == cp.BlockId { min = mid } else { max = mid - 1 @@ -437,6 +452,7 @@ func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, ne f.updateMatchersValidRange() if newRange.initialized { rs := rawdb.FilterMapsRange{ + Version: databaseVersion, HeadIndexed: newRange.headIndexed, HeadDelimiter: newRange.headDelimiter, BlocksFirst: newRange.blocks.First(), @@ -497,7 +513,7 @@ func (f *FilterMaps) getLogByLvIndex(lvIndex uint64) (*types.Log, error) { } } // get block receipts - receipts := f.indexedView.getReceipts(firstBlockNumber) + receipts := f.indexedView.Receipts(firstBlockNumber) if receipts == nil { return nil, fmt.Errorf("failed to retrieve receipts for block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err) } @@ -558,7 +574,7 @@ func (f *FilterMaps) getFilterMapRow(mapIndex, rowIndex uint32, baseLayerOnly bo } f.baseRowsCache.Add(baseMapRowIndex, baseRows) } - baseRow := baseRows[mapIndex&(f.baseRowGroupLength-1)] + baseRow := slices.Clone(baseRows[mapIndex&(f.baseRowGroupLength-1)]) if baseLayerOnly { return baseRow, nil } @@ -595,7 +611,9 @@ func (f *FilterMaps) storeFilterMapRowsOfGroup(batch ethdb.Batch, mapIndices []u if uint32(len(mapIndices)) != f.baseRowGroupLength { // skip base rows read if all rows are replaced var ok bool baseRows, ok = f.baseRowsCache.Get(baseMapRowIndex) - if !ok { + if ok { + baseRows = slices.Clone(baseRows) + } else { var err error baseRows, err = rawdb.ReadFilterMapBaseRows(f.db, baseMapRowIndex, f.baseRowGroupLength, f.logMapWidth) if err != nil { @@ -641,7 +659,7 @@ func (f *FilterMaps) mapRowIndex(mapIndex, rowIndex uint32) uint64 { // called from outside the indexerLoop goroutine. func (f *FilterMaps) getBlockLvPointer(blockNumber uint64) (uint64, error) { if blockNumber >= f.indexedRange.blocks.AfterLast() && f.indexedRange.headIndexed { - return f.indexedRange.headDelimiter, nil + return f.indexedRange.headDelimiter + 1, nil } if lvPointer, ok := f.lvPointerCache.Get(blockNumber); ok { return lvPointer, nil diff --git a/core/filtermaps/indexer.go b/core/filtermaps/indexer.go index 9a5424da4a..787197345a 100644 --- a/core/filtermaps/indexer.go +++ b/core/filtermaps/indexer.go @@ -44,7 +44,7 @@ func (f *FilterMaps) indexerLoop() { for !f.stop { if !f.indexedRange.initialized { - if f.targetView.headNumber == 0 { + if f.targetView.HeadNumber() == 0 { // initialize when chain head is available f.processSingleEvent(true) continue @@ -165,6 +165,9 @@ func (f *FilterMaps) waitForNewHead() { // processEvents processes all events, blocking only if a block processing is // happening and indexing should be suspended. func (f *FilterMaps) processEvents() { + if f.testProcessEventsHook != nil { + f.testProcessEventsHook() + } for f.processSingleEvent(f.blockProcessing) { } } @@ -249,7 +252,7 @@ func (f *FilterMaps) tryIndexHead() error { log.Info("Log index head rendering in progress", "first block", f.indexedRange.blocks.First(), "last block", f.indexedRange.blocks.Last(), "processed", f.indexedRange.blocks.AfterLast()-f.ptrHeadIndex, - "remaining", f.indexedView.headNumber-f.indexedRange.blocks.Last(), + "remaining", f.indexedView.HeadNumber()-f.indexedRange.blocks.Last(), "elapsed", common.PrettyDuration(time.Since(f.startedHeadIndexAt))) f.loggedHeadIndex = true f.lastLogHeadIndex = time.Now() @@ -418,10 +421,10 @@ func (f *FilterMaps) needTailEpoch(epoch uint32) bool { // tailTargetBlock returns the target value for the tail block number according // to the log history parameter and the current index head. func (f *FilterMaps) tailTargetBlock() uint64 { - if f.history == 0 || f.indexedView.headNumber < f.history { + if f.history == 0 || f.indexedView.HeadNumber() < f.history { return 0 } - return f.indexedView.headNumber + 1 - f.history + return f.indexedView.HeadNumber() + 1 - f.history } // tailPartialBlocks returns the number of rendered blocks in the partially diff --git a/core/filtermaps/indexer_test.go b/core/filtermaps/indexer_test.go index a02f8d2459..e60130ba4b 100644 --- a/core/filtermaps/indexer_test.go +++ b/core/filtermaps/indexer_test.go @@ -17,8 +17,10 @@ package filtermaps import ( + "context" crand "crypto/rand" "crypto/sha256" + "encoding/binary" "math/big" "math/rand" "sync" @@ -31,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" ) var testParams = Params{ @@ -104,6 +107,7 @@ func TestIndexerRandomRange(t *testing.T) { fork, head = rand.Intn(len(forks)), rand.Intn(1001) ts.chain.setCanonicalChain(forks[fork][:head+1]) case 2: + checkSnapshot = false if head < 1000 { checkSnapshot = !noHistory && head != 0 // no snapshot generated for block 0 // add blocks after the current head @@ -158,6 +162,115 @@ func TestIndexerRandomRange(t *testing.T) { } } +func TestIndexerMatcherView(t *testing.T) { + testIndexerMatcherView(t, false) +} + +func TestIndexerMatcherViewWithConcurrentRead(t *testing.T) { + testIndexerMatcherView(t, true) +} + +func testIndexerMatcherView(t *testing.T, concurrentRead bool) { + ts := newTestSetup(t) + defer ts.close() + + forks := make([][]common.Hash, 20) + hashes := make([]common.Hash, 20) + ts.chain.addBlocks(100, 5, 2, 4, true) + ts.setHistory(0, false) + for i := range forks { + if i != 0 { + ts.chain.setHead(100 - i) + ts.chain.addBlocks(i, 5, 2, 4, true) + } + ts.fm.WaitIdle() + forks[i] = ts.chain.getCanonicalChain() + hashes[i] = ts.matcherViewHash() + } + fork := len(forks) - 1 + for i := 0; i < 5000; i++ { + oldFork := fork + fork = rand.Intn(len(forks)) + stopCh := make(chan chan struct{}) + if concurrentRead { + go func() { + for { + ts.matcherViewHash() + select { + case ch := <-stopCh: + close(ch) + return + default: + } + } + }() + } + ts.chain.setCanonicalChain(forks[fork]) + ts.fm.WaitIdle() + if concurrentRead { + ch := make(chan struct{}) + stopCh <- ch + <-ch + } + hash := ts.matcherViewHash() + if hash != hashes[fork] { + t.Fatalf("Matcher view hash mismatch when switching from for %d to %d", oldFork, fork) + } + } +} + +func TestLogsByIndex(t *testing.T) { + ts := newTestSetup(t) + defer func() { + ts.fm.testProcessEventsHook = nil + ts.close() + }() + + ts.chain.addBlocks(1000, 10, 3, 4, true) + ts.setHistory(0, false) + ts.fm.WaitIdle() + firstLog := make([]uint64, 1001) // first valid log position per block + lastLog := make([]uint64, 1001) // last valid log position per block + for i := uint64(0); i <= ts.fm.indexedRange.headDelimiter; i++ { + log, err := ts.fm.getLogByLvIndex(i) + if err != nil { + t.Fatalf("Error getting log by index %d: %v", i, err) + } + if log != nil { + if firstLog[log.BlockNumber] == 0 { + firstLog[log.BlockNumber] = i + } + lastLog[log.BlockNumber] = i + } + } + var failed bool + ts.fm.testProcessEventsHook = func() { + if ts.fm.indexedRange.blocks.IsEmpty() { + return + } + if lvi := firstLog[ts.fm.indexedRange.blocks.First()]; lvi != 0 { + log, err := ts.fm.getLogByLvIndex(lvi) + if log == nil || err != nil { + t.Errorf("Error getting first log of indexed block range: %v", err) + failed = true + } + } + if lvi := lastLog[ts.fm.indexedRange.blocks.Last()]; lvi != 0 { + log, err := ts.fm.getLogByLvIndex(lvi) + if log == nil || err != nil { + t.Errorf("Error getting last log of indexed block range: %v", err) + failed = true + } + } + } + chain := ts.chain.getCanonicalChain() + for i := 0; i < 1000 && !failed; i++ { + head := rand.Intn(len(chain)) + ts.chain.setCanonicalChain(chain[:head+1]) + ts.fm.WaitIdle() + } +} + func TestIndexerCompareDb(t *testing.T) { ts := newTestSetup(t) defer ts.close() @@ -291,6 +404,55 @@ func (ts *testSetup) fmDbHash() common.Hash { return result } +func (ts *testSetup) matcherViewHash() common.Hash { + mb := ts.fm.NewMatcherBackend() + defer mb.Close() + + ctx := context.Background() + params := mb.GetParams() + hasher := sha256.New() + var headPtr uint64 + for b := uint64(0); ; b++ { + lvptr, err := mb.GetBlockLvPointer(ctx, b) + if err != nil || (b > 0 && lvptr == headPtr) { + break + } + var enc [8]byte + binary.LittleEndian.PutUint64(enc[:], lvptr) + hasher.Write(enc[:]) + headPtr = lvptr + } + headMap := uint32(headPtr >> params.logValuesPerMap) + var enc [12]byte + for r := uint32(0); r < params.mapHeight; r++ { + binary.LittleEndian.PutUint32(enc[:4], r) + for m := uint32(0); m <= headMap; m++ { + binary.LittleEndian.PutUint32(enc[4:8], m) + row, _ := mb.GetFilterMapRow(ctx, m, r, false) + for _, v := range row { + binary.LittleEndian.PutUint32(enc[8:], v) + hasher.Write(enc[:]) + } + } + } + var hash common.Hash + hasher.Sum(hash[:0]) + for i := 0; i < 50; i++ { + hasher.Reset() + hasher.Write(hash[:]) + lvptr := binary.LittleEndian.Uint64(hash[:8]) % headPtr + if log, _ := mb.GetLogByLvIndex(ctx, lvptr); log != nil { + enc, err := rlp.EncodeToBytes(log) + if err != nil { + panic(err) + } + hasher.Write(enc) + } + hasher.Sum(hash[:0]) + } + return hash +} + func (ts *testSetup) close() { if ts.fm != nil { ts.fm.Stop() diff --git a/core/filtermaps/map_renderer.go b/core/filtermaps/map_renderer.go index 28f943abb3..f59a01c032 100644 --- a/core/filtermaps/map_renderer.go +++ b/core/filtermaps/map_renderer.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "math" + "slices" "sort" "time" @@ -84,7 +85,7 @@ func (f *FilterMaps) renderMapsBefore(renderBefore uint32) (*mapRenderer, error) if err != nil { return nil, err } - if snapshot := f.lastCanonicalSnapshotBefore(renderBefore); snapshot != nil && snapshot.mapIndex >= nextMap { + if snapshot := f.lastCanonicalSnapshotOfMap(nextMap); snapshot != nil { return f.renderMapsFromSnapshot(snapshot) } if nextMap >= renderBefore { @@ -97,17 +98,17 @@ func (f *FilterMaps) renderMapsBefore(renderBefore uint32) (*mapRenderer, error) // snapshot made at a block boundary. func (f *FilterMaps) renderMapsFromSnapshot(cp *renderedMap) (*mapRenderer, error) { f.testSnapshotUsed = true - iter, err := f.newLogIteratorFromBlockDelimiter(cp.lastBlock) + iter, err := f.newLogIteratorFromBlockDelimiter(cp.lastBlock, cp.headDelimiter) if err != nil { return nil, fmt.Errorf("failed to create log iterator from block delimiter %d: %v", cp.lastBlock, err) } return &mapRenderer{ f: f, currentMap: &renderedMap{ - filterMap: cp.filterMap.copy(), + filterMap: cp.filterMap.fullCopy(), mapIndex: cp.mapIndex, lastBlock: cp.lastBlock, - blockLvPtrs: cp.blockLvPtrs, + blockLvPtrs: slices.Clone(cp.blockLvPtrs), }, finishedMaps: make(map[uint32]*renderedMap), finished: common.NewRange(cp.mapIndex, 0), @@ -137,14 +138,15 @@ func (f *FilterMaps) renderMapsFromMapBoundary(firstMap, renderBefore uint32, st }, nil } -// lastCanonicalSnapshotBefore returns the latest cached snapshot that matches -// the current targetView. -func (f *FilterMaps) lastCanonicalSnapshotBefore(renderBefore uint32) *renderedMap { +// lastCanonicalSnapshotOfMap returns the latest cached snapshot of the given map +// that is also consistent with the current targetView. +func (f *FilterMaps) lastCanonicalSnapshotOfMap(mapIndex uint32) *renderedMap { var best *renderedMap for _, blockNumber := range f.renderSnapshots.Keys() { if cp, _ := f.renderSnapshots.Get(blockNumber); cp != nil && blockNumber < f.indexedRange.blocks.AfterLast() && - blockNumber <= f.targetView.headNumber && f.targetView.getBlockId(blockNumber) == cp.lastBlockId && - cp.mapIndex < renderBefore && (best == nil || blockNumber > best.lastBlock) { + blockNumber <= f.indexedView.HeadNumber() && f.indexedView.BlockId(blockNumber) == cp.lastBlockId && + blockNumber <= f.targetView.HeadNumber() && f.targetView.BlockId(blockNumber) == cp.lastBlockId && + cp.mapIndex == mapIndex && (best == nil || blockNumber > best.lastBlock) { best = cp } } @@ -171,10 +173,9 @@ func (f *FilterMaps) lastCanonicalMapBoundaryBefore(renderBefore uint32) (nextMa if err != nil { return 0, 0, 0, fmt.Errorf("failed to retrieve last block of reverse iterated map %d: %v", mapIndex, err) } - if lastBlock >= f.indexedView.headNumber || lastBlock >= f.targetView.headNumber || - lastBlockId != f.targetView.getBlockId(lastBlock) { - // map is not full or inconsistent with targetView; roll back - continue + if (f.indexedRange.headIndexed && mapIndex >= f.indexedRange.maps.Last()) || + lastBlock >= f.targetView.HeadNumber() || lastBlockId != f.targetView.BlockId(lastBlock) { + continue // map is not full or inconsistent with targetView; roll back } lvPtr, err := f.getBlockLvPointer(lastBlock) if err != nil { @@ -244,10 +245,10 @@ func (f *FilterMaps) loadHeadSnapshot() error { } } f.renderSnapshots.Add(f.indexedRange.blocks.Last(), &renderedMap{ - filterMap: fm, + filterMap: fm.fullCopy(), mapIndex: f.indexedRange.maps.Last(), lastBlock: f.indexedRange.blocks.Last(), - lastBlockId: f.indexedView.getBlockId(f.indexedRange.blocks.Last()), + lastBlockId: f.indexedView.BlockId(f.indexedRange.blocks.Last()), blockLvPtrs: lvPtrs, finished: true, headDelimiter: f.indexedRange.headDelimiter, @@ -257,11 +258,14 @@ func (f *FilterMaps) loadHeadSnapshot() error { // makeSnapshot creates a snapshot of the current state of the rendered map. func (r *mapRenderer) makeSnapshot() { - r.f.renderSnapshots.Add(r.iterator.blockNumber, &renderedMap{ - filterMap: r.currentMap.filterMap.copy(), + if r.iterator.blockNumber != r.currentMap.lastBlock || r.iterator.chainView != r.f.targetView { + panic("iterator state inconsistent with current rendered map") + } + r.f.renderSnapshots.Add(r.currentMap.lastBlock, &renderedMap{ + filterMap: r.currentMap.filterMap.fastCopy(), mapIndex: r.currentMap.mapIndex, - lastBlock: r.iterator.blockNumber, - lastBlockId: r.f.targetView.getBlockId(r.currentMap.lastBlock), + lastBlock: r.currentMap.lastBlock, + lastBlockId: r.iterator.chainView.BlockId(r.currentMap.lastBlock), blockLvPtrs: r.currentMap.blockLvPtrs, finished: true, headDelimiter: r.iterator.lvIndex, @@ -367,7 +371,7 @@ func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) { r.currentMap.finished = true r.currentMap.headDelimiter = r.iterator.lvIndex } - r.currentMap.lastBlockId = r.f.targetView.getBlockId(r.currentMap.lastBlock) + r.currentMap.lastBlockId = r.f.targetView.BlockId(r.currentMap.lastBlock) totalTime += time.Since(start) mapRenderTimer.Update(totalTime) mapLogValueMeter.Mark(logValuesProcessed) @@ -533,6 +537,7 @@ func (r *mapRenderer) getTempRange() (filterMapsRange, error) { } else { tempRange.blocks.SetAfterLast(0) } + tempRange.headIndexed = false tempRange.headDelimiter = 0 } return tempRange, nil @@ -563,8 +568,8 @@ func (r *mapRenderer) getUpdatedRange() (filterMapsRange, error) { lm := r.finishedMaps[r.finished.Last()] newRange.headIndexed = lm.finished if lm.finished { - newRange.blocks.SetLast(r.f.targetView.headNumber) - if lm.lastBlock != r.f.targetView.headNumber { + newRange.blocks.SetLast(r.f.targetView.HeadNumber()) + if lm.lastBlock != r.f.targetView.HeadNumber() { panic("map rendering finished but last block != head block") } newRange.headDelimiter = lm.headDelimiter @@ -661,25 +666,14 @@ var errUnindexedRange = errors.New("unindexed range") // newLogIteratorFromBlockDelimiter creates a logIterator starting at the // given block's first log value entry (the block delimiter), according to the // current targetView. -func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber uint64) (*logIterator, error) { - if blockNumber > f.targetView.headNumber { - return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", blockNumber, f.targetView.headNumber) +func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber, lvIndex uint64) (*logIterator, error) { + if blockNumber > f.targetView.HeadNumber() { + return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", blockNumber, f.targetView.HeadNumber()) } if !f.indexedRange.blocks.Includes(blockNumber) { return nil, errUnindexedRange } - var lvIndex uint64 - if f.indexedRange.headIndexed && blockNumber+1 == f.indexedRange.blocks.AfterLast() { - lvIndex = f.indexedRange.headDelimiter - } else { - var err error - lvIndex, err = f.getBlockLvPointer(blockNumber + 1) - if err != nil { - return nil, fmt.Errorf("failed to retrieve log value pointer of block %d after delimiter: %v", blockNumber+1, err) - } - lvIndex-- - } - finished := blockNumber == f.targetView.headNumber + finished := blockNumber == f.targetView.HeadNumber() l := &logIterator{ chainView: f.targetView, params: &f.Params, @@ -695,11 +689,11 @@ func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber uint64) (*logI // newLogIteratorFromMapBoundary creates a logIterator starting at the given // map boundary, according to the current targetView. func (f *FilterMaps) newLogIteratorFromMapBoundary(mapIndex uint32, startBlock, startLvPtr uint64) (*logIterator, error) { - if startBlock > f.targetView.headNumber { - return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", startBlock, f.targetView.headNumber) + if startBlock > f.targetView.HeadNumber() { + return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", startBlock, f.targetView.HeadNumber()) } // get block receipts - receipts := f.targetView.getReceipts(startBlock) + receipts := f.targetView.Receipts(startBlock) if receipts == nil { return nil, fmt.Errorf("receipts not found for start block %d", startBlock) } @@ -766,7 +760,7 @@ func (l *logIterator) next() error { if l.delimiter { l.delimiter = false l.blockNumber++ - l.receipts = l.chainView.getReceipts(l.blockNumber) + l.receipts = l.chainView.Receipts(l.blockNumber) if l.receipts == nil { return fmt.Errorf("receipts not found for block %d", l.blockNumber) } @@ -803,7 +797,7 @@ func (l *logIterator) enforceValidState() { } l.logIndex = 0 } - if l.blockNumber == l.chainView.headNumber { + if l.blockNumber == l.chainView.HeadNumber() { l.finished = true } else { l.delimiter = true diff --git a/core/filtermaps/matcher.go b/core/filtermaps/matcher.go index 5738bf166a..a5eeaaa5f0 100644 --- a/core/filtermaps/matcher.go +++ b/core/filtermaps/matcher.go @@ -57,7 +57,7 @@ type MatcherBackend interface { // all states of the chain since the previous SyncLogIndex or the creation of // the matcher backend. type SyncRange struct { - HeadNumber uint64 + IndexedView *ChainView // block range where the index has not changed since the last matcher sync // and therefore the set of matches found in this region is guaranteed to // be valid and complete. diff --git a/core/filtermaps/matcher_backend.go b/core/filtermaps/matcher_backend.go index 01bae7bb22..ee18a0694c 100644 --- a/core/filtermaps/matcher_backend.go +++ b/core/filtermaps/matcher_backend.go @@ -75,6 +75,9 @@ func (fm *FilterMapsMatcherBackend) Close() { // on write. // GetFilterMapRow implements MatcherBackend. func (fm *FilterMapsMatcherBackend) GetFilterMapRow(ctx context.Context, mapIndex, rowIndex uint32, baseLayerOnly bool) (FilterRow, error) { + fm.f.indexLock.RLock() + defer fm.f.indexLock.RUnlock() + return fm.f.getFilterMapRow(mapIndex, rowIndex, baseLayerOnly) } @@ -125,7 +128,7 @@ func (fm *FilterMapsMatcherBackend) synced() { indexedBlocks.SetAfterLast(indexedBlocks.Last()) // remove partially indexed last block } fm.syncCh <- SyncRange{ - HeadNumber: fm.f.targetView.headNumber, + IndexedView: fm.f.indexedView, ValidBlocks: fm.validBlocks, IndexedBlocks: indexedBlocks, } @@ -151,7 +154,7 @@ func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange case <-ctx.Done(): return SyncRange{}, ctx.Err() case <-fm.f.disabledCh: - return SyncRange{HeadNumber: fm.f.targetView.headNumber}, nil + return SyncRange{IndexedView: fm.f.indexedView}, nil } select { case vr := <-syncCh: @@ -159,7 +162,7 @@ func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange case <-ctx.Done(): return SyncRange{}, ctx.Err() case <-fm.f.disabledCh: - return SyncRange{HeadNumber: fm.f.targetView.headNumber}, nil + return SyncRange{IndexedView: fm.f.indexedView}, nil } } diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index 31e2b534be..413e4d77a8 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -76,8 +76,10 @@ func TestCreation(t *testing.T) { {20000000, 1681338454, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 1681338455}}, // Last Gray Glacier block {20000000, 1681338455, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // First Shanghai block {30000000, 1710338134, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // Last Shanghai block - {40000000, 1710338135, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // First Cancun block - {50000000, 2000000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // Future Cancun block + {40000000, 1710338135, ID{Hash: checksumToBytes(0x9f3d2254), Next: 1746612311}}, // First Cancun block + {30000000, 1746022486, ID{Hash: checksumToBytes(0x9f3d2254), Next: 1746612311}}, // Last Cancun block + {30000000, 1746612311, ID{Hash: checksumToBytes(0xc376cf8b), Next: 0}}, // First Prague block + {50000000, 2000000000, ID{Hash: checksumToBytes(0xc376cf8b), Next: 0}}, // Future Prague block }, }, // Sepolia test cases @@ -137,9 +139,11 @@ func TestCreation(t *testing.T) { // fork ID. func TestValidation(t *testing.T) { // Config that has not timestamp enabled + // TODO(lightclient): this always needs to be updated when a mainnet timestamp is set. legacyConfig := *params.MainnetChainConfig legacyConfig.ShanghaiTime = nil legacyConfig.CancunTime = nil + legacyConfig.PragueTime = nil tests := []struct { config *params.ChainConfig @@ -314,9 +318,7 @@ func TestValidation(t *testing.T) { // Local is mainnet Prague, remote announces Shanghai + knowledge about Cancun. Remote // is definitely out of sync. It may or may not need the Prague update, we don't know yet. - // - // TODO(karalabe): Enable this when Cancun **and** Prague is specced, update all the numbers - //{params.MainnetChainConfig, 0, 0, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}, nil}, + {params.MainnetChainConfig, 0, 0, ID{Hash: checksumToBytes(0x3edd5b10), Next: 1710338135}, nil}, // Local is mainnet Shanghai, remote announces Cancun. Local is out of sync, accept. {params.MainnetChainConfig, 21000000, 1700000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}, nil}, @@ -324,8 +326,7 @@ func TestValidation(t *testing.T) { // Local is mainnet Shanghai, remote announces Cancun, but is not aware of Prague. Local // out of sync. Local also knows about a future fork, but that is uncertain yet. // - // TODO(karalabe): Enable this when Cancun **and** Prague is specced, update remote checksum - //{params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x00000000), Next: 0}, nil}, + {params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0xc376cf8b), Next: 0}, nil}, // Local is mainnet Cancun. remote announces Shanghai but is not aware of further forks. // Remote needs software update. @@ -342,11 +343,11 @@ func TestValidation(t *testing.T) { // Local is mainnet Shanghai, remote is random Shanghai. {params.MainnetChainConfig, 20000000, 1681338455, ID{Hash: checksumToBytes(0x12345678), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Cancun, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Prague, far in the future. Remote announces Gopherium (non existing fork) // at some future timestamp 8888888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0x9f3d2254), Next: 8888888888}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0xc376cf8b), Next: 8888888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Shanghai. Remote is also in Shanghai, but announces Gopherium (non existing // fork) at timestamp 1668000000, before Cancun. Local is incompatible. diff --git a/eth/ethconfig/historymode.go b/core/history/historymode.go similarity index 79% rename from eth/ethconfig/historymode.go rename to core/history/historymode.go index a595d72feb..e735222d37 100644 --- a/eth/ethconfig/historymode.go +++ b/core/history/historymode.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package ethconfig +package history import ( "fmt" @@ -27,22 +27,22 @@ import ( type HistoryMode uint32 const ( - // AllHistory (default) means that all chain history down to genesis block will be kept. - AllHistory HistoryMode = iota + // KeepAll (default) means that all chain history down to genesis block will be kept. + KeepAll HistoryMode = iota - // PostMergeHistory sets the history pruning point to the merge activation block. - PostMergeHistory + // KeepPostMerge sets the history pruning point to the merge activation block. + KeepPostMerge ) func (m HistoryMode) IsValid() bool { - return m <= PostMergeHistory + return m <= KeepPostMerge } func (m HistoryMode) String() string { switch m { - case AllHistory: + case KeepAll: return "all" - case PostMergeHistory: + case KeepPostMerge: return "postmerge" default: return fmt.Sprintf("invalid HistoryMode(%d)", m) @@ -61,24 +61,24 @@ func (m HistoryMode) MarshalText() ([]byte, error) { func (m *HistoryMode) UnmarshalText(text []byte) error { switch string(text) { case "all": - *m = AllHistory + *m = KeepAll case "postmerge": - *m = PostMergeHistory + *m = KeepPostMerge default: return fmt.Errorf(`unknown sync mode %q, want "all" or "postmerge"`, text) } return nil } -type HistoryPrunePoint struct { +type PrunePoint struct { BlockNumber uint64 BlockHash common.Hash } -// HistoryPrunePoints contains the pre-defined history pruning cutoff blocks for known networks. +// PrunePoints the pre-defined history pruning cutoff blocks for known networks. // They point to the first post-merge block. Any pruning should truncate *up to* but excluding // given block. -var HistoryPrunePoints = map[common.Hash]*HistoryPrunePoint{ +var PrunePoints = map[common.Hash]*PrunePoint{ // mainnet params.MainnetGenesisHash: { BlockNumber: 15537393, @@ -91,7 +91,7 @@ var HistoryPrunePoints = map[common.Hash]*HistoryPrunePoint{ }, } -// PrunedHistoryError is returned when the requested history is pruned. +// PrunedHistoryError is returned by APIs when the requested history is pruned. type PrunedHistoryError struct{} func (e *PrunedHistoryError) Error() string { return "pruned history unavailable" } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index c413839b7b..1a5c414c8e 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -434,6 +434,7 @@ func DeleteBlockLvPointers(db ethdb.KeyValueStore, blocks common.Range[uint64], // FilterMapsRange is a storage representation of the block range covered by the // filter maps structure and the corresponting log value index range. type FilterMapsRange struct { + Version uint32 HeadIndexed bool HeadDelimiter uint64 BlocksFirst, BlocksAfterLast uint64 diff --git a/core/state/dump.go b/core/state/dump.go index c9aad4f8e2..11b5c32782 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -188,7 +188,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] c.OnAccount(address, account) accounts++ if time.Since(logged) > 8*time.Second { - log.Info("Trie dumping in progress", "at", it.Key, "accounts", accounts, + log.Info("Trie dumping in progress", "at", common.Bytes2Hex(it.Key), "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() } diff --git a/core/state_processor.go b/core/state_processor.go index 9241d091ad..322bd24f41 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -113,9 +113,13 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, err } // EIP-7002 - ProcessWithdrawalQueue(&requests, evm) + if err := ProcessWithdrawalQueue(&requests, evm); err != nil { + return nil, err + } // EIP-7251 - ProcessConsolidationQueue(&requests, evm) + if err := ProcessConsolidationQueue(&requests, evm); err != nil { + return nil, err + } } // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) @@ -265,17 +269,17 @@ func ProcessParentBlockHash(prevHash common.Hash, evm *vm.EVM) { // ProcessWithdrawalQueue calls the EIP-7002 withdrawal queue contract. // It returns the opaque request data returned by the contract. -func ProcessWithdrawalQueue(requests *[][]byte, evm *vm.EVM) { - processRequestsSystemCall(requests, evm, 0x01, params.WithdrawalQueueAddress) +func ProcessWithdrawalQueue(requests *[][]byte, evm *vm.EVM) error { + return processRequestsSystemCall(requests, evm, 0x01, params.WithdrawalQueueAddress) } // ProcessConsolidationQueue calls the EIP-7251 consolidation queue contract. // It returns the opaque request data returned by the contract. -func ProcessConsolidationQueue(requests *[][]byte, evm *vm.EVM) { - processRequestsSystemCall(requests, evm, 0x02, params.ConsolidationQueueAddress) +func ProcessConsolidationQueue(requests *[][]byte, evm *vm.EVM) error { + return processRequestsSystemCall(requests, evm, 0x02, params.ConsolidationQueueAddress) } -func processRequestsSystemCall(requests *[][]byte, evm *vm.EVM, requestType byte, addr common.Address) { +func processRequestsSystemCall(requests *[][]byte, evm *vm.EVM, requestType byte, addr common.Address) error { if tracer := evm.Config.Tracer; tracer != nil { onSystemCallStart(tracer, evm.GetVMContext()) if tracer.OnSystemCallEnd != nil { @@ -292,17 +296,20 @@ func processRequestsSystemCall(requests *[][]byte, evm *vm.EVM, requestType byte } evm.SetTxContext(NewEVMTxContext(msg)) evm.StateDB.AddAddressToAccessList(addr) - ret, _, _ := evm.Call(msg.From, *msg.To, msg.Data, 30_000_000, common.U2560) + ret, _, err := evm.Call(msg.From, *msg.To, msg.Data, 30_000_000, common.U2560) evm.StateDB.Finalise(true) - if len(ret) == 0 { - return // skip empty output + if err != nil { + return fmt.Errorf("system call failed to execute: %v", err) + } + if len(ret) == 0 { + return nil // skip empty output } - // Append prefixed requestsData to the requests list. requestsData := make([]byte, len(ret)+1) requestsData[0] = requestType copy(requestsData[1:], ret) *requests = append(*requests, requestsData) + return nil } var depositTopic = common.HexToHash("0x649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5") diff --git a/core/txindexer.go b/core/txindexer.go index 31f069995b..64a2e8c49f 100644 --- a/core/txindexer.go +++ b/core/txindexer.go @@ -196,6 +196,19 @@ func (indexer *txIndexer) repair(head uint64) { } } +// resolveHead resolves the block number of the current chain head. +func (indexer *txIndexer) resolveHead() uint64 { + headBlockHash := rawdb.ReadHeadBlockHash(indexer.db) + if headBlockHash == (common.Hash{}) { + return 0 + } + headBlockNumber := rawdb.ReadHeaderNumber(indexer.db, headBlockHash) + if headBlockNumber == nil { + return 0 + } + return *headBlockNumber +} + // loop is the scheduler of the indexer, assigning indexing/unindexing tasks depending // on the received chain event. func (indexer *txIndexer) loop(chain *BlockChain) { @@ -203,9 +216,9 @@ func (indexer *txIndexer) loop(chain *BlockChain) { // Listening to chain events and manipulate the transaction indexes. var ( - stop chan struct{} // Non-nil if background routine is active - done chan struct{} // Non-nil if background routine is active - head = rawdb.ReadHeadBlock(indexer.db).NumberU64() // The latest announced chain head + stop chan struct{} // Non-nil if background routine is active + done chan struct{} // Non-nil if background routine is active + head = indexer.resolveHead() // The latest announced chain head headCh = make(chan ChainHeadEvent) sub = chain.SubscribeChainHeadEvent(headCh) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 12a4133b40..e506da228d 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1391,6 +1391,8 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { switch { case errors.Is(err, txpool.ErrUnderpriced): addUnderpricedMeter.Mark(1) + case errors.Is(err, txpool.ErrTxGasPriceTooLow): + addUnderpricedMeter.Mark(1) case errors.Is(err, core.ErrNonceTooLow): addStaleMeter.Mark(1) case errors.Is(err, core.ErrNonceTooHigh): diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 76d21a0c9e..0a323179a6 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -1484,7 +1484,7 @@ func TestAdd(t *testing.T) { { // New account, no previous txs, nonce 0, but blob fee cap too low from: "alice", tx: makeUnsignedTx(0, 1, 1, 0), - err: txpool.ErrUnderpriced, + err: txpool.ErrTxGasPriceTooLow, }, { // Same as above but blob fee cap equals minimum, should be accepted from: "alice", diff --git a/core/txpool/errors.go b/core/txpool/errors.go index 02f5703b6c..968c9d9542 100644 --- a/core/txpool/errors.go +++ b/core/txpool/errors.go @@ -16,7 +16,9 @@ package txpool -import "errors" +import ( + "errors" +) var ( // ErrAlreadyKnown is returned if the transactions is already contained @@ -26,14 +28,19 @@ var ( // ErrInvalidSender is returned if the transaction contains an invalid signature. ErrInvalidSender = errors.New("invalid sender") - // ErrUnderpriced is returned if a transaction's gas price is below the minimum - // configured for the transaction pool. + // ErrUnderpriced is returned if a transaction's gas price is too low to be + // included in the pool. If the gas price is lower than the minimum configured + // one for the transaction pool, use ErrTxGasPriceTooLow instead. ErrUnderpriced = errors.New("transaction underpriced") // ErrReplaceUnderpriced is returned if a transaction is attempted to be replaced // with a different one without the required price bump. ErrReplaceUnderpriced = errors.New("replacement transaction underpriced") + // ErrTxGasPriceTooLow is returned if a transaction's gas price is below the + // minimum configured for the transaction pool. + ErrTxGasPriceTooLow = errors.New("transaction gas price below minimum") + // ErrAccountLimitExceeded is returned if a transaction would exceed the number // allowed by a pool for a single account. ErrAccountLimitExceeded = errors.New("account limit exceeded") diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 04f1a2234c..7bf360ff65 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1827,6 +1827,16 @@ func (t *lookup) Remove(hash common.Hash) { delete(t.txs, hash) } +// Clear resets the lookup structure, removing all stored entries. +func (t *lookup) Clear() { + t.lock.Lock() + defer t.lock.Unlock() + + t.slots = 0 + t.txs = make(map[common.Hash]*types.Transaction) + t.auths = make(map[common.Address][]common.Hash) +} + // TxsBelowTip finds all remote transactions below the given tip threshold. func (t *lookup) TxsBelowTip(threshold *big.Int) types.Transactions { found := make(types.Transactions, 0, 128) @@ -1923,7 +1933,7 @@ func (pool *LegacyPool) Clear() { for addr := range pool.queue { pool.reserver.Release(addr) } - pool.all = newLookup() + pool.all.Clear() pool.priced = newPricedList(pool.all) pool.pending = make(map[common.Address]*list) pool.queue = make(map[common.Address]*list) diff --git a/core/txpool/legacypool/legacypool2_test.go b/core/txpool/legacypool/legacypool2_test.go index 3f210e3d1b..deb06aa617 100644 --- a/core/txpool/legacypool/legacypool2_test.go +++ b/core/txpool/legacypool/legacypool2_test.go @@ -82,12 +82,14 @@ func TestTransactionFutureAttack(t *testing.T) { // Create the pool to test the limit enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) + config := testTxPoolConfig config.GlobalQueue = 100 config.GlobalSlots = 100 pool := New(config, blockchain) pool.Init(config.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() + fillPool(t, pool) pending, _ := pool.Stats() // Now, future transaction attack starts, let's add a bunch of expensive non-executables, and see if the pending-count drops @@ -180,7 +182,9 @@ func TestTransactionZAttack(t *testing.T) { ivPending := countInvalidPending() t.Logf("invalid pending: %d\n", ivPending) - // Now, DETER-Z attack starts, let's add a bunch of expensive non-executables (from N accounts) along with balance-overdraft txs (from one account), and see if the pending-count drops + // Now, DETER-Z attack starts, let's add a bunch of expensive non-executables + // (from N accounts) along with balance-overdraft txs (from one account), and + // see if the pending-count drops for j := 0; j < int(pool.config.GlobalQueue); j++ { futureTxs := types.Transactions{} key, _ := crypto.GenerateKey() diff --git a/core/txpool/legacypool/legacypool_test.go b/core/txpool/legacypool/legacypool_test.go index bb1323a7d1..2fdf890320 100644 --- a/core/txpool/legacypool/legacypool_test.go +++ b/core/txpool/legacypool/legacypool_test.go @@ -413,7 +413,7 @@ func TestInvalidTransactions(t *testing.T) { tx = transaction(1, 100000, key) pool.gasTip.Store(uint256.NewInt(1000)) - if err, want := pool.addRemote(tx), txpool.ErrUnderpriced; !errors.Is(err, want) { + if err, want := pool.addRemote(tx), txpool.ErrTxGasPriceTooLow; !errors.Is(err, want) { t.Errorf("want %v have %v", want, err) } } @@ -484,7 +484,7 @@ func TestNegativeValue(t *testing.T) { tx, _ := types.SignTx(types.NewTransaction(0, common.Address{}, big.NewInt(-1), 100, big.NewInt(1), nil), types.HomesteadSigner{}, key) from, _ := deriveSender(tx) testAddBalance(pool, from, big.NewInt(1)) - if err := pool.addRemote(tx); err != txpool.ErrNegativeValue { + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrNegativeValue) { t.Error("expected", txpool.ErrNegativeValue, "got", err) } } @@ -497,7 +497,7 @@ func TestTipAboveFeeCap(t *testing.T) { tx := dynamicFeeTx(0, 100, big.NewInt(1), big.NewInt(2), key) - if err := pool.addRemote(tx); err != core.ErrTipAboveFeeCap { + if err := pool.addRemote(tx); !errors.Is(err, core.ErrTipAboveFeeCap) { t.Error("expected", core.ErrTipAboveFeeCap, "got", err) } } @@ -512,12 +512,12 @@ func TestVeryHighValues(t *testing.T) { veryBigNumber.Lsh(veryBigNumber, 300) tx := dynamicFeeTx(0, 100, big.NewInt(1), veryBigNumber, key) - if err := pool.addRemote(tx); err != core.ErrTipVeryHigh { + if err := pool.addRemote(tx); !errors.Is(err, core.ErrTipVeryHigh) { t.Error("expected", core.ErrTipVeryHigh, "got", err) } tx2 := dynamicFeeTx(0, 100, veryBigNumber, big.NewInt(1), key) - if err := pool.addRemote(tx2); err != core.ErrFeeCapVeryHigh { + if err := pool.addRemote(tx2); !errors.Is(err, core.ErrFeeCapVeryHigh) { t.Error("expected", core.ErrFeeCapVeryHigh, "got", err) } } @@ -1424,14 +1424,14 @@ func TestRepricing(t *testing.T) { t.Fatalf("pool internal state corrupted: %v", err) } // Check that we can't add the old transactions back - if err := pool.addRemote(pricedTransaction(1, 100000, big.NewInt(1), keys[0])); !errors.Is(err, txpool.ErrUnderpriced) { - t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) + if err := pool.addRemote(pricedTransaction(1, 100000, big.NewInt(1), keys[0])); !errors.Is(err, txpool.ErrTxGasPriceTooLow) { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrTxGasPriceTooLow) } - if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); !errors.Is(err, txpool.ErrUnderpriced) { - t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) + if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); !errors.Is(err, txpool.ErrTxGasPriceTooLow) { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrTxGasPriceTooLow) } - if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(1), keys[2])); !errors.Is(err, txpool.ErrUnderpriced) { - t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) + if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(1), keys[2])); !errors.Is(err, txpool.ErrTxGasPriceTooLow) { + t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, txpool.ErrTxGasPriceTooLow) } if err := validateEvents(events, 0); err != nil { t.Fatalf("post-reprice event firing failed: %v", err) @@ -1476,14 +1476,14 @@ func TestMinGasPriceEnforced(t *testing.T) { tx := pricedTransaction(0, 100000, big.NewInt(2), key) pool.SetGasTip(big.NewInt(tx.GasPrice().Int64() + 1)) - if err := pool.Add([]*types.Transaction{tx}, true)[0]; !errors.Is(err, txpool.ErrUnderpriced) { + if err := pool.Add([]*types.Transaction{tx}, true)[0]; !errors.Is(err, txpool.ErrTxGasPriceTooLow) { t.Fatalf("Min tip not enforced") } tx = dynamicFeeTx(0, 100000, big.NewInt(3), big.NewInt(2), key) pool.SetGasTip(big.NewInt(tx.GasTipCap().Int64() + 1)) - if err := pool.Add([]*types.Transaction{tx}, true)[0]; !errors.Is(err, txpool.ErrUnderpriced) { + if err := pool.Add([]*types.Transaction{tx}, true)[0]; !errors.Is(err, txpool.ErrTxGasPriceTooLow) { t.Fatalf("Min tip not enforced") } } @@ -1560,16 +1560,16 @@ func TestRepricingDynamicFee(t *testing.T) { } // Check that we can't add the old transactions back tx := pricedTransaction(1, 100000, big.NewInt(1), keys[0]) - if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrUnderpriced) { - t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrTxGasPriceTooLow) { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrTxGasPriceTooLow) } tx = dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[1]) - if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrUnderpriced) { - t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrTxGasPriceTooLow) { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrTxGasPriceTooLow) } tx = dynamicFeeTx(2, 100000, big.NewInt(1), big.NewInt(1), keys[2]) - if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrUnderpriced) { - t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrTxGasPriceTooLow) { + t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, txpool.ErrTxGasPriceTooLow) } if err := validateEvents(events, 0); err != nil { t.Fatalf("post-reprice event firing failed: %v", err) @@ -1673,7 +1673,7 @@ func TestUnderpricing(t *testing.T) { t.Fatalf("failed to add well priced transaction: %v", err) } // Ensure that replacing a pending transaction with a future transaction fails - if err := pool.addRemoteSync(pricedTransaction(5, 100000, big.NewInt(6), keys[1])); err != ErrFutureReplacePending { + if err := pool.addRemoteSync(pricedTransaction(5, 100000, big.NewInt(6), keys[1])); !errors.Is(err, ErrFutureReplacePending) { t.Fatalf("adding future replace transaction error mismatch: have %v, want %v", err, ErrFutureReplacePending) } pending, queued = pool.Stats() @@ -1995,7 +1995,7 @@ func TestReplacement(t *testing.T) { if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), key)); err != nil { t.Fatalf("failed to add original cheap pending transaction: %v", err) } - if err := pool.addRemote(pricedTransaction(0, 100001, big.NewInt(1), key)); err != txpool.ErrReplaceUnderpriced { + if err := pool.addRemote(pricedTransaction(0, 100001, big.NewInt(1), key)); !errors.Is(err, txpool.ErrReplaceUnderpriced) { t.Fatalf("original cheap pending transaction replacement error mismatch: have %v, want %v", err, txpool.ErrReplaceUnderpriced) } if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(2), key)); err != nil { @@ -2008,7 +2008,7 @@ func TestReplacement(t *testing.T) { if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(price), key)); err != nil { t.Fatalf("failed to add original proper pending transaction: %v", err) } - if err := pool.addRemote(pricedTransaction(0, 100001, big.NewInt(threshold-1), key)); err != txpool.ErrReplaceUnderpriced { + if err := pool.addRemote(pricedTransaction(0, 100001, big.NewInt(threshold-1), key)); !errors.Is(err, txpool.ErrReplaceUnderpriced) { t.Fatalf("original proper pending transaction replacement error mismatch: have %v, want %v", err, txpool.ErrReplaceUnderpriced) } if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(threshold), key)); err != nil { @@ -2022,7 +2022,7 @@ func TestReplacement(t *testing.T) { if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(1), key)); err != nil { t.Fatalf("failed to add original cheap queued transaction: %v", err) } - if err := pool.addRemote(pricedTransaction(2, 100001, big.NewInt(1), key)); err != txpool.ErrReplaceUnderpriced { + if err := pool.addRemote(pricedTransaction(2, 100001, big.NewInt(1), key)); !errors.Is(err, txpool.ErrReplaceUnderpriced) { t.Fatalf("original cheap queued transaction replacement error mismatch: have %v, want %v", err, txpool.ErrReplaceUnderpriced) } if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(2), key)); err != nil { @@ -2032,7 +2032,7 @@ func TestReplacement(t *testing.T) { if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(price), key)); err != nil { t.Fatalf("failed to add original proper queued transaction: %v", err) } - if err := pool.addRemote(pricedTransaction(2, 100001, big.NewInt(threshold-1), key)); err != txpool.ErrReplaceUnderpriced { + if err := pool.addRemote(pricedTransaction(2, 100001, big.NewInt(threshold-1), key)); !errors.Is(err, txpool.ErrReplaceUnderpriced) { t.Fatalf("original proper queued transaction replacement error mismatch: have %v, want %v", err, txpool.ErrReplaceUnderpriced) } if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(threshold), key)); err != nil { @@ -2096,7 +2096,7 @@ func TestReplacementDynamicFee(t *testing.T) { } // 2. Don't bump tip or feecap => discard tx = dynamicFeeTx(nonce, 100001, big.NewInt(2), big.NewInt(1), key) - if err := pool.addRemote(tx); err != txpool.ErrReplaceUnderpriced { + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrReplaceUnderpriced) { t.Fatalf("original cheap %s transaction replacement error mismatch: have %v, want %v", stage, err, txpool.ErrReplaceUnderpriced) } // 3. Bump both more than min => accept @@ -2117,24 +2117,25 @@ func TestReplacementDynamicFee(t *testing.T) { if err := pool.addRemoteSync(tx); err != nil { t.Fatalf("failed to add original proper %s transaction: %v", stage, err) } + // 6. Bump tip max allowed so it's still underpriced => discard tx = dynamicFeeTx(nonce, 100000, big.NewInt(gasFeeCap), big.NewInt(tipThreshold-1), key) - if err := pool.addRemote(tx); err != txpool.ErrReplaceUnderpriced { + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrReplaceUnderpriced) { t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, txpool.ErrReplaceUnderpriced) } // 7. Bump fee cap max allowed so it's still underpriced => discard tx = dynamicFeeTx(nonce, 100000, big.NewInt(feeCapThreshold-1), big.NewInt(gasTipCap), key) - if err := pool.addRemote(tx); err != txpool.ErrReplaceUnderpriced { + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrReplaceUnderpriced) { t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, txpool.ErrReplaceUnderpriced) } // 8. Bump tip min for acceptance => accept tx = dynamicFeeTx(nonce, 100000, big.NewInt(gasFeeCap), big.NewInt(tipThreshold), key) - if err := pool.addRemote(tx); err != txpool.ErrReplaceUnderpriced { + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrReplaceUnderpriced) { t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, txpool.ErrReplaceUnderpriced) } // 9. Bump fee cap min for acceptance => accept tx = dynamicFeeTx(nonce, 100000, big.NewInt(feeCapThreshold), big.NewInt(gasTipCap), key) - if err := pool.addRemote(tx); err != txpool.ErrReplaceUnderpriced { + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrReplaceUnderpriced) { t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, txpool.ErrReplaceUnderpriced) } // 10. Check events match expected (3 new executable txs during pending, 0 during queue) diff --git a/core/txpool/locals/errors.go b/core/txpool/locals/errors.go new file mode 100644 index 0000000000..fda50bf218 --- /dev/null +++ b/core/txpool/locals/errors.go @@ -0,0 +1,46 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package locals + +import ( + "errors" + + "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" +) + +// IsTemporaryReject determines whether the given error indicates a temporary +// reason to reject a transaction from being included in the txpool. The result +// may change if the txpool's state changes later. +func IsTemporaryReject(err error) bool { + switch { + case errors.Is(err, legacypool.ErrOutOfOrderTxFromDelegated): + return true + case errors.Is(err, txpool.ErrInflightTxLimitReached): + return true + case errors.Is(err, legacypool.ErrAuthorityReserved): + return true + case errors.Is(err, txpool.ErrUnderpriced): + return true + case errors.Is(err, legacypool.ErrTxPoolOverflow): + return true + case errors.Is(err, legacypool.ErrFutureReplacePending): + return true + default: + return false + } +} diff --git a/core/txpool/locals/tx_tracker.go b/core/txpool/locals/tx_tracker.go index eccdcf422a..e08384ce71 100644 --- a/core/txpool/locals/tx_tracker.go +++ b/core/txpool/locals/tx_tracker.go @@ -74,32 +74,22 @@ func New(journalPath string, journalTime time.Duration, chainConfig *params.Chai // Track adds a transaction to the tracked set. // Note: blob-type transactions are ignored. -func (tracker *TxTracker) Track(tx *types.Transaction) error { - return tracker.TrackAll([]*types.Transaction{tx})[0] +func (tracker *TxTracker) Track(tx *types.Transaction) { + tracker.TrackAll([]*types.Transaction{tx}) } // TrackAll adds a list of transactions to the tracked set. // Note: blob-type transactions are ignored. -func (tracker *TxTracker) TrackAll(txs []*types.Transaction) []error { +func (tracker *TxTracker) TrackAll(txs []*types.Transaction) { tracker.mu.Lock() defer tracker.mu.Unlock() - var errors []error for _, tx := range txs { if tx.Type() == types.BlobTxType { - errors = append(errors, nil) - continue - } - // Ignore the transactions which are failed for fundamental - // validation such as invalid parameters. - if err := tracker.pool.ValidateTxBasics(tx); err != nil { - log.Debug("Invalid transaction submitted", "hash", tx.Hash(), "err", err) - errors = append(errors, err) continue } // If we're already tracking it, it's a no-op if _, ok := tracker.all[tx.Hash()]; ok { - errors = append(errors, nil) continue } // Theoretically, checking the error here is unnecessary since sender recovery @@ -108,11 +98,8 @@ func (tracker *TxTracker) TrackAll(txs []*types.Transaction) []error { // Therefore, the error is still checked just in case. addr, err := types.Sender(tracker.signer, tx) if err != nil { - errors = append(errors, err) continue } - errors = append(errors, nil) - tracker.all[tx.Hash()] = tx if tracker.byAddr[addr] == nil { tracker.byAddr[addr] = legacypool.NewSortedMap() @@ -124,7 +111,6 @@ func (tracker *TxTracker) TrackAll(txs []*types.Transaction) []error { } } localGauge.Update(int64(len(tracker.all))) - return errors } // recheck checks and returns any transactions that needs to be resubmitted. diff --git a/core/txpool/locals/tx_tracker_test.go b/core/txpool/locals/tx_tracker_test.go index 5585589b6c..0668d243fc 100644 --- a/core/txpool/locals/tx_tracker_test.go +++ b/core/txpool/locals/tx_tracker_test.go @@ -17,7 +17,6 @@ package locals import ( - "errors" "math/big" "testing" "time" @@ -91,10 +90,12 @@ func (env *testEnv) close() { env.chain.Stop() } +// nolint:unused func (env *testEnv) setGasTip(gasTip uint64) { env.pool.SetGasTip(new(big.Int).SetUint64(gasTip)) } +// nolint:unused func (env *testEnv) makeTx(nonce uint64, gasPrice *big.Int) *types.Transaction { if nonce == 0 { head := env.chain.CurrentHeader() @@ -121,6 +122,7 @@ func (env *testEnv) makeTxs(n int) []*types.Transaction { return txs } +// nolint:unused func (env *testEnv) commit() { head := env.chain.CurrentBlock() block := env.chain.GetBlock(head.Hash(), head.Number.Uint64()) @@ -137,60 +139,6 @@ func (env *testEnv) commit() { } } -func TestRejectInvalids(t *testing.T) { - env := newTestEnv(t, 10, 0, "") - defer env.close() - - var cases = []struct { - gasTip uint64 - tx *types.Transaction - expErr error - commit bool - }{ - { - tx: env.makeTx(5, nil), // stale - expErr: core.ErrNonceTooLow, - }, - { - tx: env.makeTx(11, nil), // future transaction - expErr: nil, - }, - { - gasTip: params.GWei, - tx: env.makeTx(0, new(big.Int).SetUint64(params.GWei/2)), // low price - expErr: txpool.ErrUnderpriced, - }, - { - tx: types.NewTransaction(10, common.Address{0x00}, big.NewInt(1000), params.TxGas, big.NewInt(params.GWei), nil), // invalid signature - expErr: types.ErrInvalidSig, - }, - { - commit: true, - tx: env.makeTx(10, nil), // stale - expErr: core.ErrNonceTooLow, - }, - { - tx: env.makeTx(11, nil), - expErr: nil, - }, - } - for i, c := range cases { - if c.gasTip != 0 { - env.setGasTip(c.gasTip) - } - if c.commit { - env.commit() - } - gotErr := env.tracker.Track(c.tx) - if c.expErr == nil && gotErr != nil { - t.Fatalf("%d, unexpected error: %v", i, gotErr) - } - if c.expErr != nil && !errors.Is(gotErr, c.expErr) { - t.Fatalf("%d, unexpected error, want: %v, got: %v", i, c.expErr, gotErr) - } - } -} - func TestResubmit(t *testing.T) { env := newTestEnv(t, 10, 0, "") defer env.close() diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 2ed38772ce..cc8f74c1b8 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -186,13 +186,15 @@ func (p *TxPool) loop(head *types.Header) { // Try to inject a busy marker and start a reset if successful select { case resetBusy <- struct{}{}: - statedb, err := p.chain.StateAt(newHead.Root) - if err != nil { - log.Crit("Failed to reset txpool state", "err", err) + // Updates the statedb with the new chain head. The head state may be + // unavailable if the initial state sync has not yet completed. + if statedb, err := p.chain.StateAt(newHead.Root); err != nil { + log.Error("Failed to reset txpool state", "err", err) + } else { + p.stateLock.Lock() + p.state = statedb + p.stateLock.Unlock() } - p.stateLock.Lock() - p.state = statedb - p.stateLock.Unlock() // Busy marker injected, start a new subpool reset go func(oldHead, newHead *types.Header) { @@ -322,31 +324,6 @@ func (p *TxPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Pr return nil, nil } -// ValidateTxBasics checks whether a transaction is valid according to the consensus -// rules, but does not check state-dependent validation such as sufficient balance. -func (p *TxPool) ValidateTxBasics(tx *types.Transaction) error { - addr, err := types.Sender(p.signer, tx) - if err != nil { - return err - } - // Reject transactions with stale nonce. Gapped-nonce future transactions - // are considered valid and will be handled by the subpool according to its - // internal policy. - p.stateLock.RLock() - nonce := p.state.GetNonce(addr) - p.stateLock.RUnlock() - - if nonce > tx.Nonce() { - return core.ErrNonceTooLow - } - for _, subpool := range p.subpools { - if subpool.Filter(tx) { - return subpool.ValidateTxBasics(tx) - } - } - return fmt.Errorf("%w: received type %d", core.ErrTxTypeNotSupported, tx.Type()) -} - // Add enqueues a batch of transactions into the pool if they are valid. Due // to the large transaction churn, add may postpone fully integrating the tx // to a later point to batch multiple ones together. diff --git a/core/txpool/validation.go b/core/txpool/validation.go index 8747724247..e370f2ce84 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -131,12 +131,12 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types } // Ensure the gasprice is high enough to cover the requirement of the calling pool if tx.GasTipCapIntCmp(opts.MinTip) < 0 { - return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrUnderpriced, tx.GasTipCap(), opts.MinTip) + return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.GasTipCap(), opts.MinTip) } if tx.Type() == types.BlobTxType { // Ensure the blob fee cap satisfies the minimum blob gas price if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 { - return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrUnderpriced, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice) + return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice) } sidecar := tx.BlobTxSidecar() if sidecar == nil { diff --git a/eth/api_backend.go b/eth/api_backend.go index b39dd4cbdb..10f7ffcbce 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -29,12 +29,13 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/filtermaps" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/locals" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" @@ -156,7 +157,7 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe } block := b.eth.blockchain.GetBlockByNumber(bn) if block == nil && bn < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return block, nil } @@ -168,7 +169,7 @@ func (b *EthAPIBackend) BlockByHash(ctx context.Context, hash common.Hash) (*typ } block := b.eth.blockchain.GetBlock(hash, *number) if block == nil && *number < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return block, nil } @@ -181,7 +182,7 @@ func (b *EthAPIBackend) GetBody(ctx context.Context, hash common.Hash, number rp body := b.eth.blockchain.GetBody(hash) if body == nil { if uint64(number) < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return nil, errors.New("block body not found") } @@ -203,7 +204,7 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r block := b.eth.blockchain.GetBlock(hash, header.Number.Uint64()) if block == nil { if header.Number.Uint64() < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return nil, errors.New("header found, but block body is missing") } @@ -307,19 +308,24 @@ func (b *EthAPIBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscri } func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { - locals := b.eth.localTxTracker - if locals != nil { - if err := locals.Track(signedTx); err != nil { - return err - } - } - // No error will be returned to user if the transaction fails stateful - // validation (e.g., no available slot), as the locally submitted transactions - // may be resubmitted later via the local tracker. err := b.eth.txPool.Add([]*types.Transaction{signedTx}, false)[0] - if err != nil && locals == nil { + + // If the local transaction tracker is not configured, returns whatever + // returned from the txpool. + if b.eth.localTxTracker == nil { return err } + // If the transaction fails with an error indicating it is invalid, or if there is + // very little chance it will be accepted later (e.g., the gas price is below the + // configured minimum, or the sender has insufficient funds to cover the cost), + // propagate the error to the user. + if err != nil && !locals.IsTemporaryReject(err) { + return err + } + // No error will be returned to user if the transaction fails with a temporary + // error and might be accepted later (e.g., the transaction pool is full). + // Locally submitted transactions will be resubmitted later via the local tracker. + b.eth.localTxTracker.Track(signedTx) return nil } @@ -437,6 +443,14 @@ func (b *EthAPIBackend) RPCTxFeeCap() float64 { return b.eth.config.RPCTxFeeCap } +func (b *EthAPIBackend) CurrentView() *filtermaps.ChainView { + head := b.eth.blockchain.CurrentBlock() + if head == nil { + return nil + } + return filtermaps.NewChainView(b.eth.blockchain, head.Number.Uint64(), head.Hash()) +} + func (b *EthAPIBackend) NewMatcherBackend() filtermaps.MatcherBackend { return b.eth.filterMaps.NewMatcherBackend() } diff --git a/eth/api_backend_test.go b/eth/api_backend_test.go new file mode 100644 index 0000000000..049f68d827 --- /dev/null +++ b/eth/api_backend_test.go @@ -0,0 +1,157 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + "context" + "crypto/ecdsa" + "errors" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/beacon" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/blobpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" + "github.com/ethereum/go-ethereum/core/txpool/locals" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" +) + +var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000_000_000_000_000) + gspec = &core.Genesis{ + Config: params.MergedTestChainConfig, + Alloc: types.GenesisAlloc{ + address: {Balance: funds}, + }, + Difficulty: common.Big0, + BaseFee: big.NewInt(params.InitialBaseFee), + } + signer = types.LatestSignerForChainID(gspec.Config.ChainID) +) + +func initBackend(withLocal bool) *EthAPIBackend { + var ( + // Create a database pre-initialize with a genesis block + db = rawdb.NewMemoryDatabase() + engine = beacon.New(ethash.NewFaker()) + ) + chain, _ := core.NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil) + + txconfig := legacypool.DefaultConfig + txconfig.Journal = "" // Don't litter the disk with test journals + + blobPool := blobpool.New(blobpool.Config{Datadir: ""}, chain, nil) + legacyPool := legacypool.New(txconfig, chain) + txpool, _ := txpool.New(txconfig.PriceLimit, chain, []txpool.SubPool{legacyPool, blobPool}) + + eth := &Ethereum{ + blockchain: chain, + txPool: txpool, + } + if withLocal { + eth.localTxTracker = locals.New("", time.Minute, gspec.Config, txpool) + } + return &EthAPIBackend{ + eth: eth, + } +} + +func makeTx(nonce uint64, gasPrice *big.Int, amount *big.Int, key *ecdsa.PrivateKey) *types.Transaction { + if gasPrice == nil { + gasPrice = big.NewInt(params.GWei) + } + if amount == nil { + amount = big.NewInt(1000) + } + tx, _ := types.SignTx(types.NewTransaction(nonce, common.Address{0x00}, amount, params.TxGas, gasPrice, nil), signer, key) + return tx +} + +type unsignedAuth struct { + nonce uint64 + key *ecdsa.PrivateKey +} + +func pricedSetCodeTx(nonce uint64, gaslimit uint64, gasFee, tip *uint256.Int, key *ecdsa.PrivateKey, unsigned []unsignedAuth) *types.Transaction { + var authList []types.SetCodeAuthorization + for _, u := range unsigned { + auth, _ := types.SignSetCode(u.key, types.SetCodeAuthorization{ + ChainID: *uint256.MustFromBig(gspec.Config.ChainID), + Address: common.Address{0x42}, + Nonce: u.nonce, + }) + authList = append(authList, auth) + } + return pricedSetCodeTxWithAuth(nonce, gaslimit, gasFee, tip, key, authList) +} + +func pricedSetCodeTxWithAuth(nonce uint64, gaslimit uint64, gasFee, tip *uint256.Int, key *ecdsa.PrivateKey, authList []types.SetCodeAuthorization) *types.Transaction { + return types.MustSignNewTx(key, signer, &types.SetCodeTx{ + ChainID: uint256.MustFromBig(gspec.Config.ChainID), + Nonce: nonce, + GasTipCap: tip, + GasFeeCap: gasFee, + Gas: gaslimit, + To: common.Address{}, + Value: uint256.NewInt(100), + Data: nil, + AccessList: nil, + AuthList: authList, + }) +} + +func TestSendTx(t *testing.T) { + testSendTx(t, false) + testSendTx(t, true) +} + +func testSendTx(t *testing.T, withLocal bool) { + b := initBackend(withLocal) + + txA := pricedSetCodeTx(0, 250000, uint256.NewInt(params.GWei), uint256.NewInt(params.GWei), key, []unsignedAuth{ + { + nonce: 0, + key: key, + }, + }) + b.SendTx(context.Background(), txA) + + txB := makeTx(1, nil, nil, key) + err := b.SendTx(context.Background(), txB) + + if withLocal { + if err != nil { + t.Fatalf("Unexpected error sending tx: %v", err) + } + } else { + if !errors.Is(err, txpool.ErrInflightTxLimitReached) { + t.Fatalf("Unexpected error, want: %v, got: %v", txpool.ErrInflightTxLimitReached, err) + } + } +} diff --git a/eth/backend.go b/eth/backend.go index 6716a77562..6d1b6bae99 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -76,6 +76,7 @@ type Ethereum struct { handler *handler discmix *enode.FairMix + dropper *dropper // DB interfaces chainDb ethdb.Database // Block chain database @@ -144,7 +145,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { // Here we determine genesis hash and active ChainConfig. // We need these to figure out the consensus parameters and to set up history pruning. - chainConfig, genesisHash, err := core.LoadChainConfig(chainDb, config.Genesis) + chainConfig, _, err := core.LoadChainConfig(chainDb, config.Genesis) if err != nil { return nil, err } @@ -152,22 +153,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - - // Validate history pruning configuration. - var ( - cutoffNumber uint64 - cutoffHash common.Hash - ) - if config.HistoryMode == ethconfig.PostMergeHistory { - prunecfg, ok := ethconfig.HistoryPrunePoints[genesisHash] - if !ok { - return nil, fmt.Errorf("no history pruning point is defined for genesis %x", genesisHash) - } - cutoffNumber = prunecfg.BlockNumber - cutoffHash = prunecfg.BlockHash - log.Info("Chain cutoff configured", "number", cutoffNumber, "hash", cutoffHash) - } - // Set networkID to chainID by default. networkID := config.NetworkId if networkID == 0 { @@ -194,6 +179,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } log.Info("Initialising Ethereum protocol", "network", networkID, "dbversion", dbVer) + // Create BlockChain object. if !config.SkipBcVersionCheck { if bcVersion != nil && *bcVersion > core.BlockChainVersion { return nil, fmt.Errorf("database version is v%d, Geth %s only supports v%d", *bcVersion, version.WithMeta, core.BlockChainVersion) @@ -209,17 +195,16 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { EnablePreimageRecording: config.EnablePreimageRecording, } cacheConfig = &core.CacheConfig{ - TrieCleanLimit: config.TrieCleanCache, - TrieCleanNoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - TrieDirtyDisabled: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, - SnapshotLimit: config.SnapshotCache, - Preimages: config.Preimages, - StateHistory: config.StateHistory, - StateScheme: scheme, - HistoryPruningCutoffNumber: cutoffNumber, - HistoryPruningCutoffHash: cutoffHash, + TrieCleanLimit: config.TrieCleanCache, + TrieCleanNoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + TrieDirtyDisabled: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, + SnapshotLimit: config.SnapshotCache, + Preimages: config.Preimages, + StateHistory: config.StateHistory, + StateScheme: scheme, + ChainHistoryMode: config.HistoryMode, } ) if config.VMTrace != "" { @@ -245,6 +230,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } + + // Initialize filtermaps log index. fmConfig := filtermaps.Config{ History: config.LogHistory, Disabled: config.LogNoHistory, @@ -260,6 +247,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { eth.filterMaps = filtermaps.NewFilterMaps(chainDb, chainView, historyCutoff, finalBlock, filtermaps.DefaultParams, fmConfig) eth.closeFilterMaps = make(chan chan struct{}) + // TxPool if config.TxPool.Journal != "" { config.TxPool.Journal = stack.ResolvePath(config.TxPool.Journal) } @@ -284,6 +272,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { eth.localTxTracker = locals.New(config.TxPool.Journal, rejournal, eth.blockchain.Config(), eth.txPool) stack.RegisterLifecycle(eth.localTxTracker) } + // Permit the downloader to use the trie cache allowance during fast sync cacheLimit := cacheConfig.TrieCleanLimit + cacheConfig.TrieDirtyLimit + cacheConfig.SnapshotLimit if eth.handler, err = newHandler(&handlerConfig{ @@ -300,6 +289,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return nil, err } + eth.dropper = newDropper(eth.p2pServer.MaxDialedConns(), eth.p2pServer.MaxInboundConns()) + eth.miner = miner.New(eth, config.Miner, eth.engine) eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) eth.miner.SetPrioAddresses(config.TxPool.Locals) @@ -410,6 +401,9 @@ func (s *Ethereum) Start() error { // Start the networking layer s.handler.Start(s.p2pServer.MaxPeers) + // Start the connection manager + s.dropper.Start(s.p2pServer, func() bool { return !s.Synced() }) + // start log indexer s.filterMaps.Start() go s.updateFilterMapsHeads() @@ -511,6 +505,7 @@ func (s *Ethereum) setupDiscovery() error { func (s *Ethereum) Stop() error { // Stop all the peer-related stuff first. s.discmix.Close() + s.dropper.Stop() s.handler.Stop() // Then stop everything else. diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index d51e2ff5b5..f306e5fcfd 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -21,6 +21,7 @@ import ( "crypto/sha256" "errors" "fmt" + "math" "sync" "time" @@ -124,9 +125,13 @@ func NewSimulatedBeacon(period uint64, feeRecipient common.Address, eth *eth.Eth return nil, err } } + + // cap the dev mode period to a reasonable maximum value to avoid + // overflowing the time.Duration (int64) that it will occupy + const maxPeriod = uint64(math.MaxInt64 / time.Second) return &SimulatedBeacon{ eth: eth, - period: period, + period: min(period, maxPeriod), shutdownCh: make(chan struct{}), engineAPI: engineAPI, lastBlockTime: block.Time, diff --git a/eth/dropper.go b/eth/dropper.go new file mode 100644 index 0000000000..51f2a7a95a --- /dev/null +++ b/eth/dropper.go @@ -0,0 +1,167 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import ( + mrand "math/rand" + "slices" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/p2p" +) + +const ( + // Interval between peer drop events (uniform between min and max) + peerDropIntervalMin = 3 * time.Minute + // Interval between peer drop events (uniform between min and max) + peerDropIntervalMax = 7 * time.Minute + // Avoid dropping peers for some time after connection + doNotDropBefore = 10 * time.Minute + // How close to max should we initiate the drop timer. O should be fine, + // dropping when no more peers can be added. Larger numbers result in more + // aggressive drop behavior. + peerDropThreshold = 0 +) + +var ( + // droppedInbound is the number of inbound peers dropped + droppedInbound = metrics.NewRegisteredMeter("eth/dropper/inbound", nil) + // droppedOutbound is the number of outbound peers dropped + droppedOutbound = metrics.NewRegisteredMeter("eth/dropper/outbound", nil) +) + +// dropper monitors the state of the peer pool and makes changes as follows: +// - during sync the Downloader handles peer connections, so dropper is disabled +// - if not syncing and the peer count is close to the limit, it drops peers +// randomly every peerDropInterval to make space for new peers +// - peers are dropped separately from the inboud pool and from the dialed pool +type dropper struct { + maxDialPeers int // maximum number of dialed peers + maxInboundPeers int // maximum number of inbound peers + peersFunc getPeersFunc + syncingFunc getSyncingFunc + + // peerDropTimer introduces churn if we are close to limit capacity. + // We handle Dialed and Inbound connections separately + peerDropTimer *time.Timer + + wg sync.WaitGroup // wg for graceful shutdown + shutdownCh chan struct{} +} + +// Callback type to get the list of connected peers. +type getPeersFunc func() []*p2p.Peer + +// Callback type to get syncing status. +// Returns true while syncing, false when synced. +type getSyncingFunc func() bool + +func newDropper(maxDialPeers, maxInboundPeers int) *dropper { + cm := &dropper{ + maxDialPeers: maxDialPeers, + maxInboundPeers: maxInboundPeers, + peerDropTimer: time.NewTimer(randomDuration(peerDropIntervalMin, peerDropIntervalMax)), + shutdownCh: make(chan struct{}), + } + if peerDropIntervalMin > peerDropIntervalMax { + panic("peerDropIntervalMin duration must be less than or equal to peerDropIntervalMax duration") + } + return cm +} + +// Start the dropper. +func (cm *dropper) Start(srv *p2p.Server, syncingFunc getSyncingFunc) { + cm.peersFunc = srv.Peers + cm.syncingFunc = syncingFunc + cm.wg.Add(1) + go cm.loop() +} + +// Stop the dropper. +func (cm *dropper) Stop() { + cm.peerDropTimer.Stop() + close(cm.shutdownCh) + cm.wg.Wait() +} + +// dropRandomPeer selects one of the peers randomly and drops it from the peer pool. +func (cm *dropper) dropRandomPeer() bool { + peers := cm.peersFunc() + var numInbound int + for _, p := range peers { + if p.Inbound() { + numInbound++ + } + } + numDialed := len(peers) - numInbound + + selectDoNotDrop := func(p *p2p.Peer) bool { + // Avoid dropping trusted and static peers, or recent peers. + // Only drop peers if their respective category (dialed/inbound) + // is close to limit capacity. + return p.Trusted() || p.StaticDialed() || + p.Lifetime() < mclock.AbsTime(doNotDropBefore) || + (p.DynDialed() && cm.maxDialPeers-numDialed > peerDropThreshold) || + (p.Inbound() && cm.maxInboundPeers-numInbound > peerDropThreshold) + } + + droppable := slices.DeleteFunc(peers, selectDoNotDrop) + if len(droppable) > 0 { + p := droppable[mrand.Intn(len(droppable))] + log.Debug("Dropping random peer", "inbound", p.Inbound(), + "id", p.ID(), "duration", common.PrettyDuration(p.Lifetime()), "peercountbefore", len(peers)) + p.Disconnect(p2p.DiscUselessPeer) + if p.Inbound() { + droppedInbound.Mark(1) + } else { + droppedOutbound.Mark(1) + } + return true + } + return false +} + +// randomDuration generates a random duration between min and max. +func randomDuration(min, max time.Duration) time.Duration { + if min > max { + panic("min duration must be less than or equal to max duration") + } + return time.Duration(mrand.Int63n(int64(max-min)) + int64(min)) +} + +// loop is the main loop of the connection dropper. +func (cm *dropper) loop() { + defer cm.wg.Done() + + for { + select { + case <-cm.peerDropTimer.C: + // Drop a random peer if we are not syncing and the peer count is close to the limit. + if !cm.syncingFunc() { + cm.dropRandomPeer() + } + cm.peerDropTimer.Reset(randomDuration(peerDropIntervalMin, peerDropIntervalMax)) + case <-cm.shutdownCh: + return + } + } +} diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 365857347c..5e19824135 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/eth/gasprice" @@ -48,7 +49,7 @@ var FullNodeGPO = gasprice.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ - HistoryMode: AllHistory, + HistoryMode: history.KeepAll, SyncMode: SnapSync, NetworkId: 0, // enable auto configuration of networkID == chainID TxLookupLimit: 2350000, @@ -84,7 +85,7 @@ type Config struct { SyncMode SyncMode // HistoryMode configures chain history retention. - HistoryMode HistoryMode + HistoryMode history.HistoryMode // This can be set to list of enrtree:// URLs which will be queried for // nodes to connect to. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index bcfe8a31d6..cdcddcc772 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/eth/gasprice" @@ -19,13 +20,16 @@ func (c Config) MarshalTOML() (interface{}, error) { Genesis *core.Genesis `toml:",omitempty"` NetworkId uint64 SyncMode SyncMode - HistoryMode HistoryMode + HistoryMode history.HistoryMode EthDiscoveryURLs []string SnapDiscoveryURLs []string NoPruning bool NoPrefetch bool - TxLookupLimit uint64 `toml:",omitempty"` - TransactionHistory uint64 `toml:",omitempty"` + TxLookupLimit uint64 `toml:",omitempty"` + TransactionHistory uint64 `toml:",omitempty"` + LogHistory uint64 `toml:",omitempty"` + LogNoHistory bool `toml:",omitempty"` + LogExportCheckpoints string StateHistory uint64 `toml:",omitempty"` StateScheme string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` @@ -63,6 +67,9 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.NoPrefetch = c.NoPrefetch enc.TxLookupLimit = c.TxLookupLimit enc.TransactionHistory = c.TransactionHistory + enc.LogHistory = c.LogHistory + enc.LogNoHistory = c.LogNoHistory + enc.LogExportCheckpoints = c.LogExportCheckpoints enc.StateHistory = c.StateHistory enc.StateScheme = c.StateScheme enc.RequiredBlocks = c.RequiredBlocks @@ -97,13 +104,16 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { Genesis *core.Genesis `toml:",omitempty"` NetworkId *uint64 SyncMode *SyncMode - HistoryMode *HistoryMode + HistoryMode *history.HistoryMode EthDiscoveryURLs []string SnapDiscoveryURLs []string NoPruning *bool NoPrefetch *bool - TxLookupLimit *uint64 `toml:",omitempty"` - TransactionHistory *uint64 `toml:",omitempty"` + TxLookupLimit *uint64 `toml:",omitempty"` + TransactionHistory *uint64 `toml:",omitempty"` + LogHistory *uint64 `toml:",omitempty"` + LogNoHistory *bool `toml:",omitempty"` + LogExportCheckpoints *string StateHistory *uint64 `toml:",omitempty"` StateScheme *string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` @@ -164,6 +174,15 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TransactionHistory != nil { c.TransactionHistory = *dec.TransactionHistory } + if dec.LogHistory != nil { + c.LogHistory = *dec.LogHistory + } + if dec.LogNoHistory != nil { + c.LogNoHistory = *dec.LogNoHistory + } + if dec.LogExportCheckpoints != nil { + c.LogExportCheckpoints = *dec.LogExportCheckpoints + } if dec.StateHistory != nil { c.StateHistory = *dec.StateHistory } diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 1c192d4112..ff17ae4945 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -345,7 +345,7 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) // Track the transaction hash if the price is too low for us. // Avoid re-request this transaction when we receive another // announcement. - if errors.Is(err, txpool.ErrUnderpriced) || errors.Is(err, txpool.ErrReplaceUnderpriced) { + if errors.Is(err, txpool.ErrUnderpriced) || errors.Is(err, txpool.ErrReplaceUnderpriced) || errors.Is(err, txpool.ErrTxGasPriceTooLow) { f.underpriced.Add(batch[j].Hash(), batch[j].Time()) } // Track a few interesting failure types @@ -355,7 +355,7 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) case errors.Is(err, txpool.ErrAlreadyKnown): duplicate++ - case errors.Is(err, txpool.ErrUnderpriced) || errors.Is(err, txpool.ErrReplaceUnderpriced): + case errors.Is(err, txpool.ErrUnderpriced) || errors.Is(err, txpool.ErrReplaceUnderpriced) || errors.Is(err, txpool.ErrTxGasPriceTooLow): underpriced++ default: diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index 7f3080f5f6..c4c8cac56e 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -1244,10 +1244,12 @@ func TestTransactionFetcherUnderpricedDedup(t *testing.T) { func(txs []*types.Transaction) []error { errs := make([]error, len(txs)) for i := 0; i < len(errs); i++ { - if i%2 == 0 { + if i%3 == 0 { errs[i] = txpool.ErrUnderpriced - } else { + } else if i%3 == 1 { errs[i] = txpool.ErrReplaceUnderpriced + } else { + errs[i] = txpool.ErrTxGasPriceTooLow } } return errs diff --git a/eth/filters/api.go b/eth/filters/api.go index 6593cbef27..864dfd3746 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -28,8 +28,8 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rpc" ) @@ -360,7 +360,7 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type return nil, errInvalidBlockRange } if begin > 0 && begin < int64(api.events.backend.HistoryPruningCutoff()) { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } // Construct the range filter filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index e44b37d047..dd6643c59e 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -26,8 +26,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/filtermaps" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -88,7 +88,7 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { return nil, errors.New("unknown block") } if header.Number.Uint64() < f.sys.backend.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return f.blockLogs(ctx, header) } @@ -146,25 +146,29 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { } const ( - rangeLogsTestSync = iota - rangeLogsTestTrimmed - rangeLogsTestIndexed - rangeLogsTestUnindexed - rangeLogsTestDone + rangeLogsTestDone = iota // zero range + rangeLogsTestSync // before sync; zero range + rangeLogsTestSynced // after sync; valid blocks range + rangeLogsTestIndexed // individual search range + rangeLogsTestUnindexed // individual search range + rangeLogsTestResults // results range after search iteration + rangeLogsTestReorg // results range trimmed by reorg ) type rangeLogsTestEvent struct { - event int - begin, end uint64 + event int + blocks common.Range[uint64] } // searchSession represents a single search session. type searchSession struct { - ctx context.Context - filter *Filter - mb filtermaps.MatcherBackend - syncRange filtermaps.SyncRange // latest synchronized state with the matcher - firstBlock, lastBlock uint64 // specified search range; each can be MaxUint64 + ctx context.Context + filter *Filter + mb filtermaps.MatcherBackend + syncRange filtermaps.SyncRange // latest synchronized state with the matcher + chainView *filtermaps.ChainView // can be more recent than the indexed view in syncRange + // block ranges always refer to the current chainView + firstBlock, lastBlock uint64 // specified search range; MaxUint64 means latest block searchRange common.Range[uint64] // actual search range; end trimmed to latest head matchRange common.Range[uint64] // range in which we have results (subset of searchRange) matches []*types.Log // valid set of matches in matchRange @@ -182,84 +186,99 @@ func newSearchSession(ctx context.Context, filter *Filter, mb filtermaps.Matcher } // enforce a consistent state before starting the search in order to be able // to determine valid range later - if err := s.syncMatcher(0); err != nil { + var err error + s.syncRange, err = s.mb.SyncLogIndex(s.ctx) + if err != nil { + return nil, err + } + if err := s.updateChainView(); err != nil { return nil, err } return s, nil } -// syncMatcher performs a synchronization step with the matcher. The resulting -// syncRange structure holds information about the latest range of indexed blocks -// and the guaranteed valid blocks whose log index have not been changed since -// the previous synchronization. -// The function also performs trimming of the match set in order to always keep -// it consistent with the synced matcher state. -// Tail trimming is only performed if the first block of the valid log index range -// is higher than trimTailThreshold. This is useful because unindexed log search -// is not affected by the valid tail (on the other hand, valid head is taken into -// account in order to provide reorg safety, even though the log index is not used). -// In case of indexed search the tail is only trimmed if the first part of the -// recently obtained results might be invalid. If guaranteed valid new results -// have been added at the head of previously validated results then there is no -// need to discard those even if the index tail have been unindexed since that. -func (s *searchSession) syncMatcher(trimTailThreshold uint64) error { - if s.filter.rangeLogsTestHook != nil && !s.matchRange.IsEmpty() { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{event: rangeLogsTestSync, begin: s.matchRange.First(), end: s.matchRange.Last()} - } - var err error - s.syncRange, err = s.mb.SyncLogIndex(s.ctx) - if err != nil { - return err +// updateChainView updates to the latest view of the underlying chain and sets +// searchRange by replacing MaxUint64 (meaning latest block) with actual head +// number in the specified search range. +// If the session already had an existing chain view and set of matches then +// it also trims part of the match set that a chain reorg might have invalidated. +func (s *searchSession) updateChainView() error { + // update chain view based on current chain head (might be more recent than + // the indexed view of syncRange as the indexer updates it asynchronously + // with some delay + newChainView := s.filter.sys.backend.CurrentView() + if newChainView == nil { + return errors.New("head block not available") } + head := newChainView.HeadNumber() + // update actual search range based on current head number - first := min(s.firstBlock, s.syncRange.HeadNumber) - last := min(s.lastBlock, s.syncRange.HeadNumber) - s.searchRange = common.NewRange(first, last+1-first) - // discard everything that is not needed or might be invalid - trimRange := s.syncRange.ValidBlocks - if trimRange.First() <= trimTailThreshold { - // everything before this point is already known to be valid; if this is - // valid then keep everything before - trimRange.SetFirst(0) + firstBlock, lastBlock := s.firstBlock, s.lastBlock + if firstBlock == math.MaxUint64 { + firstBlock = head } - trimRange = trimRange.Intersection(s.searchRange) - s.trimMatches(trimRange) - if s.filter.rangeLogsTestHook != nil { - if !s.matchRange.IsEmpty() { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{event: rangeLogsTestTrimmed, begin: s.matchRange.First(), end: s.matchRange.Last()} - } else { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{event: rangeLogsTestTrimmed, begin: 0, end: 0} - } + if lastBlock == math.MaxUint64 { + lastBlock = head } + if firstBlock > lastBlock || lastBlock > head { + return errInvalidBlockRange + } + s.searchRange = common.NewRange(firstBlock, lastBlock+1-firstBlock) + + // Trim existing match set in case a reorg may have invalidated some results + if !s.matchRange.IsEmpty() { + trimRange := newChainView.SharedRange(s.chainView).Intersection(s.searchRange) + s.matchRange, s.matches = s.trimMatches(trimRange, s.matchRange, s.matches) + } + s.chainView = newChainView return nil } -// trimMatches removes any entries from the current set of matches that is outside -// the given range. -func (s *searchSession) trimMatches(trimRange common.Range[uint64]) { - s.matchRange = s.matchRange.Intersection(trimRange) - if s.matchRange.IsEmpty() { - s.matches = nil - return +// trimMatches removes any entries from the specified set of matches that is +// outside the given range. +func (s *searchSession) trimMatches(trimRange, matchRange common.Range[uint64], matches []*types.Log) (common.Range[uint64], []*types.Log) { + newRange := matchRange.Intersection(trimRange) + if newRange == matchRange { + return matchRange, matches } - for len(s.matches) > 0 && s.matches[0].BlockNumber < s.matchRange.First() { - s.matches = s.matches[1:] + if newRange.IsEmpty() { + return newRange, nil } - for len(s.matches) > 0 && s.matches[len(s.matches)-1].BlockNumber > s.matchRange.Last() { - s.matches = s.matches[:len(s.matches)-1] + for len(matches) > 0 && matches[0].BlockNumber < newRange.First() { + matches = matches[1:] } + for len(matches) > 0 && matches[len(matches)-1].BlockNumber > newRange.Last() { + matches = matches[:len(matches)-1] + } + return newRange, matches } // searchInRange performs a single range search, either indexed or unindexed. -func (s *searchSession) searchInRange(r common.Range[uint64], indexed bool) ([]*types.Log, error) { - first, last := r.First(), r.Last() +func (s *searchSession) searchInRange(r common.Range[uint64], indexed bool) (common.Range[uint64], []*types.Log, error) { if indexed { if s.filter.rangeLogsTestHook != nil { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestIndexed, first, last} + s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestIndexed, r} } - results, err := s.filter.indexedLogs(s.ctx, s.mb, first, last) - if err != filtermaps.ErrMatchAll { - return results, err + results, err := s.filter.indexedLogs(s.ctx, s.mb, r.First(), r.Last()) + if err != nil && !errors.Is(err, filtermaps.ErrMatchAll) { + return common.Range[uint64]{}, nil, err + } + if err == nil { + // sync with filtermaps matcher + if s.filter.rangeLogsTestHook != nil { + s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestSync, common.Range[uint64]{}} + } + var syncErr error + if s.syncRange, syncErr = s.mb.SyncLogIndex(s.ctx); syncErr != nil { + return common.Range[uint64]{}, nil, syncErr + } + if s.filter.rangeLogsTestHook != nil { + s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestSynced, s.syncRange.ValidBlocks} + } + // discard everything that might be invalid + trimRange := s.syncRange.ValidBlocks.Intersection(s.chainView.SharedRange(s.syncRange.IndexedView)) + matchRange, matches := s.trimMatches(trimRange, r, results) + return matchRange, matches, nil } // "match all" filters are not supported by filtermaps; fall back to // unindexed search which is the most efficient in this case @@ -267,79 +286,85 @@ func (s *searchSession) searchInRange(r common.Range[uint64], indexed bool) ([]* // fall through to unindexed case } if s.filter.rangeLogsTestHook != nil { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestUnindexed, first, last} + s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestUnindexed, r} } - return s.filter.unindexedLogs(s.ctx, first, last) + matches, err := s.filter.unindexedLogs(s.ctx, s.chainView, r.First(), r.Last()) + if err != nil { + return common.Range[uint64]{}, nil, err + } + return r, matches, nil } // doSearchIteration performs a search on a range missing from an incomplete set // of results, adds the new section and removes invalidated entries. func (s *searchSession) doSearchIteration() error { switch { - case s.syncRange.IndexedBlocks.IsEmpty(): - // indexer is not ready; fallback to completely unindexed search, do not check valid range - var err error - s.matchRange = s.searchRange - s.matches, err = s.searchInRange(s.searchRange, false) - return err - case s.matchRange.IsEmpty(): // no results yet; try search in entire range indexedSearchRange := s.searchRange.Intersection(s.syncRange.IndexedBlocks) - var err error if s.forceUnindexed = indexedSearchRange.IsEmpty(); !s.forceUnindexed { // indexed search on the intersection of indexed and searched range - s.matchRange = indexedSearchRange - s.matches, err = s.searchInRange(indexedSearchRange, true) + matchRange, matches, err := s.searchInRange(indexedSearchRange, true) if err != nil { return err } - return s.syncMatcher(0) // trim everything that the matcher considers potentially invalid + s.matchRange = matchRange + s.matches = matches + return nil } else { - // no intersection of indexed and searched range; unindexed search on the whole searched range - s.matchRange = s.searchRange - s.matches, err = s.searchInRange(s.searchRange, false) + // no intersection of indexed and searched range; unindexed search on + // the whole searched range + matchRange, matches, err := s.searchInRange(s.searchRange, false) if err != nil { return err } - return s.syncMatcher(math.MaxUint64) // unindexed search is not affected by the tail of valid range + s.matchRange = matchRange + s.matches = matches + return nil } case !s.matchRange.IsEmpty() && s.matchRange.First() > s.searchRange.First(): - // we have results but tail section is missing; do unindexed search for - // the tail part but still allow indexed search for missing head section + // Results are available, but the tail section is missing. Perform an unindexed + // search for the missing tail, while still allowing indexed search for the head. + // + // The unindexed search is necessary because the tail portion of the indexes + // has been pruned. tailRange := common.NewRange(s.searchRange.First(), s.matchRange.First()-s.searchRange.First()) - tailMatches, err := s.searchInRange(tailRange, false) + _, tailMatches, err := s.searchInRange(tailRange, false) if err != nil { return err } s.matches = append(tailMatches, s.matches...) s.matchRange = tailRange.Union(s.matchRange) - return s.syncMatcher(math.MaxUint64) // unindexed search is not affected by the tail of valid range + return nil case !s.matchRange.IsEmpty() && s.matchRange.First() == s.searchRange.First() && s.searchRange.AfterLast() > s.matchRange.AfterLast(): - // we have results but head section is missing + // Results are available, but the head section is missing. Try to perform + // the indexed search for the missing head, or fallback to unindexed search + // if the tail portion of indexed range has been pruned. headRange := common.NewRange(s.matchRange.AfterLast(), s.searchRange.AfterLast()-s.matchRange.AfterLast()) if !s.forceUnindexed { indexedHeadRange := headRange.Intersection(s.syncRange.IndexedBlocks) if !indexedHeadRange.IsEmpty() && indexedHeadRange.First() == headRange.First() { - // indexed head range search is possible headRange = indexedHeadRange } else { + // The tail portion of the indexes has been pruned, falling back + // to unindexed search. s.forceUnindexed = true } } - headMatches, err := s.searchInRange(headRange, !s.forceUnindexed) + headMatchRange, headMatches, err := s.searchInRange(headRange, !s.forceUnindexed) if err != nil { return err } - s.matches = append(s.matches, headMatches...) - s.matchRange = s.matchRange.Union(headRange) - if s.forceUnindexed { - return s.syncMatcher(math.MaxUint64) // unindexed search is not affected by the tail of valid range - } else { - return s.syncMatcher(headRange.First()) // trim if the tail of latest head search results might be invalid + if headMatchRange.First() != s.matchRange.AfterLast() { + // improbable corner case, first part of new head range invalidated by tail unindexing + s.matches, s.matchRange = headMatches, headMatchRange + return nil } + s.matches = append(s.matches, headMatches...) + s.matchRange = s.matchRange.Union(headMatchRange) + return nil default: panic("invalid search session state") @@ -349,7 +374,7 @@ func (s *searchSession) doSearchIteration() error { func (f *Filter) rangeLogs(ctx context.Context, firstBlock, lastBlock uint64) ([]*types.Log, error) { if f.rangeLogsTestHook != nil { defer func() { - f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestDone, 0, 0} + f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestDone, common.Range[uint64]{}} close(f.rangeLogsTestHook) }() } @@ -366,7 +391,17 @@ func (f *Filter) rangeLogs(ctx context.Context, firstBlock, lastBlock uint64) ([ } for session.searchRange != session.matchRange { if err := session.doSearchIteration(); err != nil { - return session.matches, err + return nil, err + } + if f.rangeLogsTestHook != nil { + f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestResults, session.matchRange} + } + mr := session.matchRange + if err := session.updateChainView(); err != nil { + return nil, err + } + if f.rangeLogsTestHook != nil && session.matchRange != mr { + f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestReorg, session.matchRange} } } return session.matches, nil @@ -382,7 +417,7 @@ func (f *Filter) indexedLogs(ctx context.Context, mb filtermaps.MatcherBackend, // unindexedLogs returns the logs matching the filter criteria based on raw block // iteration and bloom matching. -func (f *Filter) unindexedLogs(ctx context.Context, begin, end uint64) ([]*types.Log, error) { +func (f *Filter) unindexedLogs(ctx context.Context, chainView *filtermaps.ChainView, begin, end uint64) ([]*types.Log, error) { start := time.Now() log.Debug("Performing unindexed log search", "begin", begin, "end", end) var matches []*types.Log @@ -392,9 +427,14 @@ func (f *Filter) unindexedLogs(ctx context.Context, begin, end uint64) ([]*types return matches, ctx.Err() default: } - header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber)) - if header == nil || err != nil { - return matches, err + if blockNumber > chainView.HeadNumber() { + // check here so that we can return matches up until head along with + // the error + return matches, errInvalidBlockRange + } + header := chainView.Header(blockNumber) + if header == nil { + return matches, errors.New("header not found") } found, err := f.blockLogs(ctx, header) if err != nil { diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index aca3c03ad6..10e433f09b 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -30,8 +30,8 @@ import ( "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/filtermaps" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" @@ -71,6 +71,7 @@ type Backend interface { SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + CurrentView() *filtermaps.ChainView NewMatcherBackend() filtermaps.MatcherBackend } @@ -311,7 +312,7 @@ func (es *EventSystem) SubscribeLogs(crit ethereum.FilterQuery, logs chan []*typ } // Queries beyond the pruning cutoff are not supported. if uint64(from) < es.backend.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } // only interested in new mined logs diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 3bb019d105..fa5d4fe897 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -154,6 +154,11 @@ func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc return b.chainFeed.Subscribe(ch) } +func (b *testBackend) CurrentView() *filtermaps.ChainView { + head := b.CurrentBlock() + return filtermaps.NewChainView(b, head.Number.Uint64(), head.Hash()) +} + func (b *testBackend) NewMatcherBackend() filtermaps.MatcherBackend { return b.fm.NewMatcherBackend() } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 4026c03e89..d6065230f8 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -453,7 +453,8 @@ func TestRangeLogs(t *testing.T) { addresses = []common.Address{{}} ) - expEvent := func(exp rangeLogsTestEvent) { + expEvent := func(expEvent int, expFirst, expAfterLast uint64) { + exp := rangeLogsTestEvent{expEvent, common.NewRange[uint64](expFirst, expAfterLast-expFirst)} event++ ev := <-filter.rangeLogsTestHook if ev != exp { @@ -472,7 +473,6 @@ func TestRangeLogs(t *testing.T) { for range filter.rangeLogsTestHook { } }(filter) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 0, 0}) } updateHead := func() { @@ -483,81 +483,122 @@ func TestRangeLogs(t *testing.T) { // test case #1 newFilter(300, 500) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 401, 500}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 401, 500}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 401, 500}) - expEvent(rangeLogsTestEvent{rangeLogsTestUnindexed, 300, 400}) + expEvent(rangeLogsTestIndexed, 401, 501) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 401, 601) + expEvent(rangeLogsTestResults, 401, 501) + expEvent(rangeLogsTestUnindexed, 300, 401) if _, err := bc.InsertChain(chain[600:700]); err != nil { t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 300, 500}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 300, 500}) // unindexed search is not affected by trimmed tail - expEvent(rangeLogsTestEvent{rangeLogsTestDone, 0, 0}) + expEvent(rangeLogsTestResults, 300, 501) + expEvent(rangeLogsTestDone, 0, 0) // test case #2 newFilter(400, int64(rpc.LatestBlockNumber)) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 501, 700}) + expEvent(rangeLogsTestIndexed, 501, 701) if _, err := bc.InsertChain(chain[700:800]); err != nil { t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 501, 700}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 601, 698}) - expEvent(rangeLogsTestEvent{rangeLogsTestUnindexed, 400, 600}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 400, 698}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 400, 698}) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 699, 800}) - if err := bc.SetHead(750); err != nil { + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 601, 699) + expEvent(rangeLogsTestResults, 601, 699) + expEvent(rangeLogsTestUnindexed, 400, 601) + expEvent(rangeLogsTestResults, 400, 699) + expEvent(rangeLogsTestIndexed, 699, 801) + if _, err := bc.SetCanonical(chain[749]); err != nil { // set head to block 750 t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 400, 800}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 400, 748}) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 749, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 400, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 400, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestDone, 0, 0}) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 601, 749) + expEvent(rangeLogsTestResults, 400, 749) + expEvent(rangeLogsTestIndexed, 749, 751) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 751) + expEvent(rangeLogsTestResults, 400, 751) + expEvent(rangeLogsTestDone, 0, 0) // test case #3 newFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber)) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 750, 750}) - if err := bc.SetHead(740); err != nil { + expEvent(rangeLogsTestIndexed, 750, 751) + if _, err := bc.SetCanonical(chain[739]); err != nil { t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 750, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 0, 0}) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 740, 740}) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 739) + expEvent(rangeLogsTestResults, 0, 0) + expEvent(rangeLogsTestIndexed, 740, 741) if _, err := bc.InsertChain(chain[740:750]); err != nil { t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 740, 740}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 0, 0}) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 750, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 750, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 750, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestDone, 0, 0}) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 739) + expEvent(rangeLogsTestResults, 0, 0) + expEvent(rangeLogsTestIndexed, 750, 751) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 751) + expEvent(rangeLogsTestResults, 750, 751) + expEvent(rangeLogsTestDone, 0, 0) // test case #4 + if _, err := bc.SetCanonical(chain[499]); err != nil { + t.Fatal(err) + } + updateHead() newFilter(400, int64(rpc.LatestBlockNumber)) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 551, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 551, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 551, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestUnindexed, 400, 550}) + expEvent(rangeLogsTestIndexed, 400, 501) + if _, err := bc.InsertChain(chain[500:650]); err != nil { + t.Fatal(err) + } + updateHead() + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 451, 499) + expEvent(rangeLogsTestResults, 451, 499) + expEvent(rangeLogsTestUnindexed, 400, 451) + expEvent(rangeLogsTestResults, 400, 499) + // indexed head extension seems possible + expEvent(rangeLogsTestIndexed, 499, 651) + // further head extension causes tail unindexing in searched range + if _, err := bc.InsertChain(chain[650:750]); err != nil { + t.Fatal(err) + } + updateHead() + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 649) + // tail trimmed to 551; cannot merge with existing results + expEvent(rangeLogsTestResults, 551, 649) + expEvent(rangeLogsTestUnindexed, 400, 551) + expEvent(rangeLogsTestResults, 400, 649) + expEvent(rangeLogsTestIndexed, 649, 751) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 751) + expEvent(rangeLogsTestResults, 400, 751) + expEvent(rangeLogsTestDone, 0, 0) + + // test case #5 + newFilter(400, int64(rpc.LatestBlockNumber)) + expEvent(rangeLogsTestIndexed, 551, 751) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 751) + expEvent(rangeLogsTestResults, 551, 751) + expEvent(rangeLogsTestUnindexed, 400, 551) if _, err := bc.InsertChain(chain[750:1000]); err != nil { t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 400, 750}) - // indexed range affected by tail pruning so we have to discard the entire - // match set - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 0, 0}) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 801, 1000}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 801, 1000}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 801, 1000}) - expEvent(rangeLogsTestEvent{rangeLogsTestUnindexed, 400, 800}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 400, 1000}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 400, 1000}) + expEvent(rangeLogsTestResults, 400, 751) + // indexed tail already beyond results head; revert to unindexed head search + expEvent(rangeLogsTestUnindexed, 751, 1001) + if _, err := bc.SetCanonical(chain[899]); err != nil { + t.Fatal(err) + } + updateHead() + expEvent(rangeLogsTestResults, 400, 1001) + expEvent(rangeLogsTestReorg, 400, 901) + expEvent(rangeLogsTestDone, 0, 0) } diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 59830e9fe8..e5a197f640 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -108,7 +108,9 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { bf.results.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit) if blobGasUsed := bf.header.BlobGasUsed; blobGasUsed != nil { maxBlobGas := eip4844.MaxBlobGasPerBlock(config, bf.header.Time) - bf.results.blobGasUsedRatio = float64(*blobGasUsed) / float64(maxBlobGas) + if maxBlobGas != 0 { + bf.results.blobGasUsedRatio = float64(*blobGasUsed) / float64(maxBlobGas) + } } if len(percentiles) == 0 { diff --git a/eth/handler.go b/eth/handler.go index b2ad6effdb..8283d7d02f 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -484,7 +484,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { total := new(big.Int).Exp(direct, big.NewInt(2), nil) // Stabilise total peer count a bit based on sqrt peers var ( - signer = types.LatestSignerForChainID(h.chain.Config().ChainID) // Don't care about chain status, we just need *a* sender + signer = types.LatestSigner(h.chain.Config()) // Don't care about chain status, we just need *a* sender hasher = crypto.NewKeccakState() hash = make([]byte, 32) ) diff --git a/ethclient/simulated/backend_test.go b/ethclient/simulated/backend_test.go index fc78e84362..303e480a09 100644 --- a/ethclient/simulated/backend_test.go +++ b/ethclient/simulated/backend_test.go @@ -25,16 +25,14 @@ import ( "testing" "time" - "go.uber.org/goleak" - - "github.com/ethereum/go-ethereum/crypto/kzg4844" - "github.com/holiman/uint256" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/params" + "github.com/holiman/uint256" + "go.uber.org/goleak" ) var _ bind.ContractBackend = (Client)(nil) diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 3fbf32e22e..aff051937d 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -619,6 +619,9 @@ func (b testBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) func (b testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { panic("implement me") } +func (b testBackend) CurrentView() *filtermaps.ChainView { + panic("implement me") +} func (b testBackend) NewMatcherBackend() filtermaps.MatcherBackend { panic("implement me") } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index c4bf2e0591..e28cb93296 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -95,6 +95,7 @@ type Backend interface { SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + CurrentView() *filtermaps.ChainView NewMatcherBackend() filtermaps.MatcherBackend } diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index ba346b132f..9241b509da 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -314,9 +314,13 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, return nil, nil, err } // EIP-7002 - core.ProcessWithdrawalQueue(&requests, evm) + if err := core.ProcessWithdrawalQueue(&requests, evm); err != nil { + return nil, nil, err + } // EIP-7251 - core.ProcessConsolidationQueue(&requests, evm) + if err := core.ProcessConsolidationQueue(&requests, evm); err != nil { + return nil, nil, err + } } if requests != nil { reqHash := types.CalcRequestsHash(requests) diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index b4d11a3033..9dd6a54729 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -401,6 +401,7 @@ func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) func (b *backendMock) Engine() consensus.Engine { return nil } +func (b *backendMock) CurrentView() *filtermaps.ChainView { return nil } func (b *backendMock) NewMatcherBackend() filtermaps.MatcherBackend { return nil } func (b *backendMock) HistoryPruningCutoff() uint64 { return 0 } diff --git a/miner/worker.go b/miner/worker.go index 8fb42e31bc..d80cb8913b 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -127,9 +127,13 @@ func (miner *Miner) generateWork(params *generateParams, witness bool) *newPaylo return &newPayloadResult{err: err} } // EIP-7002 - core.ProcessWithdrawalQueue(&requests, work.evm) + if err := core.ProcessWithdrawalQueue(&requests, work.evm); err != nil { + return &newPayloadResult{err: err} + } // EIP-7251 consolidations - core.ProcessConsolidationQueue(&requests, work.evm) + if err := core.ProcessConsolidationQueue(&requests, work.evm); err != nil { + return &newPayloadResult{err: err} + } } if requests != nil { reqHash := types.CalcRequestsHash(requests) diff --git a/p2p/metrics.go b/p2p/metrics.go index 1fd0f26db3..29c2acb0cb 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -49,7 +49,11 @@ var ( serveSuccessMeter = metrics.NewRegisteredMeter("p2p/serves/success", nil) dialMeter = metrics.NewRegisteredMeter("p2p/dials", nil) dialSuccessMeter = metrics.NewRegisteredMeter("p2p/dials/success", nil) - dialConnectionError = metrics.NewRegisteredMeter("p2p/dials/error/connection", nil) + dialConnectionError = metrics.NewRegisteredMeter("p2p/dials/error/connection", nil) // dial timeout; no route to host; connection refused; network is unreachable + + // count peers that stayed connected for at least 1 min + serve1MinSuccessMeter = metrics.NewRegisteredMeter("p2p/serves/success/1min", nil) + dial1MinSuccessMeter = metrics.NewRegisteredMeter("p2p/dials/success/1min", nil) // handshake error meters dialTooManyPeers = metrics.NewRegisteredMeter("p2p/dials/error/saturated", nil) @@ -57,34 +61,41 @@ var ( dialSelf = metrics.NewRegisteredMeter("p2p/dials/error/self", nil) dialUselessPeer = metrics.NewRegisteredMeter("p2p/dials/error/useless", nil) dialUnexpectedIdentity = metrics.NewRegisteredMeter("p2p/dials/error/id/unexpected", nil) - dialEncHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/enc", nil) - dialProtoHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/proto", nil) + dialEncHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/enc", nil) // EOF; connection reset during handshake; message too big; i/o timeout + dialProtoHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/proto", nil) // EOF + + // capture the rest of errors that are not handled by the above meters + dialOtherError = metrics.NewRegisteredMeter("p2p/dials/error/other", nil) ) -// markDialError matches errors that occur while setting up a dial connection -// to the corresponding meter. +// markDialError matches errors that occur while setting up a dial connection to the +// corresponding meter. We don't maintain meters for evert possible error, just for +// the most interesting ones. func markDialError(err error) { if !metrics.Enabled() { return } - if err2 := errors.Unwrap(err); err2 != nil { - err = err2 - } - switch err { - case DiscTooManyPeers: + + var reason DiscReason + var handshakeErr *protoHandshakeError + d := errors.As(err, &reason) + switch { + case d && reason == DiscTooManyPeers: dialTooManyPeers.Mark(1) - case DiscAlreadyConnected: + case d && reason == DiscAlreadyConnected: dialAlreadyConnected.Mark(1) - case DiscSelf: + case d && reason == DiscSelf: dialSelf.Mark(1) - case DiscUselessPeer: + case d && reason == DiscUselessPeer: dialUselessPeer.Mark(1) - case DiscUnexpectedIdentity: + case d && reason == DiscUnexpectedIdentity: dialUnexpectedIdentity.Mark(1) - case errEncHandshakeError: - dialEncHandshakeError.Mark(1) - case errProtoHandshakeError: + case errors.As(err, &handshakeErr): dialProtoHandshakeError.Mark(1) + case errors.Is(err, errEncHandshakeError): + dialEncHandshakeError.Mark(1) + default: + dialOtherError.Mark(1) } } diff --git a/p2p/peer.go b/p2p/peer.go index a01df63d0c..9a0a750ac8 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -220,11 +220,35 @@ func (p *Peer) String() string { return fmt.Sprintf("Peer %x %v", id[:8], p.RemoteAddr()) } -// Inbound returns true if the peer is an inbound connection +// Inbound returns true if the peer is an inbound (not dialed) connection. func (p *Peer) Inbound() bool { return p.rw.is(inboundConn) } +// Trusted returns true if the peer is configured as trusted. +// Trusted peers are accepted in above the MaxInboundConns limit. +// The peer can be either inbound or dialed. +func (p *Peer) Trusted() bool { + return p.rw.is(trustedConn) +} + +// DynDialed returns true if the peer was dialed successfully (passed handshake) and +// it is not configured as static. +func (p *Peer) DynDialed() bool { + return p.rw.is(dynDialedConn) +} + +// StaticDialed returns true if the peer was dialed successfully (passed handshake) and +// it is configured as static. +func (p *Peer) StaticDialed() bool { + return p.rw.is(staticDialedConn) +} + +// Lifetime returns the time since peer creation. +func (p *Peer) Lifetime() mclock.AbsTime { + return mclock.Now() - p.created +} + func newPeer(log log.Logger, conn *conn, protocols []Protocol) *Peer { protomap := matchProtocols(protocols, conn.caps, conn) p := &Peer{ @@ -254,6 +278,8 @@ func (p *Peer) run() (remoteRequested bool, err error) { p.wg.Add(2) go p.readLoop(readErr) go p.pingLoop() + live1min := time.NewTimer(1 * time.Minute) + defer live1min.Stop() // Start all protocol handlers. writeStart <- struct{}{} @@ -285,6 +311,12 @@ loop: case err = <-p.disc: reason = discReasonForError(err) break loop + case <-live1min.C: + if p.Inbound() { + serve1MinSuccessMeter.Mark(1) + } else { + dial1MinSuccessMeter.Mark(1) + } } } diff --git a/p2p/server.go b/p2p/server.go index c1564352e5..d9105976dd 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -66,11 +66,15 @@ const ( ) var ( - errServerStopped = errors.New("server stopped") - errEncHandshakeError = errors.New("rlpx enc error") - errProtoHandshakeError = errors.New("rlpx proto error") + errServerStopped = errors.New("server stopped") + errEncHandshakeError = errors.New("rlpx enc error") ) +type protoHandshakeError struct{ err error } + +func (e *protoHandshakeError) Error() string { return fmt.Sprintf("rlpx proto error: %v", e.err) } +func (e *protoHandshakeError) Unwrap() error { return e.err } + // Server manages all peer connections. type Server struct { // Config fields may not be modified while the server is running. @@ -508,7 +512,7 @@ func (srv *Server) setupDiscovery() error { func (srv *Server) setupDialScheduler() { config := dialConfig{ self: srv.localnode.ID(), - maxDialPeers: srv.maxDialedConns(), + maxDialPeers: srv.MaxDialedConns(), maxActiveDials: srv.MaxPendingPeers, log: srv.Logger, netRestrict: srv.NetRestrict, @@ -527,11 +531,11 @@ func (srv *Server) setupDialScheduler() { } } -func (srv *Server) maxInboundConns() int { - return srv.MaxPeers - srv.maxDialedConns() +func (srv *Server) MaxInboundConns() int { + return srv.MaxPeers - srv.MaxDialedConns() } -func (srv *Server) maxDialedConns() (limit int) { +func (srv *Server) MaxDialedConns() (limit int) { if srv.NoDial || srv.MaxPeers == 0 { return 0 } @@ -736,7 +740,7 @@ func (srv *Server) postHandshakeChecks(peers map[enode.ID]*Peer, inboundCount in switch { case !c.is(trustedConn) && len(peers) >= srv.MaxPeers: return DiscTooManyPeers - case !c.is(trustedConn) && c.is(inboundConn) && inboundCount >= srv.maxInboundConns(): + case !c.is(trustedConn) && c.is(inboundConn) && inboundCount >= srv.MaxInboundConns(): return DiscTooManyPeers case peers[c.node.ID()] != nil: return DiscAlreadyConnected @@ -907,7 +911,7 @@ func (srv *Server) setupConn(c *conn, dialDest *enode.Node) error { phs, err := c.doProtoHandshake(srv.ourHandshake) if err != nil { clog.Trace("Failed p2p handshake", "err", err) - return fmt.Errorf("%w: %v", errProtoHandshakeError, err) + return &protoHandshakeError{err: err} } if id := c.node.ID(); !bytes.Equal(crypto.Keccak256(phs.ID), id[:]) { clog.Trace("Wrong devp2p handshake identity", "phsid", hex.EncodeToString(phs.ID)) diff --git a/p2p/server_test.go b/p2p/server_test.go index a0491e984a..d42926cf4c 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -410,11 +410,11 @@ func TestServerSetupConn(t *testing.T) { wantCloseErr: DiscUnexpectedIdentity, }, { - tt: &setupTransport{pubkey: clientpub, protoHandshakeErr: errProtoHandshakeError}, + tt: &setupTransport{pubkey: clientpub, protoHandshakeErr: DiscTooManyPeers}, dialDest: enode.NewV4(clientpub, nil, 0, 0), flags: dynDialedConn, wantCalls: "doEncHandshake,doProtoHandshake,close,", - wantCloseErr: errProtoHandshakeError, + wantCloseErr: DiscTooManyPeers, }, { tt: &setupTransport{pubkey: srvpub, phs: protoHandshake{ID: crypto.FromECDSAPub(srvpub)[1:]}}, diff --git a/params/config.go b/params/config.go index 8f9e02583b..67aa6b2225 100644 --- a/params/config.go +++ b/params/config.go @@ -60,10 +60,12 @@ var ( TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 ShanghaiTime: newUint64(1681338455), CancunTime: newUint64(1710338135), + PragueTime: newUint64(1746612311), DepositContractAddress: common.HexToAddress("0x00000000219ab540356cbb839cbe05303d7705fa"), Ethash: new(EthashConfig), BlobScheduleConfig: &BlobScheduleConfig{ Cancun: DefaultCancunBlobConfig, + Prague: DefaultPragueBlobConfig, }, } // HoleskyChainConfig contains the chain parameters to run a node on the Holesky test network. @@ -907,6 +909,23 @@ func (c *ChainConfig) LatestFork(time uint64) forks.Fork { } } +// Timestamp returns the timestamp associated with the fork or returns nil if +// the fork isn't defined or isn't a time-based fork. +func (c *ChainConfig) Timestamp(fork forks.Fork) *uint64 { + switch { + case fork == forks.Osaka: + return c.OsakaTime + case fork == forks.Prague: + return c.PragueTime + case fork == forks.Cancun: + return c.CancunTime + case fork == forks.Shanghai: + return c.ShanghaiTime + default: + return nil + } +} + // isForkBlockIncompatible returns true if a fork scheduled at block s1 cannot be // rescheduled to block s2 because head is already past the fork. func isForkBlockIncompatible(s1, s2, head *big.Int) bool { diff --git a/params/forks/forks.go b/params/forks/forks.go index 02f6e5b612..5c9612a625 100644 --- a/params/forks/forks.go +++ b/params/forks/forks.go @@ -20,7 +20,7 @@ package forks type Fork int const ( - Frontier = iota + Frontier Fork = iota FrontierThawing Homestead DAO @@ -41,3 +41,35 @@ const ( Prague Osaka ) + +// String implements fmt.Stringer. +func (f Fork) String() string { + s, ok := forkToString[f] + if !ok { + return "Unknown fork" + } + return s +} + +var forkToString = map[Fork]string{ + Frontier: "Frontier", + FrontierThawing: "Frontier Thawing", + Homestead: "Homestead", + DAO: "DAO", + TangerineWhistle: "Tangerine Whistle", + SpuriousDragon: "Spurious Dragon", + Byzantium: "Byzantium", + Constantinople: "Constantinople", + Petersburg: "Petersburg", + Istanbul: "Istanbul", + MuirGlacier: "Muir Glacier", + Berlin: "Berlin", + London: "London", + ArrowGlacier: "Arrow Glacier", + GrayGlacier: "Gray Glacier", + Paris: "Paris", + Shanghai: "Shanghai", + Cancun: "Cancun", + Prague: "Prague", + Osaka: "Osaka", +} diff --git a/version/version.go b/version/version.go index 945b3b58a2..53a1bae280 100644 --- a/version/version.go +++ b/version/version.go @@ -19,6 +19,6 @@ package version const ( Major = 1 // Major version component of the current release Minor = 15 // Minor version component of the current release - Patch = 8 // Patch version component of the current release + Patch = 9 // Patch version component of the current release Meta = "stable" // Version metadata to append to the version string )