diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b569180852..681b5e7c66 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,12 +9,11 @@ beacon/light/ @zsfelfoldi beacon/merkle/ @zsfelfoldi beacon/types/ @zsfelfoldi @fjl beacon/params/ @zsfelfoldi @fjl -cmd/clef/ @holiman -cmd/evm/ @holiman @MariusVanDerWijden @lightclient -core/state/ @rjl493456442 @holiman -crypto/ @gballet @jwasinger @holiman @fjl -core/ @holiman @rjl493456442 -eth/ @holiman @rjl493456442 +cmd/evm/ @MariusVanDerWijden @lightclient +core/state/ @rjl493456442 +crypto/ @gballet @jwasinger @fjl +core/ @rjl493456442 +eth/ @rjl493456442 eth/catalyst/ @MariusVanDerWijden @lightclient @fjl @jwasinger eth/tracers/ @s1na ethclient/ @fjl @@ -26,11 +25,9 @@ core/tracing/ @s1na graphql/ @s1na internal/ethapi/ @fjl @s1na @lightclient internal/era/ @lightclient -metrics/ @holiman -miner/ @MariusVanDerWijden @holiman @fjl @rjl493456442 +miner/ @MariusVanDerWijden @fjl @rjl493456442 node/ @fjl p2p/ @fjl @zsfelfoldi rlp/ @fjl -params/ @fjl @holiman @karalabe @gballet @rjl493456442 @zsfelfoldi -rpc/ @fjl @holiman -signer/ @holiman +params/ @fjl @karalabe @gballet @rjl493456442 @zsfelfoldi +rpc/ @fjl diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 2279509542..c57a9a947d 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -31,11 +31,11 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/era" @@ -625,7 +625,7 @@ func pruneHistory(ctx *cli.Context) error { defer chain.Stop() // Determine the prune point. This will be the first PoS block. - prunePoint, ok := ethconfig.HistoryPrunePoints[chain.Genesis().Hash()] + prunePoint, ok := history.PrunePoints[chain.Genesis().Hash()] if !ok || prunePoint == nil { return errors.New("prune point not found") } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index ab46e059f3..289030ae65 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -100,7 +100,6 @@ var ( utils.LightNoSyncServeFlag, // deprecated utils.EthRequiredBlocksFlag, utils.LegacyWhitelistFlag, // deprecated - utils.BloomFilterSizeFlag, utils.CacheFlag, utils.CacheDatabaseFlag, utils.CacheTrieFlag, diff --git a/cmd/workload/testsuite.go b/cmd/workload/testsuite.go index e7019e2055..e8e25e7731 100644 --- a/cmd/workload/testsuite.go +++ b/cmd/workload/testsuite.go @@ -23,7 +23,7 @@ import ( "os" "slices" - "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/log" @@ -124,13 +124,13 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { cfg.filterQueryFile = "queries/filter_queries_mainnet.json" cfg.historyTestFile = "queries/history_mainnet.json" cfg.historyPruneBlock = new(uint64) - *cfg.historyPruneBlock = ethconfig.HistoryPrunePoints[params.MainnetGenesisHash].BlockNumber + *cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber case ctx.Bool(testSepoliaFlag.Name): cfg.fsys = builtinTestFiles cfg.filterQueryFile = "queries/filter_queries_sepolia.json" cfg.historyTestFile = "queries/history_sepolia.json" cfg.historyPruneBlock = new(uint64) - *cfg.historyPruneBlock = ethconfig.HistoryPrunePoints[params.SepoliaGenesisHash].BlockNumber + *cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber default: cfg.fsys = os.DirFS(".") cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name) diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 8af4057693..5217979236 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -50,8 +50,11 @@ func testHeaderVerification(t *testing.T, scheme string) { headers[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) defer chain.Stop() + if err != nil { + t.Fatal(err) + } for i := 0; i < len(blocks); i++ { for j, valid := range []bool{true, false} { @@ -163,8 +166,11 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { postHeaders[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil) defer chain.Stop() + if err != nil { + t.Fatal(err) + } // Verify the blocks before the merging for i := 0; i < len(preBlocks); i++ { diff --git a/core/blockchain.go b/core/blockchain.go index d56996dadb..6667f64911 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "math" "math/big" "runtime" "slices" @@ -36,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -99,6 +101,10 @@ var ( errInvalidNewChain = errors.New("invalid new chain") ) +var ( + forkReadyInterval = 3 * time.Minute +) + const ( bodyCacheLimit = 256 blockCacheLimit = 256 @@ -158,8 +164,7 @@ type CacheConfig struct { // This defines the cutoff block for history expiry. // Blocks before this number may be unavailable in the chain database. - HistoryPruningCutoffNumber uint64 - HistoryPruningCutoffHash common.Hash + ChainHistoryMode history.HistoryMode } // triedbConfig derives the configures for trie database. @@ -255,6 +260,7 @@ type BlockChain struct { currentSnapBlock atomic.Pointer[types.Header] // Current head of snap-sync currentFinalBlock atomic.Pointer[types.Header] // Latest (consensus) finalized block currentSafeBlock atomic.Pointer[types.Header] // Latest (consensus) safe block + historyPrunePoint atomic.Pointer[history.PrunePoint] bodyCache *lru.Cache[common.Hash, *types.Body] bodyRLPCache *lru.Cache[common.Hash, rlp.RawValue] @@ -274,6 +280,8 @@ type BlockChain struct { processor Processor // Block transaction processor interface vmConfig vm.Config logger *tracing.Hooks + + lastForkReadyAlert time.Time // Last time there was a fork readiness print out } // NewBlockChain returns a fully initialised block chain using information @@ -513,19 +521,33 @@ func (bc *BlockChain) loadLastState() error { log.Warn("Empty database, resetting chain") return bc.Reset() } - // Make sure the entire head block is available - headBlock := bc.GetBlockByHash(head) + headHeader := bc.GetHeaderByHash(head) + if headHeader == nil { + // Corrupt or empty database, init from scratch + log.Warn("Head header missing, resetting chain", "hash", head) + return bc.Reset() + } + + var headBlock *types.Block + if cmp := headHeader.Number.Cmp(new(big.Int)); cmp == 1 { + // Make sure the entire head block is available. + headBlock = bc.GetBlockByHash(head) + } else if cmp == 0 { + // On a pruned node the block body might not be available. But a pruned + // block should never be the head block. The only exception is when, as + // a last resort, chain is reset to genesis. + headBlock = bc.genesisBlock + } if headBlock == nil { // Corrupt or empty database, init from scratch log.Warn("Head block missing, resetting chain", "hash", head) return bc.Reset() } // Everything seems to be fine, set as the head block - bc.currentBlock.Store(headBlock.Header()) + bc.currentBlock.Store(headHeader) headBlockGauge.Update(int64(headBlock.NumberU64())) // Restore the last known head header - headHeader := headBlock.Header() if head := rawdb.ReadHeadHeaderHash(bc.db); head != (common.Hash{}) { if header := bc.GetHeaderByHash(head); header != nil { headHeader = header @@ -533,6 +555,12 @@ func (bc *BlockChain) loadLastState() error { } bc.hc.SetCurrentHeader(headHeader) + // Initialize history pruning. + latest := max(headBlock.NumberU64(), headHeader.Number.Uint64()) + if err := bc.initializeHistoryPruning(latest); err != nil { + return err + } + // Restore the last known head snap block bc.currentSnapBlock.Store(headBlock.Header()) headFastBlockGauge.Update(int64(headBlock.NumberU64())) @@ -555,6 +583,7 @@ func (bc *BlockChain) loadLastState() error { headSafeBlockGauge.Update(int64(block.NumberU64())) } } + // Issue a status log for the user var ( currentSnapBlock = bc.CurrentSnapBlock() @@ -573,9 +602,57 @@ func (bc *BlockChain) loadLastState() error { if pivot := rawdb.ReadLastPivotNumber(bc.db); pivot != nil { log.Info("Loaded last snap-sync pivot marker", "number", *pivot) } + if pruning := bc.historyPrunePoint.Load(); pruning != nil { + log.Info("Chain history is pruned", "earliest", pruning.BlockNumber, "hash", pruning.BlockHash) + } return nil } +// initializeHistoryPruning sets bc.historyPrunePoint. +func (bc *BlockChain) initializeHistoryPruning(latest uint64) error { + freezerTail, _ := bc.db.Tail() + + switch bc.cacheConfig.ChainHistoryMode { + case history.KeepAll: + if freezerTail == 0 { + return nil + } + // The database was pruned somehow, so we need to figure out if it's a known + // configuration or an error. + predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] + if predefinedPoint == nil || freezerTail != predefinedPoint.BlockNumber { + log.Error("Chain history database is pruned with unknown configuration", "tail", freezerTail) + return fmt.Errorf("unexpected database tail") + } + bc.historyPrunePoint.Store(predefinedPoint) + return nil + + case history.KeepPostMerge: + if freezerTail == 0 && latest != 0 { + // This is the case where a user is trying to run with --history.chain + // postmerge directly on an existing DB. We could just trigger the pruning + // here, but it'd be a bit dangerous since they may not have intended this + // action to happen. So just tell them how to do it. + log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cacheConfig.ChainHistoryMode.String())) + log.Error(fmt.Sprintf("Run 'geth prune-history' to prune pre-merge history.")) + return fmt.Errorf("history pruning requested via configuration") + } + predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] + if predefinedPoint == nil { + log.Error("Chain history pruning is not supported for this network", "genesis", bc.genesisBlock.Hash()) + return fmt.Errorf("history pruning requested for unknown network") + } else if freezerTail > 0 && freezerTail != predefinedPoint.BlockNumber { + log.Error("Chain history database is pruned to unknown block", "tail", freezerTail) + return fmt.Errorf("unexpected database tail") + } + bc.historyPrunePoint.Store(predefinedPoint) + return nil + + default: + return fmt.Errorf("invalid history mode: %d", bc.cacheConfig.ChainHistoryMode) + } +} + // SetHead rewinds the local chain to a new head. Depending on whether the node // was snap synced or full synced and in which state, the method will try to // delete minimal data from disk whilst retaining chain consistency. @@ -586,11 +663,15 @@ func (bc *BlockChain) SetHead(head uint64) error { // Send chain head event to update the transaction pool header := bc.CurrentBlock() if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil { - // This should never happen. In practice, previously currentBlock - // contained the entire block whereas now only a "marker", so there - // is an ever so slight chance for a race we should handle. - log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) - return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + // In a pruned node the genesis block will not exist in the freezer. + // It should not happen that we set head to any other pruned block. + if header.Number.Uint64() > 0 { + // This should never happen. In practice, previously currentBlock + // contained the entire block whereas now only a "marker", so there + // is an ever so slight chance for a race we should handle. + log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) + return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + } } bc.chainHeadFeed.Send(ChainHeadEvent{Header: header}) return nil @@ -607,11 +688,15 @@ func (bc *BlockChain) SetHeadWithTimestamp(timestamp uint64) error { // Send chain head event to update the transaction pool header := bc.CurrentBlock() if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil { - // This should never happen. In practice, previously currentBlock - // contained the entire block whereas now only a "marker", so there - // is an ever so slight chance for a race we should handle. - log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) - return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + // In a pruned node the genesis block will not exist in the freezer. + // It should not happen that we set head to any other pruned block. + if header.Number.Uint64() > 0 { + // This should never happen. In practice, previously currentBlock + // contained the entire block whereas now only a "marker", so there + // is an ever so slight chance for a race we should handle. + log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) + return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + } } bc.chainHeadFeed.Send(ChainHeadEvent{Header: header}) return nil @@ -1014,7 +1099,9 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { bc.hc.SetCurrentHeader(bc.genesisBlock.Header()) bc.currentSnapBlock.Store(bc.genesisBlock.Header()) headFastBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) - return nil + + // Reset history pruning status. + return bc.initializeHistoryPruning(0) } // Export writes the active chain to the given writer. @@ -1804,6 +1891,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness trieDiffNodes, trieBufNodes, _ := bc.triedb.Size() stats.report(chain, it.index, snapDiffItems, snapBufItems, trieDiffNodes, trieBufNodes, setHead) + // Print confirmation that a future fork is scheduled, but not yet active. + bc.logForkReadiness(block) + if !setHead { // After merge we expect few side chains. Simply count // all blocks the CL gives us for GC processing time @@ -1837,6 +1927,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness "root", block.Root()) } } + stats.ignored += it.remaining() return witness, it.index, err } @@ -2434,6 +2525,23 @@ func (bc *BlockChain) reportBlock(block *types.Block, res *ProcessResult, err er log.Error(summarizeBadBlock(block, receipts, bc.Config(), err)) } +// logForkReadiness will write a log when a future fork is scheduled, but not +// active. This is useful so operators know their client is ready for the fork. +func (bc *BlockChain) logForkReadiness(block *types.Block) { + c := bc.Config() + current, last := c.LatestFork(block.Time()), c.LatestFork(math.MaxUint64) + t := c.Timestamp(last) + if t == nil { + return + } + at := time.Unix(int64(*t), 0) + if current < last && time.Now().After(bc.lastForkReadyAlert.Add(forkReadyInterval)) { + log.Info("Ready for fork activation", "fork", last, "date", at.Format(time.RFC822), + "remaining", time.Until(at).Round(time.Second), "timestamp", at.Unix()) + bc.lastForkReadyAlert = time.Now() + } +} + // summarizeBadBlock returns a string summarizing the bad block and other // relevant information. func summarizeBadBlock(block *types.Block, receipts []*types.Receipt, config *params.ChainConfig, err error) string { diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 4114723469..a8c2e26d18 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -410,7 +410,11 @@ func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) { // HistoryPruningCutoff returns the configured history pruning point. // Blocks before this might not be available in the database. func (bc *BlockChain) HistoryPruningCutoff() (uint64, common.Hash) { - return bc.cacheConfig.HistoryPruningCutoffNumber, bc.cacheConfig.HistoryPruningCutoffHash + pt := bc.historyPrunePoint.Load() + if pt == nil { + return 0, bc.genesisBlock.Hash() + } + return pt.BlockNumber, pt.BlockHash } // TrieDB retrieves the low level trie database used for data storage. diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 3f7c03b93c..289eff0a8f 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -4257,13 +4258,7 @@ func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) { // be persisted without the receipts and bodies; chain after should be persisted // normally. func TestInsertChainWithCutoff(t *testing.T) { - testInsertChainWithCutoff(t, 32, 32) // cutoff = 32, ancientLimit = 32 - testInsertChainWithCutoff(t, 32, 64) // cutoff = 32, ancientLimit = 64 (entire chain in ancient) - testInsertChainWithCutoff(t, 32, 65) // cutoff = 32, ancientLimit = 65 (64 blocks in ancient, 1 block in live) -} - -func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64) { - // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) + const chainLength = 64 // Configure and generate a sample block chain var ( @@ -4278,24 +4273,51 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64) signer = types.LatestSigner(gspec.Config) engine = beacon.New(ethash.NewFaker()) ) - _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, int(2*cutoff), func(i int, block *BlockGen) { + _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, chainLength, func(i int, block *BlockGen) { block.SetCoinbase(common.Address{0x00}) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) if err != nil { panic(err) } block.AddTx(tx) }) + + // Run the actual tests. + t.Run("cutoff-32/ancientLimit-32", func(t *testing.T) { + // cutoff = 32, ancientLimit = 32 + testInsertChainWithCutoff(t, 32, 32, gspec, blocks, receipts) + }) + t.Run("cutoff-32/ancientLimit-64", func(t *testing.T) { + // cutoff = 32, ancientLimit = 64 (entire chain in ancient) + testInsertChainWithCutoff(t, 32, 64, gspec, blocks, receipts) + }) + t.Run("cutoff-32/ancientLimit-64", func(t *testing.T) { + // cutoff = 32, ancientLimit = 65 (64 blocks in ancient, 1 block in live) + testInsertChainWithCutoff(t, 32, 65, gspec, blocks, receipts) + }) +} + +func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64, genesis *Genesis, blocks []*types.Block, receipts []types.Receipts) { + // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) + + // Add a known pruning point for the duration of the test. + ghash := genesis.ToBlock().Hash() + cutoffBlock := blocks[cutoff-1] + history.PrunePoints[ghash] = &history.PrunePoint{ + BlockNumber: cutoffBlock.NumberU64(), + BlockHash: cutoffBlock.Hash(), + } + defer func() { + delete(history.PrunePoints, ghash) + }() + + // Enable pruning in cache config. + config := DefaultCacheConfigWithScheme(rawdb.PathScheme) + config.ChainHistoryMode = history.KeepPostMerge + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false) defer db.Close() - - cutoffBlock := blocks[cutoff-1] - config := DefaultCacheConfigWithScheme(rawdb.PathScheme) - config.HistoryPruningCutoffNumber = cutoffBlock.NumberU64() - config.HistoryPruningCutoffHash = cutoffBlock.Hash() - - chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil) + chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), genesis, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil) defer chain.Stop() var ( @@ -4326,8 +4348,8 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64) t.Errorf("head header #%d: header mismatch: want: %v, got: %v", headHeader.Number, blocks[len(blocks)-1].Hash(), headHeader.Hash()) } headBlock := chain.CurrentBlock() - if headBlock.Hash() != gspec.ToBlock().Hash() { - t.Errorf("head block #%d: header mismatch: want: %v, got: %v", headBlock.Number, gspec.ToBlock().Hash(), headBlock.Hash()) + if headBlock.Hash() != ghash { + t.Errorf("head block #%d: header mismatch: want: %v, got: %v", headBlock.Number, ghash, headBlock.Hash()) } // Iterate over all chain data components, and cross reference diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index 18b1c7dc79..fa2d6e3ffb 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -50,6 +50,7 @@ var ( ) const ( + databaseVersion = 1 // reindexed if database version does not match cachedLastBlocks = 1000 // last block of map pointers cachedLvPointers = 1000 // first log value pointer of block pointers cachedBaseRows = 100 // groups of base layer filter row data @@ -138,13 +139,25 @@ type FilterMaps struct { // as transparent (uncached/unchanged). type filterMap []FilterRow -// copy returns a copy of the given filter map. Note that the row slices are -// copied but their contents are not. This permits extending the rows further +// fastCopy returns a copy of the given filter map. Note that the row slices are +// copied but their contents are not. This permits appending to the rows further // (which happens during map rendering) without affecting the validity of // copies made for snapshots during rendering. -func (fm filterMap) copy() filterMap { +// Appending to the rows of both the original map and the fast copy, or two fast +// copies of the same map would result in data corruption, therefore a fast copy +// should always be used in a read only way. +func (fm filterMap) fastCopy() filterMap { + return slices.Clone(fm) +} + +// fullCopy returns a copy of the given filter map, also making a copy of each +// individual filter row, ensuring that a modification to either one will never +// affect the other. +func (fm filterMap) fullCopy() filterMap { c := make(filterMap, len(fm)) - copy(c, fm) + for i, row := range fm { + c[i] = slices.Clone(row) + } return c } @@ -207,8 +220,9 @@ type Config struct { // NewFilterMaps creates a new FilterMaps and starts the indexer. func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, finalBlock uint64, params Params, config Config) *FilterMaps { rs, initialized, err := rawdb.ReadFilterMapsRange(db) - if err != nil { - log.Error("Error reading log index range", "error", err) + if err != nil || rs.Version != databaseVersion { + rs, initialized = rawdb.FilterMapsRange{}, false + log.Warn("Invalid log index database version; resetting log index") } params.deriveFields() f := &FilterMaps{ @@ -437,6 +451,7 @@ func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, ne f.updateMatchersValidRange() if newRange.initialized { rs := rawdb.FilterMapsRange{ + Version: databaseVersion, HeadIndexed: newRange.headIndexed, HeadDelimiter: newRange.headDelimiter, BlocksFirst: newRange.blocks.First(), diff --git a/core/filtermaps/indexer_test.go b/core/filtermaps/indexer_test.go index a02f8d2459..4dddd27087 100644 --- a/core/filtermaps/indexer_test.go +++ b/core/filtermaps/indexer_test.go @@ -17,8 +17,10 @@ package filtermaps import ( + "context" crand "crypto/rand" "crypto/sha256" + "encoding/binary" "math/big" "math/rand" "sync" @@ -31,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" ) var testParams = Params{ @@ -104,6 +107,7 @@ func TestIndexerRandomRange(t *testing.T) { fork, head = rand.Intn(len(forks)), rand.Intn(1001) ts.chain.setCanonicalChain(forks[fork][:head+1]) case 2: + checkSnapshot = false if head < 1000 { checkSnapshot = !noHistory && head != 0 // no snapshot generated for block 0 // add blocks after the current head @@ -158,6 +162,63 @@ func TestIndexerRandomRange(t *testing.T) { } } +func TestIndexerMatcherView(t *testing.T) { + testIndexerMatcherView(t, false) +} + +func TestIndexerMatcherViewWithConcurrentRead(t *testing.T) { + testIndexerMatcherView(t, true) +} + +func testIndexerMatcherView(t *testing.T, concurrentRead bool) { + ts := newTestSetup(t) + defer ts.close() + + forks := make([][]common.Hash, 20) + hashes := make([]common.Hash, 20) + ts.chain.addBlocks(100, 5, 2, 4, true) + ts.setHistory(0, false) + for i := range forks { + if i != 0 { + ts.chain.setHead(100 - i) + ts.chain.addBlocks(i, 5, 2, 4, true) + } + ts.fm.WaitIdle() + forks[i] = ts.chain.getCanonicalChain() + hashes[i] = ts.matcherViewHash() + } + fork := len(forks) - 1 + for i := 0; i < 5000; i++ { + oldFork := fork + fork = rand.Intn(len(forks)) + stopCh := make(chan chan struct{}) + if concurrentRead { + go func() { + for { + ts.matcherViewHash() + select { + case ch := <-stopCh: + close(ch) + return + default: + } + } + }() + } + ts.chain.setCanonicalChain(forks[fork]) + ts.fm.WaitIdle() + if concurrentRead { + ch := make(chan struct{}) + stopCh <- ch + <-ch + } + hash := ts.matcherViewHash() + if hash != hashes[fork] { + t.Fatalf("Matcher view hash mismatch when switching from for %d to %d", oldFork, fork) + } + } +} + func TestIndexerCompareDb(t *testing.T) { ts := newTestSetup(t) defer ts.close() @@ -291,6 +352,55 @@ func (ts *testSetup) fmDbHash() common.Hash { return result } +func (ts *testSetup) matcherViewHash() common.Hash { + mb := ts.fm.NewMatcherBackend() + defer mb.Close() + + ctx := context.Background() + params := mb.GetParams() + hasher := sha256.New() + var headPtr uint64 + for b := uint64(0); ; b++ { + lvptr, err := mb.GetBlockLvPointer(ctx, b) + if err != nil || (b > 0 && lvptr == headPtr) { + break + } + var enc [8]byte + binary.LittleEndian.PutUint64(enc[:], lvptr) + hasher.Write(enc[:]) + headPtr = lvptr + } + headMap := uint32(headPtr >> params.logValuesPerMap) + var enc [12]byte + for r := uint32(0); r < params.mapHeight; r++ { + binary.LittleEndian.PutUint32(enc[:4], r) + for m := uint32(0); m <= headMap; m++ { + binary.LittleEndian.PutUint32(enc[4:8], m) + row, _ := mb.GetFilterMapRow(ctx, m, r, false) + for _, v := range row { + binary.LittleEndian.PutUint32(enc[8:], v) + hasher.Write(enc[:]) + } + } + } + var hash common.Hash + hasher.Sum(hash[:0]) + for i := 0; i < 50; i++ { + hasher.Reset() + hasher.Write(hash[:]) + lvptr := binary.LittleEndian.Uint64(hash[:8]) % headPtr + if log, _ := mb.GetLogByLvIndex(ctx, lvptr); log != nil { + enc, err := rlp.EncodeToBytes(log) + if err != nil { + panic(err) + } + hasher.Write(enc) + } + hasher.Sum(hash[:0]) + } + return hash +} + func (ts *testSetup) close() { if ts.fm != nil { ts.fm.Stop() diff --git a/core/filtermaps/map_renderer.go b/core/filtermaps/map_renderer.go index 28f943abb3..cd960ad31c 100644 --- a/core/filtermaps/map_renderer.go +++ b/core/filtermaps/map_renderer.go @@ -84,7 +84,7 @@ func (f *FilterMaps) renderMapsBefore(renderBefore uint32) (*mapRenderer, error) if err != nil { return nil, err } - if snapshot := f.lastCanonicalSnapshotBefore(renderBefore); snapshot != nil && snapshot.mapIndex >= nextMap { + if snapshot := f.lastCanonicalSnapshotOfMap(nextMap); snapshot != nil { return f.renderMapsFromSnapshot(snapshot) } if nextMap >= renderBefore { @@ -97,14 +97,14 @@ func (f *FilterMaps) renderMapsBefore(renderBefore uint32) (*mapRenderer, error) // snapshot made at a block boundary. func (f *FilterMaps) renderMapsFromSnapshot(cp *renderedMap) (*mapRenderer, error) { f.testSnapshotUsed = true - iter, err := f.newLogIteratorFromBlockDelimiter(cp.lastBlock) + iter, err := f.newLogIteratorFromBlockDelimiter(cp.lastBlock, cp.headDelimiter) if err != nil { return nil, fmt.Errorf("failed to create log iterator from block delimiter %d: %v", cp.lastBlock, err) } return &mapRenderer{ f: f, currentMap: &renderedMap{ - filterMap: cp.filterMap.copy(), + filterMap: cp.filterMap.fullCopy(), mapIndex: cp.mapIndex, lastBlock: cp.lastBlock, blockLvPtrs: cp.blockLvPtrs, @@ -137,14 +137,14 @@ func (f *FilterMaps) renderMapsFromMapBoundary(firstMap, renderBefore uint32, st }, nil } -// lastCanonicalSnapshotBefore returns the latest cached snapshot that matches -// the current targetView. -func (f *FilterMaps) lastCanonicalSnapshotBefore(renderBefore uint32) *renderedMap { +// lastCanonicalSnapshotOfMap returns the latest cached snapshot of the given map +// that is also consistent with the current targetView. +func (f *FilterMaps) lastCanonicalSnapshotOfMap(mapIndex uint32) *renderedMap { var best *renderedMap for _, blockNumber := range f.renderSnapshots.Keys() { if cp, _ := f.renderSnapshots.Get(blockNumber); cp != nil && blockNumber < f.indexedRange.blocks.AfterLast() && blockNumber <= f.targetView.headNumber && f.targetView.getBlockId(blockNumber) == cp.lastBlockId && - cp.mapIndex < renderBefore && (best == nil || blockNumber > best.lastBlock) { + cp.mapIndex == mapIndex && (best == nil || blockNumber > best.lastBlock) { best = cp } } @@ -171,10 +171,9 @@ func (f *FilterMaps) lastCanonicalMapBoundaryBefore(renderBefore uint32) (nextMa if err != nil { return 0, 0, 0, fmt.Errorf("failed to retrieve last block of reverse iterated map %d: %v", mapIndex, err) } - if lastBlock >= f.indexedView.headNumber || lastBlock >= f.targetView.headNumber || - lastBlockId != f.targetView.getBlockId(lastBlock) { - // map is not full or inconsistent with targetView; roll back - continue + if (f.indexedRange.headIndexed && mapIndex >= f.indexedRange.maps.Last()) || + lastBlock >= f.targetView.headNumber || lastBlockId != f.targetView.getBlockId(lastBlock) { + continue // map is not full or inconsistent with targetView; roll back } lvPtr, err := f.getBlockLvPointer(lastBlock) if err != nil { @@ -257,11 +256,14 @@ func (f *FilterMaps) loadHeadSnapshot() error { // makeSnapshot creates a snapshot of the current state of the rendered map. func (r *mapRenderer) makeSnapshot() { - r.f.renderSnapshots.Add(r.iterator.blockNumber, &renderedMap{ - filterMap: r.currentMap.filterMap.copy(), + if r.iterator.blockNumber != r.currentMap.lastBlock || r.iterator.chainView != r.f.targetView { + panic("iterator state inconsistent with current rendered map") + } + r.f.renderSnapshots.Add(r.currentMap.lastBlock, &renderedMap{ + filterMap: r.currentMap.filterMap.fastCopy(), mapIndex: r.currentMap.mapIndex, - lastBlock: r.iterator.blockNumber, - lastBlockId: r.f.targetView.getBlockId(r.currentMap.lastBlock), + lastBlock: r.currentMap.lastBlock, + lastBlockId: r.iterator.chainView.getBlockId(r.currentMap.lastBlock), blockLvPtrs: r.currentMap.blockLvPtrs, finished: true, headDelimiter: r.iterator.lvIndex, @@ -661,24 +663,13 @@ var errUnindexedRange = errors.New("unindexed range") // newLogIteratorFromBlockDelimiter creates a logIterator starting at the // given block's first log value entry (the block delimiter), according to the // current targetView. -func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber uint64) (*logIterator, error) { +func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber, lvIndex uint64) (*logIterator, error) { if blockNumber > f.targetView.headNumber { return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", blockNumber, f.targetView.headNumber) } if !f.indexedRange.blocks.Includes(blockNumber) { return nil, errUnindexedRange } - var lvIndex uint64 - if f.indexedRange.headIndexed && blockNumber+1 == f.indexedRange.blocks.AfterLast() { - lvIndex = f.indexedRange.headDelimiter - } else { - var err error - lvIndex, err = f.getBlockLvPointer(blockNumber + 1) - if err != nil { - return nil, fmt.Errorf("failed to retrieve log value pointer of block %d after delimiter: %v", blockNumber+1, err) - } - lvIndex-- - } finished := blockNumber == f.targetView.headNumber l := &logIterator{ chainView: f.targetView, diff --git a/core/filtermaps/matcher_backend.go b/core/filtermaps/matcher_backend.go index 01bae7bb22..335ac84551 100644 --- a/core/filtermaps/matcher_backend.go +++ b/core/filtermaps/matcher_backend.go @@ -75,6 +75,9 @@ func (fm *FilterMapsMatcherBackend) Close() { // on write. // GetFilterMapRow implements MatcherBackend. func (fm *FilterMapsMatcherBackend) GetFilterMapRow(ctx context.Context, mapIndex, rowIndex uint32, baseLayerOnly bool) (FilterRow, error) { + fm.f.indexLock.RLock() + defer fm.f.indexLock.RUnlock() + return fm.f.getFilterMapRow(mapIndex, rowIndex, baseLayerOnly) } diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index 31e2b534be..413e4d77a8 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -76,8 +76,10 @@ func TestCreation(t *testing.T) { {20000000, 1681338454, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 1681338455}}, // Last Gray Glacier block {20000000, 1681338455, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // First Shanghai block {30000000, 1710338134, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // Last Shanghai block - {40000000, 1710338135, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // First Cancun block - {50000000, 2000000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // Future Cancun block + {40000000, 1710338135, ID{Hash: checksumToBytes(0x9f3d2254), Next: 1746612311}}, // First Cancun block + {30000000, 1746022486, ID{Hash: checksumToBytes(0x9f3d2254), Next: 1746612311}}, // Last Cancun block + {30000000, 1746612311, ID{Hash: checksumToBytes(0xc376cf8b), Next: 0}}, // First Prague block + {50000000, 2000000000, ID{Hash: checksumToBytes(0xc376cf8b), Next: 0}}, // Future Prague block }, }, // Sepolia test cases @@ -137,9 +139,11 @@ func TestCreation(t *testing.T) { // fork ID. func TestValidation(t *testing.T) { // Config that has not timestamp enabled + // TODO(lightclient): this always needs to be updated when a mainnet timestamp is set. legacyConfig := *params.MainnetChainConfig legacyConfig.ShanghaiTime = nil legacyConfig.CancunTime = nil + legacyConfig.PragueTime = nil tests := []struct { config *params.ChainConfig @@ -314,9 +318,7 @@ func TestValidation(t *testing.T) { // Local is mainnet Prague, remote announces Shanghai + knowledge about Cancun. Remote // is definitely out of sync. It may or may not need the Prague update, we don't know yet. - // - // TODO(karalabe): Enable this when Cancun **and** Prague is specced, update all the numbers - //{params.MainnetChainConfig, 0, 0, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}, nil}, + {params.MainnetChainConfig, 0, 0, ID{Hash: checksumToBytes(0x3edd5b10), Next: 1710338135}, nil}, // Local is mainnet Shanghai, remote announces Cancun. Local is out of sync, accept. {params.MainnetChainConfig, 21000000, 1700000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}, nil}, @@ -324,8 +326,7 @@ func TestValidation(t *testing.T) { // Local is mainnet Shanghai, remote announces Cancun, but is not aware of Prague. Local // out of sync. Local also knows about a future fork, but that is uncertain yet. // - // TODO(karalabe): Enable this when Cancun **and** Prague is specced, update remote checksum - //{params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x00000000), Next: 0}, nil}, + {params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0xc376cf8b), Next: 0}, nil}, // Local is mainnet Cancun. remote announces Shanghai but is not aware of further forks. // Remote needs software update. @@ -342,11 +343,11 @@ func TestValidation(t *testing.T) { // Local is mainnet Shanghai, remote is random Shanghai. {params.MainnetChainConfig, 20000000, 1681338455, ID{Hash: checksumToBytes(0x12345678), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Cancun, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Prague, far in the future. Remote announces Gopherium (non existing fork) // at some future timestamp 8888888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0x9f3d2254), Next: 8888888888}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0xc376cf8b), Next: 8888888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Shanghai. Remote is also in Shanghai, but announces Gopherium (non existing // fork) at timestamp 1668000000, before Cancun. Local is incompatible. diff --git a/eth/ethconfig/historymode.go b/core/history/historymode.go similarity index 79% rename from eth/ethconfig/historymode.go rename to core/history/historymode.go index a595d72feb..e735222d37 100644 --- a/eth/ethconfig/historymode.go +++ b/core/history/historymode.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package ethconfig +package history import ( "fmt" @@ -27,22 +27,22 @@ import ( type HistoryMode uint32 const ( - // AllHistory (default) means that all chain history down to genesis block will be kept. - AllHistory HistoryMode = iota + // KeepAll (default) means that all chain history down to genesis block will be kept. + KeepAll HistoryMode = iota - // PostMergeHistory sets the history pruning point to the merge activation block. - PostMergeHistory + // KeepPostMerge sets the history pruning point to the merge activation block. + KeepPostMerge ) func (m HistoryMode) IsValid() bool { - return m <= PostMergeHistory + return m <= KeepPostMerge } func (m HistoryMode) String() string { switch m { - case AllHistory: + case KeepAll: return "all" - case PostMergeHistory: + case KeepPostMerge: return "postmerge" default: return fmt.Sprintf("invalid HistoryMode(%d)", m) @@ -61,24 +61,24 @@ func (m HistoryMode) MarshalText() ([]byte, error) { func (m *HistoryMode) UnmarshalText(text []byte) error { switch string(text) { case "all": - *m = AllHistory + *m = KeepAll case "postmerge": - *m = PostMergeHistory + *m = KeepPostMerge default: return fmt.Errorf(`unknown sync mode %q, want "all" or "postmerge"`, text) } return nil } -type HistoryPrunePoint struct { +type PrunePoint struct { BlockNumber uint64 BlockHash common.Hash } -// HistoryPrunePoints contains the pre-defined history pruning cutoff blocks for known networks. +// PrunePoints the pre-defined history pruning cutoff blocks for known networks. // They point to the first post-merge block. Any pruning should truncate *up to* but excluding // given block. -var HistoryPrunePoints = map[common.Hash]*HistoryPrunePoint{ +var PrunePoints = map[common.Hash]*PrunePoint{ // mainnet params.MainnetGenesisHash: { BlockNumber: 15537393, @@ -91,7 +91,7 @@ var HistoryPrunePoints = map[common.Hash]*HistoryPrunePoint{ }, } -// PrunedHistoryError is returned when the requested history is pruned. +// PrunedHistoryError is returned by APIs when the requested history is pruned. type PrunedHistoryError struct{} func (e *PrunedHistoryError) Error() string { return "pruned history unavailable" } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index c413839b7b..1a5c414c8e 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -434,6 +434,7 @@ func DeleteBlockLvPointers(db ethdb.KeyValueStore, blocks common.Range[uint64], // FilterMapsRange is a storage representation of the block range covered by the // filter maps structure and the corresponting log value index range. type FilterMapsRange struct { + Version uint32 HeadIndexed bool HeadDelimiter uint64 BlocksFirst, BlocksAfterLast uint64 diff --git a/core/state/dump.go b/core/state/dump.go index c9aad4f8e2..11b5c32782 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -188,7 +188,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] c.OnAccount(address, account) accounts++ if time.Since(logged) > 8*time.Second { - log.Info("Trie dumping in progress", "at", it.Key, "accounts", accounts, + log.Info("Trie dumping in progress", "at", common.Bytes2Hex(it.Key), "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() } diff --git a/core/txindexer.go b/core/txindexer.go index 31f069995b..64a2e8c49f 100644 --- a/core/txindexer.go +++ b/core/txindexer.go @@ -196,6 +196,19 @@ func (indexer *txIndexer) repair(head uint64) { } } +// resolveHead resolves the block number of the current chain head. +func (indexer *txIndexer) resolveHead() uint64 { + headBlockHash := rawdb.ReadHeadBlockHash(indexer.db) + if headBlockHash == (common.Hash{}) { + return 0 + } + headBlockNumber := rawdb.ReadHeaderNumber(indexer.db, headBlockHash) + if headBlockNumber == nil { + return 0 + } + return *headBlockNumber +} + // loop is the scheduler of the indexer, assigning indexing/unindexing tasks depending // on the received chain event. func (indexer *txIndexer) loop(chain *BlockChain) { @@ -203,9 +216,9 @@ func (indexer *txIndexer) loop(chain *BlockChain) { // Listening to chain events and manipulate the transaction indexes. var ( - stop chan struct{} // Non-nil if background routine is active - done chan struct{} // Non-nil if background routine is active - head = rawdb.ReadHeadBlock(indexer.db).NumberU64() // The latest announced chain head + stop chan struct{} // Non-nil if background routine is active + done chan struct{} // Non-nil if background routine is active + head = indexer.resolveHead() // The latest announced chain head headCh = make(chan ChainHeadEvent) sub = chain.SubscribeChainHeadEvent(headCh) diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 04f1a2234c..7bf360ff65 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1827,6 +1827,16 @@ func (t *lookup) Remove(hash common.Hash) { delete(t.txs, hash) } +// Clear resets the lookup structure, removing all stored entries. +func (t *lookup) Clear() { + t.lock.Lock() + defer t.lock.Unlock() + + t.slots = 0 + t.txs = make(map[common.Hash]*types.Transaction) + t.auths = make(map[common.Address][]common.Hash) +} + // TxsBelowTip finds all remote transactions below the given tip threshold. func (t *lookup) TxsBelowTip(threshold *big.Int) types.Transactions { found := make(types.Transactions, 0, 128) @@ -1923,7 +1933,7 @@ func (pool *LegacyPool) Clear() { for addr := range pool.queue { pool.reserver.Release(addr) } - pool.all = newLookup() + pool.all.Clear() pool.priced = newPricedList(pool.all) pool.pending = make(map[common.Address]*list) pool.queue = make(map[common.Address]*list) diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 2ed38772ce..fc4a7be6d2 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -186,13 +186,15 @@ func (p *TxPool) loop(head *types.Header) { // Try to inject a busy marker and start a reset if successful select { case resetBusy <- struct{}{}: - statedb, err := p.chain.StateAt(newHead.Root) - if err != nil { - log.Crit("Failed to reset txpool state", "err", err) + // Updates the statedb with the new chain head. The head state may be + // unavailable if the initial state sync has not yet completed. + if statedb, err := p.chain.StateAt(newHead.Root); err != nil { + log.Error("Failed to reset txpool state", "err", err) + } else { + p.stateLock.Lock() + p.state = statedb + p.stateLock.Unlock() } - p.stateLock.Lock() - p.state = statedb - p.stateLock.Unlock() // Busy marker injected, start a new subpool reset go func(oldHead, newHead *types.Header) { diff --git a/eth/api_backend.go b/eth/api_backend.go index b39dd4cbdb..182c081d2b 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -29,12 +29,12 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/filtermaps" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" @@ -156,7 +156,7 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe } block := b.eth.blockchain.GetBlockByNumber(bn) if block == nil && bn < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return block, nil } @@ -168,7 +168,7 @@ func (b *EthAPIBackend) BlockByHash(ctx context.Context, hash common.Hash) (*typ } block := b.eth.blockchain.GetBlock(hash, *number) if block == nil && *number < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return block, nil } @@ -181,7 +181,7 @@ func (b *EthAPIBackend) GetBody(ctx context.Context, hash common.Hash, number rp body := b.eth.blockchain.GetBody(hash) if body == nil { if uint64(number) < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return nil, errors.New("block body not found") } @@ -203,7 +203,7 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r block := b.eth.blockchain.GetBlock(hash, header.Number.Uint64()) if block == nil { if header.Number.Uint64() < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return nil, errors.New("header found, but block body is missing") } diff --git a/eth/backend.go b/eth/backend.go index c5dec77962..6d1b6bae99 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -145,7 +145,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { // Here we determine genesis hash and active ChainConfig. // We need these to figure out the consensus parameters and to set up history pruning. - chainConfig, genesisHash, err := core.LoadChainConfig(chainDb, config.Genesis) + chainConfig, _, err := core.LoadChainConfig(chainDb, config.Genesis) if err != nil { return nil, err } @@ -153,22 +153,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - - // Validate history pruning configuration. - var ( - cutoffNumber uint64 - cutoffHash common.Hash - ) - if config.HistoryMode == ethconfig.PostMergeHistory { - prunecfg, ok := ethconfig.HistoryPrunePoints[genesisHash] - if !ok { - return nil, fmt.Errorf("no history pruning point is defined for genesis %x", genesisHash) - } - cutoffNumber = prunecfg.BlockNumber - cutoffHash = prunecfg.BlockHash - log.Info("Chain cutoff configured", "number", cutoffNumber, "hash", cutoffHash) - } - // Set networkID to chainID by default. networkID := config.NetworkId if networkID == 0 { @@ -195,6 +179,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } log.Info("Initialising Ethereum protocol", "network", networkID, "dbversion", dbVer) + // Create BlockChain object. if !config.SkipBcVersionCheck { if bcVersion != nil && *bcVersion > core.BlockChainVersion { return nil, fmt.Errorf("database version is v%d, Geth %s only supports v%d", *bcVersion, version.WithMeta, core.BlockChainVersion) @@ -210,17 +195,16 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { EnablePreimageRecording: config.EnablePreimageRecording, } cacheConfig = &core.CacheConfig{ - TrieCleanLimit: config.TrieCleanCache, - TrieCleanNoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - TrieDirtyDisabled: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, - SnapshotLimit: config.SnapshotCache, - Preimages: config.Preimages, - StateHistory: config.StateHistory, - StateScheme: scheme, - HistoryPruningCutoffNumber: cutoffNumber, - HistoryPruningCutoffHash: cutoffHash, + TrieCleanLimit: config.TrieCleanCache, + TrieCleanNoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + TrieDirtyDisabled: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, + SnapshotLimit: config.SnapshotCache, + Preimages: config.Preimages, + StateHistory: config.StateHistory, + StateScheme: scheme, + ChainHistoryMode: config.HistoryMode, } ) if config.VMTrace != "" { @@ -246,6 +230,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } + + // Initialize filtermaps log index. fmConfig := filtermaps.Config{ History: config.LogHistory, Disabled: config.LogNoHistory, @@ -261,6 +247,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { eth.filterMaps = filtermaps.NewFilterMaps(chainDb, chainView, historyCutoff, finalBlock, filtermaps.DefaultParams, fmConfig) eth.closeFilterMaps = make(chan chan struct{}) + // TxPool if config.TxPool.Journal != "" { config.TxPool.Journal = stack.ResolvePath(config.TxPool.Journal) } @@ -285,6 +272,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { eth.localTxTracker = locals.New(config.TxPool.Journal, rejournal, eth.blockchain.Config(), eth.txPool) stack.RegisterLifecycle(eth.localTxTracker) } + // Permit the downloader to use the trie cache allowance during fast sync cacheLimit := cacheConfig.TrieCleanLimit + cacheConfig.TrieDirtyLimit + cacheConfig.SnapshotLimit if eth.handler, err = newHandler(&handlerConfig{ diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 365857347c..5e19824135 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/eth/gasprice" @@ -48,7 +49,7 @@ var FullNodeGPO = gasprice.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ - HistoryMode: AllHistory, + HistoryMode: history.KeepAll, SyncMode: SnapSync, NetworkId: 0, // enable auto configuration of networkID == chainID TxLookupLimit: 2350000, @@ -84,7 +85,7 @@ type Config struct { SyncMode SyncMode // HistoryMode configures chain history retention. - HistoryMode HistoryMode + HistoryMode history.HistoryMode // This can be set to list of enrtree:// URLs which will be queried for // nodes to connect to. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index bcfe8a31d6..cdcddcc772 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/eth/gasprice" @@ -19,13 +20,16 @@ func (c Config) MarshalTOML() (interface{}, error) { Genesis *core.Genesis `toml:",omitempty"` NetworkId uint64 SyncMode SyncMode - HistoryMode HistoryMode + HistoryMode history.HistoryMode EthDiscoveryURLs []string SnapDiscoveryURLs []string NoPruning bool NoPrefetch bool - TxLookupLimit uint64 `toml:",omitempty"` - TransactionHistory uint64 `toml:",omitempty"` + TxLookupLimit uint64 `toml:",omitempty"` + TransactionHistory uint64 `toml:",omitempty"` + LogHistory uint64 `toml:",omitempty"` + LogNoHistory bool `toml:",omitempty"` + LogExportCheckpoints string StateHistory uint64 `toml:",omitempty"` StateScheme string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` @@ -63,6 +67,9 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.NoPrefetch = c.NoPrefetch enc.TxLookupLimit = c.TxLookupLimit enc.TransactionHistory = c.TransactionHistory + enc.LogHistory = c.LogHistory + enc.LogNoHistory = c.LogNoHistory + enc.LogExportCheckpoints = c.LogExportCheckpoints enc.StateHistory = c.StateHistory enc.StateScheme = c.StateScheme enc.RequiredBlocks = c.RequiredBlocks @@ -97,13 +104,16 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { Genesis *core.Genesis `toml:",omitempty"` NetworkId *uint64 SyncMode *SyncMode - HistoryMode *HistoryMode + HistoryMode *history.HistoryMode EthDiscoveryURLs []string SnapDiscoveryURLs []string NoPruning *bool NoPrefetch *bool - TxLookupLimit *uint64 `toml:",omitempty"` - TransactionHistory *uint64 `toml:",omitempty"` + TxLookupLimit *uint64 `toml:",omitempty"` + TransactionHistory *uint64 `toml:",omitempty"` + LogHistory *uint64 `toml:",omitempty"` + LogNoHistory *bool `toml:",omitempty"` + LogExportCheckpoints *string StateHistory *uint64 `toml:",omitempty"` StateScheme *string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` @@ -164,6 +174,15 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TransactionHistory != nil { c.TransactionHistory = *dec.TransactionHistory } + if dec.LogHistory != nil { + c.LogHistory = *dec.LogHistory + } + if dec.LogNoHistory != nil { + c.LogNoHistory = *dec.LogNoHistory + } + if dec.LogExportCheckpoints != nil { + c.LogExportCheckpoints = *dec.LogExportCheckpoints + } if dec.StateHistory != nil { c.StateHistory = *dec.StateHistory } diff --git a/eth/filters/api.go b/eth/filters/api.go index 6593cbef27..864dfd3746 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -28,8 +28,8 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rpc" ) @@ -360,7 +360,7 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type return nil, errInvalidBlockRange } if begin > 0 && begin < int64(api.events.backend.HistoryPruningCutoff()) { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } // Construct the range filter filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index e44b37d047..78c80d8f26 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -26,8 +26,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/filtermaps" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -88,7 +88,7 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { return nil, errors.New("unknown block") } if header.Number.Uint64() < f.sys.backend.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return f.blockLogs(ctx, header) } diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index aca3c03ad6..b787f1067b 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -30,8 +30,8 @@ import ( "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/filtermaps" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" @@ -311,7 +311,7 @@ func (es *EventSystem) SubscribeLogs(crit ethereum.FilterQuery, logs chan []*typ } // Queries beyond the pruning cutoff are not supported. if uint64(from) < es.backend.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } // only interested in new mined logs diff --git a/eth/handler.go b/eth/handler.go index b2ad6effdb..8283d7d02f 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -484,7 +484,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { total := new(big.Int).Exp(direct, big.NewInt(2), nil) // Stabilise total peer count a bit based on sqrt peers var ( - signer = types.LatestSignerForChainID(h.chain.Config().ChainID) // Don't care about chain status, we just need *a* sender + signer = types.LatestSigner(h.chain.Config()) // Don't care about chain status, we just need *a* sender hasher = crypto.NewKeccakState() hash = make([]byte, 32) ) diff --git a/p2p/metrics.go b/p2p/metrics.go index 8c9804206b..29c2acb0cb 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -49,7 +49,7 @@ var ( serveSuccessMeter = metrics.NewRegisteredMeter("p2p/serves/success", nil) dialMeter = metrics.NewRegisteredMeter("p2p/dials", nil) dialSuccessMeter = metrics.NewRegisteredMeter("p2p/dials/success", nil) - dialConnectionError = metrics.NewRegisteredMeter("p2p/dials/error/connection", nil) + dialConnectionError = metrics.NewRegisteredMeter("p2p/dials/error/connection", nil) // dial timeout; no route to host; connection refused; network is unreachable // count peers that stayed connected for at least 1 min serve1MinSuccessMeter = metrics.NewRegisteredMeter("p2p/serves/success/1min", nil) @@ -61,34 +61,41 @@ var ( dialSelf = metrics.NewRegisteredMeter("p2p/dials/error/self", nil) dialUselessPeer = metrics.NewRegisteredMeter("p2p/dials/error/useless", nil) dialUnexpectedIdentity = metrics.NewRegisteredMeter("p2p/dials/error/id/unexpected", nil) - dialEncHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/enc", nil) - dialProtoHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/proto", nil) + dialEncHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/enc", nil) // EOF; connection reset during handshake; message too big; i/o timeout + dialProtoHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/proto", nil) // EOF + + // capture the rest of errors that are not handled by the above meters + dialOtherError = metrics.NewRegisteredMeter("p2p/dials/error/other", nil) ) -// markDialError matches errors that occur while setting up a dial connection -// to the corresponding meter. +// markDialError matches errors that occur while setting up a dial connection to the +// corresponding meter. We don't maintain meters for evert possible error, just for +// the most interesting ones. func markDialError(err error) { if !metrics.Enabled() { return } - if err2 := errors.Unwrap(err); err2 != nil { - err = err2 - } - switch err { - case DiscTooManyPeers: + + var reason DiscReason + var handshakeErr *protoHandshakeError + d := errors.As(err, &reason) + switch { + case d && reason == DiscTooManyPeers: dialTooManyPeers.Mark(1) - case DiscAlreadyConnected: + case d && reason == DiscAlreadyConnected: dialAlreadyConnected.Mark(1) - case DiscSelf: + case d && reason == DiscSelf: dialSelf.Mark(1) - case DiscUselessPeer: + case d && reason == DiscUselessPeer: dialUselessPeer.Mark(1) - case DiscUnexpectedIdentity: + case d && reason == DiscUnexpectedIdentity: dialUnexpectedIdentity.Mark(1) - case errEncHandshakeError: - dialEncHandshakeError.Mark(1) - case errProtoHandshakeError: + case errors.As(err, &handshakeErr): dialProtoHandshakeError.Mark(1) + case errors.Is(err, errEncHandshakeError): + dialEncHandshakeError.Mark(1) + default: + dialOtherError.Mark(1) } } diff --git a/p2p/server.go b/p2p/server.go index 4e72e29fa0..d9105976dd 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -66,11 +66,15 @@ const ( ) var ( - errServerStopped = errors.New("server stopped") - errEncHandshakeError = errors.New("rlpx enc error") - errProtoHandshakeError = errors.New("rlpx proto error") + errServerStopped = errors.New("server stopped") + errEncHandshakeError = errors.New("rlpx enc error") ) +type protoHandshakeError struct{ err error } + +func (e *protoHandshakeError) Error() string { return fmt.Sprintf("rlpx proto error: %v", e.err) } +func (e *protoHandshakeError) Unwrap() error { return e.err } + // Server manages all peer connections. type Server struct { // Config fields may not be modified while the server is running. @@ -907,7 +911,7 @@ func (srv *Server) setupConn(c *conn, dialDest *enode.Node) error { phs, err := c.doProtoHandshake(srv.ourHandshake) if err != nil { clog.Trace("Failed p2p handshake", "err", err) - return fmt.Errorf("%w: %v", errProtoHandshakeError, err) + return &protoHandshakeError{err: err} } if id := c.node.ID(); !bytes.Equal(crypto.Keccak256(phs.ID), id[:]) { clog.Trace("Wrong devp2p handshake identity", "phsid", hex.EncodeToString(phs.ID)) diff --git a/p2p/server_test.go b/p2p/server_test.go index a0491e984a..d42926cf4c 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -410,11 +410,11 @@ func TestServerSetupConn(t *testing.T) { wantCloseErr: DiscUnexpectedIdentity, }, { - tt: &setupTransport{pubkey: clientpub, protoHandshakeErr: errProtoHandshakeError}, + tt: &setupTransport{pubkey: clientpub, protoHandshakeErr: DiscTooManyPeers}, dialDest: enode.NewV4(clientpub, nil, 0, 0), flags: dynDialedConn, wantCalls: "doEncHandshake,doProtoHandshake,close,", - wantCloseErr: errProtoHandshakeError, + wantCloseErr: DiscTooManyPeers, }, { tt: &setupTransport{pubkey: srvpub, phs: protoHandshake{ID: crypto.FromECDSAPub(srvpub)[1:]}}, diff --git a/params/config.go b/params/config.go index 8f9e02583b..67aa6b2225 100644 --- a/params/config.go +++ b/params/config.go @@ -60,10 +60,12 @@ var ( TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 ShanghaiTime: newUint64(1681338455), CancunTime: newUint64(1710338135), + PragueTime: newUint64(1746612311), DepositContractAddress: common.HexToAddress("0x00000000219ab540356cbb839cbe05303d7705fa"), Ethash: new(EthashConfig), BlobScheduleConfig: &BlobScheduleConfig{ Cancun: DefaultCancunBlobConfig, + Prague: DefaultPragueBlobConfig, }, } // HoleskyChainConfig contains the chain parameters to run a node on the Holesky test network. @@ -907,6 +909,23 @@ func (c *ChainConfig) LatestFork(time uint64) forks.Fork { } } +// Timestamp returns the timestamp associated with the fork or returns nil if +// the fork isn't defined or isn't a time-based fork. +func (c *ChainConfig) Timestamp(fork forks.Fork) *uint64 { + switch { + case fork == forks.Osaka: + return c.OsakaTime + case fork == forks.Prague: + return c.PragueTime + case fork == forks.Cancun: + return c.CancunTime + case fork == forks.Shanghai: + return c.ShanghaiTime + default: + return nil + } +} + // isForkBlockIncompatible returns true if a fork scheduled at block s1 cannot be // rescheduled to block s2 because head is already past the fork. func isForkBlockIncompatible(s1, s2, head *big.Int) bool { diff --git a/params/forks/forks.go b/params/forks/forks.go index 02f6e5b612..5c9612a625 100644 --- a/params/forks/forks.go +++ b/params/forks/forks.go @@ -20,7 +20,7 @@ package forks type Fork int const ( - Frontier = iota + Frontier Fork = iota FrontierThawing Homestead DAO @@ -41,3 +41,35 @@ const ( Prague Osaka ) + +// String implements fmt.Stringer. +func (f Fork) String() string { + s, ok := forkToString[f] + if !ok { + return "Unknown fork" + } + return s +} + +var forkToString = map[Fork]string{ + Frontier: "Frontier", + FrontierThawing: "Frontier Thawing", + Homestead: "Homestead", + DAO: "DAO", + TangerineWhistle: "Tangerine Whistle", + SpuriousDragon: "Spurious Dragon", + Byzantium: "Byzantium", + Constantinople: "Constantinople", + Petersburg: "Petersburg", + Istanbul: "Istanbul", + MuirGlacier: "Muir Glacier", + Berlin: "Berlin", + London: "London", + ArrowGlacier: "Arrow Glacier", + GrayGlacier: "Gray Glacier", + Paris: "Paris", + Shanghai: "Shanghai", + Cancun: "Cancun", + Prague: "Prague", + Osaka: "Osaka", +}