From 48ec86abbbcbe42d66302e9d4839d82cd9ced36f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 15 Apr 2025 14:32:46 +0200 Subject: [PATCH 01/12] core: initialize history pruning in BlockChain (#31636) I added the history mode configuration in eth/ethconfig initially, since it seemed like the logical place. But it turns out we need access to the intended pruning setting at a deeper level, and it actually needs to be integrated with the blockchain startup procedure. With this change applied, if a node previously had its history pruned, and is subsequently restarted **without** the `--history.chain postmerge` flag, the `BlockChain` initialization code will now verify the freezer tail against the known pruning point of the predefined network and will restore pruning status. Note that this logic is quite restrictive, we allow non-zero tail only for known networks, and only for the specific pruning point that is defined. --- cmd/geth/chaincmd.go | 4 +- cmd/workload/testsuite.go | 6 +- core/block_validator_test.go | 10 ++- core/blockchain.go | 64 ++++++++++++++++++- core/blockchain_reader.go | 6 +- core/blockchain_test.go | 58 +++++++++++------ .../ethconfig => core/history}/historymode.go | 28 ++++---- eth/api_backend.go | 10 +-- eth/backend.go | 44 +++++-------- eth/ethconfig/config.go | 5 +- eth/ethconfig/gen_config.go | 31 +++++++-- eth/filters/api.go | 4 +- eth/filters/filter.go | 4 +- eth/filters/filter_system.go | 4 +- 14 files changed, 188 insertions(+), 90 deletions(-) rename {eth/ethconfig => core/history}/historymode.go (79%) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 2279509542..c57a9a947d 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -31,11 +31,11 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/era" @@ -625,7 +625,7 @@ func pruneHistory(ctx *cli.Context) error { defer chain.Stop() // Determine the prune point. This will be the first PoS block. - prunePoint, ok := ethconfig.HistoryPrunePoints[chain.Genesis().Hash()] + prunePoint, ok := history.PrunePoints[chain.Genesis().Hash()] if !ok || prunePoint == nil { return errors.New("prune point not found") } diff --git a/cmd/workload/testsuite.go b/cmd/workload/testsuite.go index e7019e2055..e8e25e7731 100644 --- a/cmd/workload/testsuite.go +++ b/cmd/workload/testsuite.go @@ -23,7 +23,7 @@ import ( "os" "slices" - "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/log" @@ -124,13 +124,13 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) { cfg.filterQueryFile = "queries/filter_queries_mainnet.json" cfg.historyTestFile = "queries/history_mainnet.json" cfg.historyPruneBlock = new(uint64) - *cfg.historyPruneBlock = ethconfig.HistoryPrunePoints[params.MainnetGenesisHash].BlockNumber + *cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber case ctx.Bool(testSepoliaFlag.Name): cfg.fsys = builtinTestFiles cfg.filterQueryFile = "queries/filter_queries_sepolia.json" cfg.historyTestFile = "queries/history_sepolia.json" cfg.historyPruneBlock = new(uint64) - *cfg.historyPruneBlock = ethconfig.HistoryPrunePoints[params.SepoliaGenesisHash].BlockNumber + *cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber default: cfg.fsys = os.DirFS(".") cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name) diff --git a/core/block_validator_test.go b/core/block_validator_test.go index 8af4057693..5217979236 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -50,8 +50,11 @@ func testHeaderVerification(t *testing.T, scheme string) { headers[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) defer chain.Stop() + if err != nil { + t.Fatal(err) + } for i := 0; i < len(blocks); i++ { for j, valid := range []bool{true, false} { @@ -163,8 +166,11 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { postHeaders[i] = block.Header() } // Run the header checker for blocks one-by-one, checking for both valid and invalid nonces - chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil) + chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil) defer chain.Stop() + if err != nil { + t.Fatal(err) + } // Verify the blocks before the merging for i := 0; i < len(preBlocks); i++ { diff --git a/core/blockchain.go b/core/blockchain.go index d56996dadb..901a93315a 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" @@ -158,8 +159,7 @@ type CacheConfig struct { // This defines the cutoff block for history expiry. // Blocks before this number may be unavailable in the chain database. - HistoryPruningCutoffNumber uint64 - HistoryPruningCutoffHash common.Hash + ChainHistoryMode history.HistoryMode } // triedbConfig derives the configures for trie database. @@ -255,6 +255,7 @@ type BlockChain struct { currentSnapBlock atomic.Pointer[types.Header] // Current head of snap-sync currentFinalBlock atomic.Pointer[types.Header] // Latest (consensus) finalized block currentSafeBlock atomic.Pointer[types.Header] // Latest (consensus) safe block + historyPrunePoint atomic.Pointer[history.PrunePoint] bodyCache *lru.Cache[common.Hash, *types.Body] bodyRLPCache *lru.Cache[common.Hash, rlp.RawValue] @@ -533,6 +534,12 @@ func (bc *BlockChain) loadLastState() error { } bc.hc.SetCurrentHeader(headHeader) + // Initialize history pruning. + latest := max(headBlock.NumberU64(), headHeader.Number.Uint64()) + if err := bc.initializeHistoryPruning(latest); err != nil { + return err + } + // Restore the last known head snap block bc.currentSnapBlock.Store(headBlock.Header()) headFastBlockGauge.Update(int64(headBlock.NumberU64())) @@ -555,6 +562,7 @@ func (bc *BlockChain) loadLastState() error { headSafeBlockGauge.Update(int64(block.NumberU64())) } } + // Issue a status log for the user var ( currentSnapBlock = bc.CurrentSnapBlock() @@ -573,9 +581,57 @@ func (bc *BlockChain) loadLastState() error { if pivot := rawdb.ReadLastPivotNumber(bc.db); pivot != nil { log.Info("Loaded last snap-sync pivot marker", "number", *pivot) } + if pruning := bc.historyPrunePoint.Load(); pruning != nil { + log.Info("Chain history is pruned", "earliest", pruning.BlockNumber, "hash", pruning.BlockHash) + } return nil } +// initializeHistoryPruning sets bc.historyPrunePoint. +func (bc *BlockChain) initializeHistoryPruning(latest uint64) error { + freezerTail, _ := bc.db.Tail() + + switch bc.cacheConfig.ChainHistoryMode { + case history.KeepAll: + if freezerTail == 0 { + return nil + } + // The database was pruned somehow, so we need to figure out if it's a known + // configuration or an error. + predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] + if predefinedPoint == nil || freezerTail != predefinedPoint.BlockNumber { + log.Error("Chain history database is pruned with unknown configuration", "tail", freezerTail) + return fmt.Errorf("unexpected database tail") + } + bc.historyPrunePoint.Store(predefinedPoint) + return nil + + case history.KeepPostMerge: + if freezerTail == 0 && latest != 0 { + // This is the case where a user is trying to run with --history.chain + // postmerge directly on an existing DB. We could just trigger the pruning + // here, but it'd be a bit dangerous since they may not have intended this + // action to happen. So just tell them how to do it. + log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cacheConfig.ChainHistoryMode.String())) + log.Error(fmt.Sprintf("Run 'geth prune-history' to prune pre-merge history.")) + return fmt.Errorf("history pruning requested via configuration") + } + predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] + if predefinedPoint == nil { + log.Error("Chain history pruning is not supported for this network", "genesis", bc.genesisBlock.Hash()) + return fmt.Errorf("history pruning requested for unknown network") + } else if freezerTail != predefinedPoint.BlockNumber { + log.Error("Chain history database is pruned to unknown block", "tail", freezerTail) + return fmt.Errorf("unexpected database tail") + } + bc.historyPrunePoint.Store(predefinedPoint) + return nil + + default: + return fmt.Errorf("invalid history mode: %d", bc.cacheConfig.ChainHistoryMode) + } +} + // SetHead rewinds the local chain to a new head. Depending on whether the node // was snap synced or full synced and in which state, the method will try to // delete minimal data from disk whilst retaining chain consistency. @@ -1014,7 +1070,9 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { bc.hc.SetCurrentHeader(bc.genesisBlock.Header()) bc.currentSnapBlock.Store(bc.genesisBlock.Header()) headFastBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) - return nil + + // Reset history pruning status. + return bc.initializeHistoryPruning(0) } // Export writes the active chain to the given writer. diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 4114723469..a8c2e26d18 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -410,7 +410,11 @@ func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) { // HistoryPruningCutoff returns the configured history pruning point. // Blocks before this might not be available in the database. func (bc *BlockChain) HistoryPruningCutoff() (uint64, common.Hash) { - return bc.cacheConfig.HistoryPruningCutoffNumber, bc.cacheConfig.HistoryPruningCutoffHash + pt := bc.historyPrunePoint.Load() + if pt == nil { + return 0, bc.genesisBlock.Hash() + } + return pt.BlockNumber, pt.BlockHash } // TrieDB retrieves the low level trie database used for data storage. diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 3f7c03b93c..289eff0a8f 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -4257,13 +4258,7 @@ func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) { // be persisted without the receipts and bodies; chain after should be persisted // normally. func TestInsertChainWithCutoff(t *testing.T) { - testInsertChainWithCutoff(t, 32, 32) // cutoff = 32, ancientLimit = 32 - testInsertChainWithCutoff(t, 32, 64) // cutoff = 32, ancientLimit = 64 (entire chain in ancient) - testInsertChainWithCutoff(t, 32, 65) // cutoff = 32, ancientLimit = 65 (64 blocks in ancient, 1 block in live) -} - -func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64) { - // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) + const chainLength = 64 // Configure and generate a sample block chain var ( @@ -4278,24 +4273,51 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64) signer = types.LatestSigner(gspec.Config) engine = beacon.New(ethash.NewFaker()) ) - _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, int(2*cutoff), func(i int, block *BlockGen) { + _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, chainLength, func(i int, block *BlockGen) { block.SetCoinbase(common.Address{0x00}) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) if err != nil { panic(err) } block.AddTx(tx) }) + + // Run the actual tests. + t.Run("cutoff-32/ancientLimit-32", func(t *testing.T) { + // cutoff = 32, ancientLimit = 32 + testInsertChainWithCutoff(t, 32, 32, gspec, blocks, receipts) + }) + t.Run("cutoff-32/ancientLimit-64", func(t *testing.T) { + // cutoff = 32, ancientLimit = 64 (entire chain in ancient) + testInsertChainWithCutoff(t, 32, 64, gspec, blocks, receipts) + }) + t.Run("cutoff-32/ancientLimit-64", func(t *testing.T) { + // cutoff = 32, ancientLimit = 65 (64 blocks in ancient, 1 block in live) + testInsertChainWithCutoff(t, 32, 65, gspec, blocks, receipts) + }) +} + +func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64, genesis *Genesis, blocks []*types.Block, receipts []types.Receipts) { + // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) + + // Add a known pruning point for the duration of the test. + ghash := genesis.ToBlock().Hash() + cutoffBlock := blocks[cutoff-1] + history.PrunePoints[ghash] = &history.PrunePoint{ + BlockNumber: cutoffBlock.NumberU64(), + BlockHash: cutoffBlock.Hash(), + } + defer func() { + delete(history.PrunePoints, ghash) + }() + + // Enable pruning in cache config. + config := DefaultCacheConfigWithScheme(rawdb.PathScheme) + config.ChainHistoryMode = history.KeepPostMerge + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false) defer db.Close() - - cutoffBlock := blocks[cutoff-1] - config := DefaultCacheConfigWithScheme(rawdb.PathScheme) - config.HistoryPruningCutoffNumber = cutoffBlock.NumberU64() - config.HistoryPruningCutoffHash = cutoffBlock.Hash() - - chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil) + chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), genesis, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil) defer chain.Stop() var ( @@ -4326,8 +4348,8 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64) t.Errorf("head header #%d: header mismatch: want: %v, got: %v", headHeader.Number, blocks[len(blocks)-1].Hash(), headHeader.Hash()) } headBlock := chain.CurrentBlock() - if headBlock.Hash() != gspec.ToBlock().Hash() { - t.Errorf("head block #%d: header mismatch: want: %v, got: %v", headBlock.Number, gspec.ToBlock().Hash(), headBlock.Hash()) + if headBlock.Hash() != ghash { + t.Errorf("head block #%d: header mismatch: want: %v, got: %v", headBlock.Number, ghash, headBlock.Hash()) } // Iterate over all chain data components, and cross reference diff --git a/eth/ethconfig/historymode.go b/core/history/historymode.go similarity index 79% rename from eth/ethconfig/historymode.go rename to core/history/historymode.go index a595d72feb..e735222d37 100644 --- a/eth/ethconfig/historymode.go +++ b/core/history/historymode.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package ethconfig +package history import ( "fmt" @@ -27,22 +27,22 @@ import ( type HistoryMode uint32 const ( - // AllHistory (default) means that all chain history down to genesis block will be kept. - AllHistory HistoryMode = iota + // KeepAll (default) means that all chain history down to genesis block will be kept. + KeepAll HistoryMode = iota - // PostMergeHistory sets the history pruning point to the merge activation block. - PostMergeHistory + // KeepPostMerge sets the history pruning point to the merge activation block. + KeepPostMerge ) func (m HistoryMode) IsValid() bool { - return m <= PostMergeHistory + return m <= KeepPostMerge } func (m HistoryMode) String() string { switch m { - case AllHistory: + case KeepAll: return "all" - case PostMergeHistory: + case KeepPostMerge: return "postmerge" default: return fmt.Sprintf("invalid HistoryMode(%d)", m) @@ -61,24 +61,24 @@ func (m HistoryMode) MarshalText() ([]byte, error) { func (m *HistoryMode) UnmarshalText(text []byte) error { switch string(text) { case "all": - *m = AllHistory + *m = KeepAll case "postmerge": - *m = PostMergeHistory + *m = KeepPostMerge default: return fmt.Errorf(`unknown sync mode %q, want "all" or "postmerge"`, text) } return nil } -type HistoryPrunePoint struct { +type PrunePoint struct { BlockNumber uint64 BlockHash common.Hash } -// HistoryPrunePoints contains the pre-defined history pruning cutoff blocks for known networks. +// PrunePoints the pre-defined history pruning cutoff blocks for known networks. // They point to the first post-merge block. Any pruning should truncate *up to* but excluding // given block. -var HistoryPrunePoints = map[common.Hash]*HistoryPrunePoint{ +var PrunePoints = map[common.Hash]*PrunePoint{ // mainnet params.MainnetGenesisHash: { BlockNumber: 15537393, @@ -91,7 +91,7 @@ var HistoryPrunePoints = map[common.Hash]*HistoryPrunePoint{ }, } -// PrunedHistoryError is returned when the requested history is pruned. +// PrunedHistoryError is returned by APIs when the requested history is pruned. type PrunedHistoryError struct{} func (e *PrunedHistoryError) Error() string { return "pruned history unavailable" } diff --git a/eth/api_backend.go b/eth/api_backend.go index b39dd4cbdb..182c081d2b 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -29,12 +29,12 @@ import ( "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/filtermaps" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" @@ -156,7 +156,7 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe } block := b.eth.blockchain.GetBlockByNumber(bn) if block == nil && bn < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return block, nil } @@ -168,7 +168,7 @@ func (b *EthAPIBackend) BlockByHash(ctx context.Context, hash common.Hash) (*typ } block := b.eth.blockchain.GetBlock(hash, *number) if block == nil && *number < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return block, nil } @@ -181,7 +181,7 @@ func (b *EthAPIBackend) GetBody(ctx context.Context, hash common.Hash, number rp body := b.eth.blockchain.GetBody(hash) if body == nil { if uint64(number) < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return nil, errors.New("block body not found") } @@ -203,7 +203,7 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r block := b.eth.blockchain.GetBlock(hash, header.Number.Uint64()) if block == nil { if header.Number.Uint64() < b.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return nil, errors.New("header found, but block body is missing") } diff --git a/eth/backend.go b/eth/backend.go index c5dec77962..6d1b6bae99 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -145,7 +145,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { // Here we determine genesis hash and active ChainConfig. // We need these to figure out the consensus parameters and to set up history pruning. - chainConfig, genesisHash, err := core.LoadChainConfig(chainDb, config.Genesis) + chainConfig, _, err := core.LoadChainConfig(chainDb, config.Genesis) if err != nil { return nil, err } @@ -153,22 +153,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - - // Validate history pruning configuration. - var ( - cutoffNumber uint64 - cutoffHash common.Hash - ) - if config.HistoryMode == ethconfig.PostMergeHistory { - prunecfg, ok := ethconfig.HistoryPrunePoints[genesisHash] - if !ok { - return nil, fmt.Errorf("no history pruning point is defined for genesis %x", genesisHash) - } - cutoffNumber = prunecfg.BlockNumber - cutoffHash = prunecfg.BlockHash - log.Info("Chain cutoff configured", "number", cutoffNumber, "hash", cutoffHash) - } - // Set networkID to chainID by default. networkID := config.NetworkId if networkID == 0 { @@ -195,6 +179,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } log.Info("Initialising Ethereum protocol", "network", networkID, "dbversion", dbVer) + // Create BlockChain object. if !config.SkipBcVersionCheck { if bcVersion != nil && *bcVersion > core.BlockChainVersion { return nil, fmt.Errorf("database version is v%d, Geth %s only supports v%d", *bcVersion, version.WithMeta, core.BlockChainVersion) @@ -210,17 +195,16 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { EnablePreimageRecording: config.EnablePreimageRecording, } cacheConfig = &core.CacheConfig{ - TrieCleanLimit: config.TrieCleanCache, - TrieCleanNoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - TrieDirtyDisabled: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, - SnapshotLimit: config.SnapshotCache, - Preimages: config.Preimages, - StateHistory: config.StateHistory, - StateScheme: scheme, - HistoryPruningCutoffNumber: cutoffNumber, - HistoryPruningCutoffHash: cutoffHash, + TrieCleanLimit: config.TrieCleanCache, + TrieCleanNoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + TrieDirtyDisabled: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, + SnapshotLimit: config.SnapshotCache, + Preimages: config.Preimages, + StateHistory: config.StateHistory, + StateScheme: scheme, + ChainHistoryMode: config.HistoryMode, } ) if config.VMTrace != "" { @@ -246,6 +230,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } + + // Initialize filtermaps log index. fmConfig := filtermaps.Config{ History: config.LogHistory, Disabled: config.LogNoHistory, @@ -261,6 +247,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { eth.filterMaps = filtermaps.NewFilterMaps(chainDb, chainView, historyCutoff, finalBlock, filtermaps.DefaultParams, fmConfig) eth.closeFilterMaps = make(chan chan struct{}) + // TxPool if config.TxPool.Journal != "" { config.TxPool.Journal = stack.ResolvePath(config.TxPool.Journal) } @@ -285,6 +272,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { eth.localTxTracker = locals.New(config.TxPool.Journal, rejournal, eth.blockchain.Config(), eth.txPool) stack.RegisterLifecycle(eth.localTxTracker) } + // Permit the downloader to use the trie cache allowance during fast sync cacheLimit := cacheConfig.TrieCleanLimit + cacheConfig.TrieDirtyLimit + cacheConfig.SnapshotLimit if eth.handler, err = newHandler(&handlerConfig{ diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 365857347c..5e19824135 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/eth/gasprice" @@ -48,7 +49,7 @@ var FullNodeGPO = gasprice.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ - HistoryMode: AllHistory, + HistoryMode: history.KeepAll, SyncMode: SnapSync, NetworkId: 0, // enable auto configuration of networkID == chainID TxLookupLimit: 2350000, @@ -84,7 +85,7 @@ type Config struct { SyncMode SyncMode // HistoryMode configures chain history retention. - HistoryMode HistoryMode + HistoryMode history.HistoryMode // This can be set to list of enrtree:// URLs which will be queried for // nodes to connect to. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index bcfe8a31d6..cdcddcc772 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -7,6 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/eth/gasprice" @@ -19,13 +20,16 @@ func (c Config) MarshalTOML() (interface{}, error) { Genesis *core.Genesis `toml:",omitempty"` NetworkId uint64 SyncMode SyncMode - HistoryMode HistoryMode + HistoryMode history.HistoryMode EthDiscoveryURLs []string SnapDiscoveryURLs []string NoPruning bool NoPrefetch bool - TxLookupLimit uint64 `toml:",omitempty"` - TransactionHistory uint64 `toml:",omitempty"` + TxLookupLimit uint64 `toml:",omitempty"` + TransactionHistory uint64 `toml:",omitempty"` + LogHistory uint64 `toml:",omitempty"` + LogNoHistory bool `toml:",omitempty"` + LogExportCheckpoints string StateHistory uint64 `toml:",omitempty"` StateScheme string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` @@ -63,6 +67,9 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.NoPrefetch = c.NoPrefetch enc.TxLookupLimit = c.TxLookupLimit enc.TransactionHistory = c.TransactionHistory + enc.LogHistory = c.LogHistory + enc.LogNoHistory = c.LogNoHistory + enc.LogExportCheckpoints = c.LogExportCheckpoints enc.StateHistory = c.StateHistory enc.StateScheme = c.StateScheme enc.RequiredBlocks = c.RequiredBlocks @@ -97,13 +104,16 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { Genesis *core.Genesis `toml:",omitempty"` NetworkId *uint64 SyncMode *SyncMode - HistoryMode *HistoryMode + HistoryMode *history.HistoryMode EthDiscoveryURLs []string SnapDiscoveryURLs []string NoPruning *bool NoPrefetch *bool - TxLookupLimit *uint64 `toml:",omitempty"` - TransactionHistory *uint64 `toml:",omitempty"` + TxLookupLimit *uint64 `toml:",omitempty"` + TransactionHistory *uint64 `toml:",omitempty"` + LogHistory *uint64 `toml:",omitempty"` + LogNoHistory *bool `toml:",omitempty"` + LogExportCheckpoints *string StateHistory *uint64 `toml:",omitempty"` StateScheme *string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` @@ -164,6 +174,15 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TransactionHistory != nil { c.TransactionHistory = *dec.TransactionHistory } + if dec.LogHistory != nil { + c.LogHistory = *dec.LogHistory + } + if dec.LogNoHistory != nil { + c.LogNoHistory = *dec.LogNoHistory + } + if dec.LogExportCheckpoints != nil { + c.LogExportCheckpoints = *dec.LogExportCheckpoints + } if dec.StateHistory != nil { c.StateHistory = *dec.StateHistory } diff --git a/eth/filters/api.go b/eth/filters/api.go index 6593cbef27..864dfd3746 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -28,8 +28,8 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rpc" ) @@ -360,7 +360,7 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type return nil, errInvalidBlockRange } if begin > 0 && begin < int64(api.events.backend.HistoryPruningCutoff()) { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } // Construct the range filter filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index e44b37d047..78c80d8f26 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -26,8 +26,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/filtermaps" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -88,7 +88,7 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { return nil, errors.New("unknown block") } if header.Number.Uint64() < f.sys.backend.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } return f.blockLogs(ctx, header) } diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index aca3c03ad6..b787f1067b 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -30,8 +30,8 @@ import ( "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/filtermaps" + "github.com/ethereum/go-ethereum/core/history" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" @@ -311,7 +311,7 @@ func (es *EventSystem) SubscribeLogs(crit ethereum.FilterQuery, logs chan []*typ } // Queries beyond the pruning cutoff are not supported. if uint64(from) < es.backend.HistoryPruningCutoff() { - return nil, ðconfig.PrunedHistoryError{} + return nil, &history.PrunedHistoryError{} } // only interested in new mined logs From 476f117211d0618ac3fa762a347089005fa90d2e Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 15 Apr 2025 20:34:34 +0800 Subject: [PATCH 02/12] all: remove martin from CODEOWNERS (#31637) Thank you, @holiman, for being an integral part of the Go-Ethereum and for your invaluable contributions over the years. This will always be your home and you're welcome back anytime! --- .github/CODEOWNERS | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b569180852..681b5e7c66 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -9,12 +9,11 @@ beacon/light/ @zsfelfoldi beacon/merkle/ @zsfelfoldi beacon/types/ @zsfelfoldi @fjl beacon/params/ @zsfelfoldi @fjl -cmd/clef/ @holiman -cmd/evm/ @holiman @MariusVanDerWijden @lightclient -core/state/ @rjl493456442 @holiman -crypto/ @gballet @jwasinger @holiman @fjl -core/ @holiman @rjl493456442 -eth/ @holiman @rjl493456442 +cmd/evm/ @MariusVanDerWijden @lightclient +core/state/ @rjl493456442 +crypto/ @gballet @jwasinger @fjl +core/ @rjl493456442 +eth/ @rjl493456442 eth/catalyst/ @MariusVanDerWijden @lightclient @fjl @jwasinger eth/tracers/ @s1na ethclient/ @fjl @@ -26,11 +25,9 @@ core/tracing/ @s1na graphql/ @s1na internal/ethapi/ @fjl @s1na @lightclient internal/era/ @lightclient -metrics/ @holiman -miner/ @MariusVanDerWijden @holiman @fjl @rjl493456442 +miner/ @MariusVanDerWijden @fjl @rjl493456442 node/ @fjl p2p/ @fjl @zsfelfoldi rlp/ @fjl -params/ @fjl @holiman @karalabe @gballet @rjl493456442 @zsfelfoldi -rpc/ @fjl @holiman -signer/ @holiman +params/ @fjl @karalabe @gballet @rjl493456442 @zsfelfoldi +rpc/ @fjl From 6928ec5d924604cc403a74ef3bbca29505443ab6 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Tue, 15 Apr 2025 20:40:30 +0200 Subject: [PATCH 03/12] p2p: fix dial metrics not picking up the right error (#31621) Our metrics related to dial errors were off. The original error was not wrapped, so the caller function had no chance of picking it up. Therefore the most common error, which is "TooManyPeers", was not correctly counted. The metrics were originally introduced in https://github.com/ethereum/go-ethereum/pull/27621 I was thinking of various possible solutions. - the one proposed here wraps both the new error and the origial error. It is not a pattern we use in other parts of the code, but works. This is maybe the smallest possible change. - as an alternate, I could write a proper `errProtoHandshakeError` with it's own wrapped error - finally, I'm not even sure we need `errProtoHandshakeError`, maybe we could just pass up the original error. --------- Signed-off-by: Csaba Kiraly Co-authored-by: Felix Lange --- p2p/metrics.go | 41 ++++++++++++++++++++++++----------------- p2p/server.go | 12 ++++++++---- p2p/server_test.go | 4 ++-- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/p2p/metrics.go b/p2p/metrics.go index 8c9804206b..29c2acb0cb 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -49,7 +49,7 @@ var ( serveSuccessMeter = metrics.NewRegisteredMeter("p2p/serves/success", nil) dialMeter = metrics.NewRegisteredMeter("p2p/dials", nil) dialSuccessMeter = metrics.NewRegisteredMeter("p2p/dials/success", nil) - dialConnectionError = metrics.NewRegisteredMeter("p2p/dials/error/connection", nil) + dialConnectionError = metrics.NewRegisteredMeter("p2p/dials/error/connection", nil) // dial timeout; no route to host; connection refused; network is unreachable // count peers that stayed connected for at least 1 min serve1MinSuccessMeter = metrics.NewRegisteredMeter("p2p/serves/success/1min", nil) @@ -61,34 +61,41 @@ var ( dialSelf = metrics.NewRegisteredMeter("p2p/dials/error/self", nil) dialUselessPeer = metrics.NewRegisteredMeter("p2p/dials/error/useless", nil) dialUnexpectedIdentity = metrics.NewRegisteredMeter("p2p/dials/error/id/unexpected", nil) - dialEncHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/enc", nil) - dialProtoHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/proto", nil) + dialEncHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/enc", nil) // EOF; connection reset during handshake; message too big; i/o timeout + dialProtoHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/proto", nil) // EOF + + // capture the rest of errors that are not handled by the above meters + dialOtherError = metrics.NewRegisteredMeter("p2p/dials/error/other", nil) ) -// markDialError matches errors that occur while setting up a dial connection -// to the corresponding meter. +// markDialError matches errors that occur while setting up a dial connection to the +// corresponding meter. We don't maintain meters for evert possible error, just for +// the most interesting ones. func markDialError(err error) { if !metrics.Enabled() { return } - if err2 := errors.Unwrap(err); err2 != nil { - err = err2 - } - switch err { - case DiscTooManyPeers: + + var reason DiscReason + var handshakeErr *protoHandshakeError + d := errors.As(err, &reason) + switch { + case d && reason == DiscTooManyPeers: dialTooManyPeers.Mark(1) - case DiscAlreadyConnected: + case d && reason == DiscAlreadyConnected: dialAlreadyConnected.Mark(1) - case DiscSelf: + case d && reason == DiscSelf: dialSelf.Mark(1) - case DiscUselessPeer: + case d && reason == DiscUselessPeer: dialUselessPeer.Mark(1) - case DiscUnexpectedIdentity: + case d && reason == DiscUnexpectedIdentity: dialUnexpectedIdentity.Mark(1) - case errEncHandshakeError: - dialEncHandshakeError.Mark(1) - case errProtoHandshakeError: + case errors.As(err, &handshakeErr): dialProtoHandshakeError.Mark(1) + case errors.Is(err, errEncHandshakeError): + dialEncHandshakeError.Mark(1) + default: + dialOtherError.Mark(1) } } diff --git a/p2p/server.go b/p2p/server.go index 4e72e29fa0..d9105976dd 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -66,11 +66,15 @@ const ( ) var ( - errServerStopped = errors.New("server stopped") - errEncHandshakeError = errors.New("rlpx enc error") - errProtoHandshakeError = errors.New("rlpx proto error") + errServerStopped = errors.New("server stopped") + errEncHandshakeError = errors.New("rlpx enc error") ) +type protoHandshakeError struct{ err error } + +func (e *protoHandshakeError) Error() string { return fmt.Sprintf("rlpx proto error: %v", e.err) } +func (e *protoHandshakeError) Unwrap() error { return e.err } + // Server manages all peer connections. type Server struct { // Config fields may not be modified while the server is running. @@ -907,7 +911,7 @@ func (srv *Server) setupConn(c *conn, dialDest *enode.Node) error { phs, err := c.doProtoHandshake(srv.ourHandshake) if err != nil { clog.Trace("Failed p2p handshake", "err", err) - return fmt.Errorf("%w: %v", errProtoHandshakeError, err) + return &protoHandshakeError{err: err} } if id := c.node.ID(); !bytes.Equal(crypto.Keccak256(phs.ID), id[:]) { clog.Trace("Wrong devp2p handshake identity", "phsid", hex.EncodeToString(phs.ID)) diff --git a/p2p/server_test.go b/p2p/server_test.go index a0491e984a..d42926cf4c 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -410,11 +410,11 @@ func TestServerSetupConn(t *testing.T) { wantCloseErr: DiscUnexpectedIdentity, }, { - tt: &setupTransport{pubkey: clientpub, protoHandshakeErr: errProtoHandshakeError}, + tt: &setupTransport{pubkey: clientpub, protoHandshakeErr: DiscTooManyPeers}, dialDest: enode.NewV4(clientpub, nil, 0, 0), flags: dynDialedConn, wantCalls: "doEncHandshake,doProtoHandshake,close,", - wantCloseErr: errProtoHandshakeError, + wantCloseErr: DiscTooManyPeers, }, { tt: &setupTransport{pubkey: srvpub, phs: protoHandshake{ID: crypto.FromECDSAPub(srvpub)[1:]}}, From e3e9d7ccb677fe687f11d63c96be4b1b09211695 Mon Sep 17 00:00:00 2001 From: Delweng Date: Wed, 16 Apr 2025 15:50:05 +0800 Subject: [PATCH 04/12] cmd/geth: remove the unused bloomfilter.size flag (#31646) --- cmd/geth/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index ab46e059f3..289030ae65 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -100,7 +100,6 @@ var ( utils.LightNoSyncServeFlag, // deprecated utils.EthRequiredBlocksFlag, utils.LegacyWhitelistFlag, // deprecated - utils.BloomFilterSizeFlag, utils.CacheFlag, utils.CacheDatabaseFlag, utils.CacheTrieFlag, From ebb3eb29d33770543fc0864a82e47a24d1c3e460 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Wed, 16 Apr 2025 23:30:13 +0200 Subject: [PATCH 05/12] core/filtermaps: fix map renderer reorg issue (#31642) This PR fixes a bug in the map renderer that sometimes used an obsolete block log value pointer to initialize the iterator for rendering from a snapshot. This bug was triggered by chain reorgs and sometimes caused indexing errors and invalid search results. A few other conditions are also made safer that were not reported to cause issues yet but could potentially be unsafe in some corner cases. A new unit test is also added that reproduced the bug but passes with the new fixes. Fixes https://github.com/ethereum/go-ethereum/issues/31593 Might also fix https://github.com/ethereum/go-ethereum/issues/31589 though this issue has not been reproduced yet, but it appears to be related to a log index database corruption around a specific block, similarly to the other issue. Note that running this branch resets and regenerates the log index database. For this purpose a `Version` field has been added to `rawdb.FilterMapsRange` which will also make this easier in the future if a breaking database change is needed or the existing one is considered potentially broken due to a bug, like in this case. --- core/filtermaps/filtermaps.go | 27 +++++-- core/filtermaps/indexer_test.go | 110 +++++++++++++++++++++++++++++ core/filtermaps/map_renderer.go | 45 +++++------- core/filtermaps/matcher_backend.go | 3 + core/rawdb/accessors_indexes.go | 1 + 5 files changed, 153 insertions(+), 33 deletions(-) diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index 18b1c7dc79..fa2d6e3ffb 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -50,6 +50,7 @@ var ( ) const ( + databaseVersion = 1 // reindexed if database version does not match cachedLastBlocks = 1000 // last block of map pointers cachedLvPointers = 1000 // first log value pointer of block pointers cachedBaseRows = 100 // groups of base layer filter row data @@ -138,13 +139,25 @@ type FilterMaps struct { // as transparent (uncached/unchanged). type filterMap []FilterRow -// copy returns a copy of the given filter map. Note that the row slices are -// copied but their contents are not. This permits extending the rows further +// fastCopy returns a copy of the given filter map. Note that the row slices are +// copied but their contents are not. This permits appending to the rows further // (which happens during map rendering) without affecting the validity of // copies made for snapshots during rendering. -func (fm filterMap) copy() filterMap { +// Appending to the rows of both the original map and the fast copy, or two fast +// copies of the same map would result in data corruption, therefore a fast copy +// should always be used in a read only way. +func (fm filterMap) fastCopy() filterMap { + return slices.Clone(fm) +} + +// fullCopy returns a copy of the given filter map, also making a copy of each +// individual filter row, ensuring that a modification to either one will never +// affect the other. +func (fm filterMap) fullCopy() filterMap { c := make(filterMap, len(fm)) - copy(c, fm) + for i, row := range fm { + c[i] = slices.Clone(row) + } return c } @@ -207,8 +220,9 @@ type Config struct { // NewFilterMaps creates a new FilterMaps and starts the indexer. func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, finalBlock uint64, params Params, config Config) *FilterMaps { rs, initialized, err := rawdb.ReadFilterMapsRange(db) - if err != nil { - log.Error("Error reading log index range", "error", err) + if err != nil || rs.Version != databaseVersion { + rs, initialized = rawdb.FilterMapsRange{}, false + log.Warn("Invalid log index database version; resetting log index") } params.deriveFields() f := &FilterMaps{ @@ -437,6 +451,7 @@ func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, ne f.updateMatchersValidRange() if newRange.initialized { rs := rawdb.FilterMapsRange{ + Version: databaseVersion, HeadIndexed: newRange.headIndexed, HeadDelimiter: newRange.headDelimiter, BlocksFirst: newRange.blocks.First(), diff --git a/core/filtermaps/indexer_test.go b/core/filtermaps/indexer_test.go index a02f8d2459..4dddd27087 100644 --- a/core/filtermaps/indexer_test.go +++ b/core/filtermaps/indexer_test.go @@ -17,8 +17,10 @@ package filtermaps import ( + "context" crand "crypto/rand" "crypto/sha256" + "encoding/binary" "math/big" "math/rand" "sync" @@ -31,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" ) var testParams = Params{ @@ -104,6 +107,7 @@ func TestIndexerRandomRange(t *testing.T) { fork, head = rand.Intn(len(forks)), rand.Intn(1001) ts.chain.setCanonicalChain(forks[fork][:head+1]) case 2: + checkSnapshot = false if head < 1000 { checkSnapshot = !noHistory && head != 0 // no snapshot generated for block 0 // add blocks after the current head @@ -158,6 +162,63 @@ func TestIndexerRandomRange(t *testing.T) { } } +func TestIndexerMatcherView(t *testing.T) { + testIndexerMatcherView(t, false) +} + +func TestIndexerMatcherViewWithConcurrentRead(t *testing.T) { + testIndexerMatcherView(t, true) +} + +func testIndexerMatcherView(t *testing.T, concurrentRead bool) { + ts := newTestSetup(t) + defer ts.close() + + forks := make([][]common.Hash, 20) + hashes := make([]common.Hash, 20) + ts.chain.addBlocks(100, 5, 2, 4, true) + ts.setHistory(0, false) + for i := range forks { + if i != 0 { + ts.chain.setHead(100 - i) + ts.chain.addBlocks(i, 5, 2, 4, true) + } + ts.fm.WaitIdle() + forks[i] = ts.chain.getCanonicalChain() + hashes[i] = ts.matcherViewHash() + } + fork := len(forks) - 1 + for i := 0; i < 5000; i++ { + oldFork := fork + fork = rand.Intn(len(forks)) + stopCh := make(chan chan struct{}) + if concurrentRead { + go func() { + for { + ts.matcherViewHash() + select { + case ch := <-stopCh: + close(ch) + return + default: + } + } + }() + } + ts.chain.setCanonicalChain(forks[fork]) + ts.fm.WaitIdle() + if concurrentRead { + ch := make(chan struct{}) + stopCh <- ch + <-ch + } + hash := ts.matcherViewHash() + if hash != hashes[fork] { + t.Fatalf("Matcher view hash mismatch when switching from for %d to %d", oldFork, fork) + } + } +} + func TestIndexerCompareDb(t *testing.T) { ts := newTestSetup(t) defer ts.close() @@ -291,6 +352,55 @@ func (ts *testSetup) fmDbHash() common.Hash { return result } +func (ts *testSetup) matcherViewHash() common.Hash { + mb := ts.fm.NewMatcherBackend() + defer mb.Close() + + ctx := context.Background() + params := mb.GetParams() + hasher := sha256.New() + var headPtr uint64 + for b := uint64(0); ; b++ { + lvptr, err := mb.GetBlockLvPointer(ctx, b) + if err != nil || (b > 0 && lvptr == headPtr) { + break + } + var enc [8]byte + binary.LittleEndian.PutUint64(enc[:], lvptr) + hasher.Write(enc[:]) + headPtr = lvptr + } + headMap := uint32(headPtr >> params.logValuesPerMap) + var enc [12]byte + for r := uint32(0); r < params.mapHeight; r++ { + binary.LittleEndian.PutUint32(enc[:4], r) + for m := uint32(0); m <= headMap; m++ { + binary.LittleEndian.PutUint32(enc[4:8], m) + row, _ := mb.GetFilterMapRow(ctx, m, r, false) + for _, v := range row { + binary.LittleEndian.PutUint32(enc[8:], v) + hasher.Write(enc[:]) + } + } + } + var hash common.Hash + hasher.Sum(hash[:0]) + for i := 0; i < 50; i++ { + hasher.Reset() + hasher.Write(hash[:]) + lvptr := binary.LittleEndian.Uint64(hash[:8]) % headPtr + if log, _ := mb.GetLogByLvIndex(ctx, lvptr); log != nil { + enc, err := rlp.EncodeToBytes(log) + if err != nil { + panic(err) + } + hasher.Write(enc) + } + hasher.Sum(hash[:0]) + } + return hash +} + func (ts *testSetup) close() { if ts.fm != nil { ts.fm.Stop() diff --git a/core/filtermaps/map_renderer.go b/core/filtermaps/map_renderer.go index 28f943abb3..cd960ad31c 100644 --- a/core/filtermaps/map_renderer.go +++ b/core/filtermaps/map_renderer.go @@ -84,7 +84,7 @@ func (f *FilterMaps) renderMapsBefore(renderBefore uint32) (*mapRenderer, error) if err != nil { return nil, err } - if snapshot := f.lastCanonicalSnapshotBefore(renderBefore); snapshot != nil && snapshot.mapIndex >= nextMap { + if snapshot := f.lastCanonicalSnapshotOfMap(nextMap); snapshot != nil { return f.renderMapsFromSnapshot(snapshot) } if nextMap >= renderBefore { @@ -97,14 +97,14 @@ func (f *FilterMaps) renderMapsBefore(renderBefore uint32) (*mapRenderer, error) // snapshot made at a block boundary. func (f *FilterMaps) renderMapsFromSnapshot(cp *renderedMap) (*mapRenderer, error) { f.testSnapshotUsed = true - iter, err := f.newLogIteratorFromBlockDelimiter(cp.lastBlock) + iter, err := f.newLogIteratorFromBlockDelimiter(cp.lastBlock, cp.headDelimiter) if err != nil { return nil, fmt.Errorf("failed to create log iterator from block delimiter %d: %v", cp.lastBlock, err) } return &mapRenderer{ f: f, currentMap: &renderedMap{ - filterMap: cp.filterMap.copy(), + filterMap: cp.filterMap.fullCopy(), mapIndex: cp.mapIndex, lastBlock: cp.lastBlock, blockLvPtrs: cp.blockLvPtrs, @@ -137,14 +137,14 @@ func (f *FilterMaps) renderMapsFromMapBoundary(firstMap, renderBefore uint32, st }, nil } -// lastCanonicalSnapshotBefore returns the latest cached snapshot that matches -// the current targetView. -func (f *FilterMaps) lastCanonicalSnapshotBefore(renderBefore uint32) *renderedMap { +// lastCanonicalSnapshotOfMap returns the latest cached snapshot of the given map +// that is also consistent with the current targetView. +func (f *FilterMaps) lastCanonicalSnapshotOfMap(mapIndex uint32) *renderedMap { var best *renderedMap for _, blockNumber := range f.renderSnapshots.Keys() { if cp, _ := f.renderSnapshots.Get(blockNumber); cp != nil && blockNumber < f.indexedRange.blocks.AfterLast() && blockNumber <= f.targetView.headNumber && f.targetView.getBlockId(blockNumber) == cp.lastBlockId && - cp.mapIndex < renderBefore && (best == nil || blockNumber > best.lastBlock) { + cp.mapIndex == mapIndex && (best == nil || blockNumber > best.lastBlock) { best = cp } } @@ -171,10 +171,9 @@ func (f *FilterMaps) lastCanonicalMapBoundaryBefore(renderBefore uint32) (nextMa if err != nil { return 0, 0, 0, fmt.Errorf("failed to retrieve last block of reverse iterated map %d: %v", mapIndex, err) } - if lastBlock >= f.indexedView.headNumber || lastBlock >= f.targetView.headNumber || - lastBlockId != f.targetView.getBlockId(lastBlock) { - // map is not full or inconsistent with targetView; roll back - continue + if (f.indexedRange.headIndexed && mapIndex >= f.indexedRange.maps.Last()) || + lastBlock >= f.targetView.headNumber || lastBlockId != f.targetView.getBlockId(lastBlock) { + continue // map is not full or inconsistent with targetView; roll back } lvPtr, err := f.getBlockLvPointer(lastBlock) if err != nil { @@ -257,11 +256,14 @@ func (f *FilterMaps) loadHeadSnapshot() error { // makeSnapshot creates a snapshot of the current state of the rendered map. func (r *mapRenderer) makeSnapshot() { - r.f.renderSnapshots.Add(r.iterator.blockNumber, &renderedMap{ - filterMap: r.currentMap.filterMap.copy(), + if r.iterator.blockNumber != r.currentMap.lastBlock || r.iterator.chainView != r.f.targetView { + panic("iterator state inconsistent with current rendered map") + } + r.f.renderSnapshots.Add(r.currentMap.lastBlock, &renderedMap{ + filterMap: r.currentMap.filterMap.fastCopy(), mapIndex: r.currentMap.mapIndex, - lastBlock: r.iterator.blockNumber, - lastBlockId: r.f.targetView.getBlockId(r.currentMap.lastBlock), + lastBlock: r.currentMap.lastBlock, + lastBlockId: r.iterator.chainView.getBlockId(r.currentMap.lastBlock), blockLvPtrs: r.currentMap.blockLvPtrs, finished: true, headDelimiter: r.iterator.lvIndex, @@ -661,24 +663,13 @@ var errUnindexedRange = errors.New("unindexed range") // newLogIteratorFromBlockDelimiter creates a logIterator starting at the // given block's first log value entry (the block delimiter), according to the // current targetView. -func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber uint64) (*logIterator, error) { +func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber, lvIndex uint64) (*logIterator, error) { if blockNumber > f.targetView.headNumber { return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", blockNumber, f.targetView.headNumber) } if !f.indexedRange.blocks.Includes(blockNumber) { return nil, errUnindexedRange } - var lvIndex uint64 - if f.indexedRange.headIndexed && blockNumber+1 == f.indexedRange.blocks.AfterLast() { - lvIndex = f.indexedRange.headDelimiter - } else { - var err error - lvIndex, err = f.getBlockLvPointer(blockNumber + 1) - if err != nil { - return nil, fmt.Errorf("failed to retrieve log value pointer of block %d after delimiter: %v", blockNumber+1, err) - } - lvIndex-- - } finished := blockNumber == f.targetView.headNumber l := &logIterator{ chainView: f.targetView, diff --git a/core/filtermaps/matcher_backend.go b/core/filtermaps/matcher_backend.go index 01bae7bb22..335ac84551 100644 --- a/core/filtermaps/matcher_backend.go +++ b/core/filtermaps/matcher_backend.go @@ -75,6 +75,9 @@ func (fm *FilterMapsMatcherBackend) Close() { // on write. // GetFilterMapRow implements MatcherBackend. func (fm *FilterMapsMatcherBackend) GetFilterMapRow(ctx context.Context, mapIndex, rowIndex uint32, baseLayerOnly bool) (FilterRow, error) { + fm.f.indexLock.RLock() + defer fm.f.indexLock.RUnlock() + return fm.f.getFilterMapRow(mapIndex, rowIndex, baseLayerOnly) } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index c413839b7b..1a5c414c8e 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -434,6 +434,7 @@ func DeleteBlockLvPointers(db ethdb.KeyValueStore, blocks common.Range[uint64], // FilterMapsRange is a storage representation of the block range covered by the // filter maps structure and the corresponting log value index range. type FilterMapsRange struct { + Version uint32 HeadIndexed bool HeadDelimiter uint64 BlocksFirst, BlocksAfterLast uint64 From 846d578cc3c9629f226f70e39c25c461e4cbf143 Mon Sep 17 00:00:00 2001 From: maskpp Date: Thu, 17 Apr 2025 05:46:38 +0800 Subject: [PATCH 06/12] core/state: fix log format (#31610) Log `key` in hexadecimal string format. --- core/state/dump.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/dump.go b/core/state/dump.go index c9aad4f8e2..11b5c32782 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -188,7 +188,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] c.OnAccount(address, account) accounts++ if time.Since(logged) > 8*time.Second { - log.Info("Trie dumping in progress", "at", it.Key, "accounts", accounts, + log.Info("Trie dumping in progress", "at", common.Bytes2Hex(it.Key), "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start))) logged = time.Now() } From 87974974a7b3fcce873851205b889f7d839d7fb7 Mon Sep 17 00:00:00 2001 From: Miro Date: Wed, 16 Apr 2025 22:36:53 -0400 Subject: [PATCH 07/12] core/txpool/legacypool: fix data race of txlookup access (#31641) --- core/txpool/legacypool/legacypool.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 04f1a2234c..7bf360ff65 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1827,6 +1827,16 @@ func (t *lookup) Remove(hash common.Hash) { delete(t.txs, hash) } +// Clear resets the lookup structure, removing all stored entries. +func (t *lookup) Clear() { + t.lock.Lock() + defer t.lock.Unlock() + + t.slots = 0 + t.txs = make(map[common.Hash]*types.Transaction) + t.auths = make(map[common.Address][]common.Hash) +} + // TxsBelowTip finds all remote transactions below the given tip threshold. func (t *lookup) TxsBelowTip(threshold *big.Int) types.Transactions { found := make(types.Transactions, 0, 128) @@ -1923,7 +1933,7 @@ func (pool *LegacyPool) Clear() { for addr := range pool.queue { pool.reserver.Release(addr) } - pool.all = newLookup() + pool.all.Clear() pool.priced = newPricedList(pool.all) pool.pending = make(map[common.Address]*list) pool.queue = make(map[common.Address]*list) From cb21177aa8b15861067c430f12439e40fc1ad124 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 17 Apr 2025 04:39:21 +0200 Subject: [PATCH 08/12] core: fix history pruning initialization for empty DB (#31656) This fixes an issue where running geth with `--history.chain postmerge` would not work on an empty database. ``` ERROR[04-16|23:11:12.913] Chain history database is pruned to unknown block tail=0 Fatal: Failed to register the Ethereum service: unexpected database tail ``` --- core/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 901a93315a..203dcd2693 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -620,7 +620,7 @@ func (bc *BlockChain) initializeHistoryPruning(latest uint64) error { if predefinedPoint == nil { log.Error("Chain history pruning is not supported for this network", "genesis", bc.genesisBlock.Hash()) return fmt.Errorf("history pruning requested for unknown network") - } else if freezerTail != predefinedPoint.BlockNumber { + } else if freezerTail > 0 && freezerTail != predefinedPoint.BlockNumber { log.Error("Chain history database is pruned to unknown block", "tail", freezerTail) return fmt.Errorf("unexpected database tail") } From e4448233940904e9c36f227b71f274c9df80a250 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Thu, 17 Apr 2025 10:32:40 +0200 Subject: [PATCH 09/12] core: fix sync reset in pruned nodes (#31638) This is an attempt at fixing #31601. I think what happens is the startup logic will try to get the full block body (it's `bc.loadLastState`) and fail because genesis block has been pruned from the freezer. This will cause it to keep repeating the reset logic, causing a deadlock. This can happen when due to an unsuccessful sync we don't have the state for the head (or any other state) fully, and try to redo the snap sync. --------- Co-authored-by: Gary Rong --- core/blockchain.go | 50 +++++++++++++++++++++++++++++++------------ core/txindexer.go | 19 +++++++++++++--- core/txpool/txpool.go | 14 ++++++------ 3 files changed, 60 insertions(+), 23 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 203dcd2693..c4caec66ed 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -514,19 +514,33 @@ func (bc *BlockChain) loadLastState() error { log.Warn("Empty database, resetting chain") return bc.Reset() } - // Make sure the entire head block is available - headBlock := bc.GetBlockByHash(head) + headHeader := bc.GetHeaderByHash(head) + if headHeader == nil { + // Corrupt or empty database, init from scratch + log.Warn("Head header missing, resetting chain", "hash", head) + return bc.Reset() + } + + var headBlock *types.Block + if cmp := headHeader.Number.Cmp(new(big.Int)); cmp == 1 { + // Make sure the entire head block is available. + headBlock = bc.GetBlockByHash(head) + } else if cmp == 0 { + // On a pruned node the block body might not be available. But a pruned + // block should never be the head block. The only exception is when, as + // a last resort, chain is reset to genesis. + headBlock = bc.genesisBlock + } if headBlock == nil { // Corrupt or empty database, init from scratch log.Warn("Head block missing, resetting chain", "hash", head) return bc.Reset() } // Everything seems to be fine, set as the head block - bc.currentBlock.Store(headBlock.Header()) + bc.currentBlock.Store(headHeader) headBlockGauge.Update(int64(headBlock.NumberU64())) // Restore the last known head header - headHeader := headBlock.Header() if head := rawdb.ReadHeadHeaderHash(bc.db); head != (common.Hash{}) { if header := bc.GetHeaderByHash(head); header != nil { headHeader = header @@ -642,11 +656,15 @@ func (bc *BlockChain) SetHead(head uint64) error { // Send chain head event to update the transaction pool header := bc.CurrentBlock() if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil { - // This should never happen. In practice, previously currentBlock - // contained the entire block whereas now only a "marker", so there - // is an ever so slight chance for a race we should handle. - log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) - return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + // In a pruned node the genesis block will not exist in the freezer. + // It should not happen that we set head to any other pruned block. + if header.Number.Uint64() > 0 { + // This should never happen. In practice, previously currentBlock + // contained the entire block whereas now only a "marker", so there + // is an ever so slight chance for a race we should handle. + log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) + return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + } } bc.chainHeadFeed.Send(ChainHeadEvent{Header: header}) return nil @@ -663,11 +681,15 @@ func (bc *BlockChain) SetHeadWithTimestamp(timestamp uint64) error { // Send chain head event to update the transaction pool header := bc.CurrentBlock() if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil { - // This should never happen. In practice, previously currentBlock - // contained the entire block whereas now only a "marker", so there - // is an ever so slight chance for a race we should handle. - log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) - return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + // In a pruned node the genesis block will not exist in the freezer. + // It should not happen that we set head to any other pruned block. + if header.Number.Uint64() > 0 { + // This should never happen. In practice, previously currentBlock + // contained the entire block whereas now only a "marker", so there + // is an ever so slight chance for a race we should handle. + log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash()) + return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4]) + } } bc.chainHeadFeed.Send(ChainHeadEvent{Header: header}) return nil diff --git a/core/txindexer.go b/core/txindexer.go index 31f069995b..64a2e8c49f 100644 --- a/core/txindexer.go +++ b/core/txindexer.go @@ -196,6 +196,19 @@ func (indexer *txIndexer) repair(head uint64) { } } +// resolveHead resolves the block number of the current chain head. +func (indexer *txIndexer) resolveHead() uint64 { + headBlockHash := rawdb.ReadHeadBlockHash(indexer.db) + if headBlockHash == (common.Hash{}) { + return 0 + } + headBlockNumber := rawdb.ReadHeaderNumber(indexer.db, headBlockHash) + if headBlockNumber == nil { + return 0 + } + return *headBlockNumber +} + // loop is the scheduler of the indexer, assigning indexing/unindexing tasks depending // on the received chain event. func (indexer *txIndexer) loop(chain *BlockChain) { @@ -203,9 +216,9 @@ func (indexer *txIndexer) loop(chain *BlockChain) { // Listening to chain events and manipulate the transaction indexes. var ( - stop chan struct{} // Non-nil if background routine is active - done chan struct{} // Non-nil if background routine is active - head = rawdb.ReadHeadBlock(indexer.db).NumberU64() // The latest announced chain head + stop chan struct{} // Non-nil if background routine is active + done chan struct{} // Non-nil if background routine is active + head = indexer.resolveHead() // The latest announced chain head headCh = make(chan ChainHeadEvent) sub = chain.SubscribeChainHeadEvent(headCh) diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 2ed38772ce..fc4a7be6d2 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -186,13 +186,15 @@ func (p *TxPool) loop(head *types.Header) { // Try to inject a busy marker and start a reset if successful select { case resetBusy <- struct{}{}: - statedb, err := p.chain.StateAt(newHead.Root) - if err != nil { - log.Crit("Failed to reset txpool state", "err", err) + // Updates the statedb with the new chain head. The head state may be + // unavailable if the initial state sync has not yet completed. + if statedb, err := p.chain.StateAt(newHead.Root); err != nil { + log.Error("Failed to reset txpool state", "err", err) + } else { + p.stateLock.Lock() + p.state = statedb + p.stateLock.Unlock() } - p.stateLock.Lock() - p.state = statedb - p.stateLock.Unlock() // Busy marker injected, start a new subpool reset go func(oldHead, newHead *types.Header) { From 01786f329f2fea24ad0d8308dce3ec72880ff5d7 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Thu, 17 Apr 2025 10:33:59 +0200 Subject: [PATCH 10/12] eth: fix transaction sender cache miss before broadcast (#31657) BroadcastTransactions needs the Sender address to route message flows from the same Sender address consistently to the same random subset of peers. It however spent considerable time calculating the Sender addresses, even if the Sender address was already calculated and cached in other parts of the code. Since we only need the mapping, we can use any signer, and the one that had already been used is a better choice because of cache reuse. --- eth/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/handler.go b/eth/handler.go index b2ad6effdb..8283d7d02f 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -484,7 +484,7 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { total := new(big.Int).Exp(direct, big.NewInt(2), nil) // Stabilise total peer count a bit based on sqrt peers var ( - signer = types.LatestSignerForChainID(h.chain.Config().ChainID) // Don't care about chain status, we just need *a* sender + signer = types.LatestSigner(h.chain.Config()) // Don't care about chain status, we just need *a* sender hasher = crypto.NewKeccakState() hash = make([]byte, 32) ) From 50b5f3125b0f13293eda17a8724103eafd249384 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Thu, 17 Apr 2025 02:46:00 -0600 Subject: [PATCH 11/12] params: add prague timestamp for mainnet (#31535) https://eips.ethereum.org/EIPS/eip-7600#activation Timestamp: `1746612311` Fork id: `0xc376cf8b` --- core/forkid/forkid_test.go | 19 ++++++++++--------- params/config.go | 2 ++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index 31e2b534be..413e4d77a8 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -76,8 +76,10 @@ func TestCreation(t *testing.T) { {20000000, 1681338454, ID{Hash: checksumToBytes(0xf0afd0e3), Next: 1681338455}}, // Last Gray Glacier block {20000000, 1681338455, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // First Shanghai block {30000000, 1710338134, ID{Hash: checksumToBytes(0xdce96c2d), Next: 1710338135}}, // Last Shanghai block - {40000000, 1710338135, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // First Cancun block - {50000000, 2000000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}}, // Future Cancun block + {40000000, 1710338135, ID{Hash: checksumToBytes(0x9f3d2254), Next: 1746612311}}, // First Cancun block + {30000000, 1746022486, ID{Hash: checksumToBytes(0x9f3d2254), Next: 1746612311}}, // Last Cancun block + {30000000, 1746612311, ID{Hash: checksumToBytes(0xc376cf8b), Next: 0}}, // First Prague block + {50000000, 2000000000, ID{Hash: checksumToBytes(0xc376cf8b), Next: 0}}, // Future Prague block }, }, // Sepolia test cases @@ -137,9 +139,11 @@ func TestCreation(t *testing.T) { // fork ID. func TestValidation(t *testing.T) { // Config that has not timestamp enabled + // TODO(lightclient): this always needs to be updated when a mainnet timestamp is set. legacyConfig := *params.MainnetChainConfig legacyConfig.ShanghaiTime = nil legacyConfig.CancunTime = nil + legacyConfig.PragueTime = nil tests := []struct { config *params.ChainConfig @@ -314,9 +318,7 @@ func TestValidation(t *testing.T) { // Local is mainnet Prague, remote announces Shanghai + knowledge about Cancun. Remote // is definitely out of sync. It may or may not need the Prague update, we don't know yet. - // - // TODO(karalabe): Enable this when Cancun **and** Prague is specced, update all the numbers - //{params.MainnetChainConfig, 0, 0, ID{Hash: checksumToBytes(0x3edd5b10), Next: 4370000}, nil}, + {params.MainnetChainConfig, 0, 0, ID{Hash: checksumToBytes(0x3edd5b10), Next: 1710338135}, nil}, // Local is mainnet Shanghai, remote announces Cancun. Local is out of sync, accept. {params.MainnetChainConfig, 21000000, 1700000000, ID{Hash: checksumToBytes(0x9f3d2254), Next: 0}, nil}, @@ -324,8 +326,7 @@ func TestValidation(t *testing.T) { // Local is mainnet Shanghai, remote announces Cancun, but is not aware of Prague. Local // out of sync. Local also knows about a future fork, but that is uncertain yet. // - // TODO(karalabe): Enable this when Cancun **and** Prague is specced, update remote checksum - //{params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0x00000000), Next: 0}, nil}, + {params.MainnetChainConfig, 21000000, 1678000000, ID{Hash: checksumToBytes(0xc376cf8b), Next: 0}, nil}, // Local is mainnet Cancun. remote announces Shanghai but is not aware of further forks. // Remote needs software update. @@ -342,11 +343,11 @@ func TestValidation(t *testing.T) { // Local is mainnet Shanghai, remote is random Shanghai. {params.MainnetChainConfig, 20000000, 1681338455, ID{Hash: checksumToBytes(0x12345678), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Cancun, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Prague, far in the future. Remote announces Gopherium (non existing fork) // at some future timestamp 8888888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0x9f3d2254), Next: 8888888888}, ErrLocalIncompatibleOrStale}, + {params.MainnetChainConfig, 88888888, 8888888888, ID{Hash: checksumToBytes(0xc376cf8b), Next: 8888888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Shanghai. Remote is also in Shanghai, but announces Gopherium (non existing // fork) at timestamp 1668000000, before Cancun. Local is incompatible. diff --git a/params/config.go b/params/config.go index 8f9e02583b..03b797863c 100644 --- a/params/config.go +++ b/params/config.go @@ -60,10 +60,12 @@ var ( TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 ShanghaiTime: newUint64(1681338455), CancunTime: newUint64(1710338135), + PragueTime: newUint64(1746612311), DepositContractAddress: common.HexToAddress("0x00000000219ab540356cbb839cbe05303d7705fa"), Ethash: new(EthashConfig), BlobScheduleConfig: &BlobScheduleConfig{ Cancun: DefaultCancunBlobConfig, + Prague: DefaultPragueBlobConfig, }, } // HoleskyChainConfig contains the chain parameters to run a node on the Holesky test network. From 13b157a461c88678cd4e15ca005e7b45d823431b Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Thu, 17 Apr 2025 02:46:47 -0600 Subject: [PATCH 12/12] core,params: add fork readiness indicator in logs (#31340) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #31310 This has been requested a few times in the past and I think it is a nice quality-of-life improvement for users. At a predetermined interval, there will now be a "Fork ready" log when a future fork is scheduled, but not yet active. It can only possibly print after block import, which kinda avoids the scenario where the client isn't progressing or is syncing and the user thinks it's "ready" because it sees a ready log. New output: ```console INFO [03-08|21:32:57.472] Imported new potential chain segment number=7 hash=aa24ee..f09e62 blocks=1 txs=0 mgas=0.000 elapsed="874.916µs" mgasps=0.000 snapdiffs=973.00B triediffs=7.05KiB triedirty=0.00B INFO [03-08|21:32:57.473] Ready for fork activation fork=Prague date="18 Mar 25 19:29 CET" remaining=237h57m0s timestamp=1,742,322,597 INFO [03-08|21:32:57.475] Chain head was updated number=7 hash=aa24ee..f09e62 root=19b0de..8d32f2 elapsed="129.125µs" ``` Easiest way to verify this behavior is to apply this patch and run `geth --dev --dev.period=12` ```patch diff --git a/params/config.go b/params/config.go index 9c7719d901..030c4f80e7 100644 --- a/params/config.go +++ b/params/config.go @@ -174,7 +174,7 @@ var ( ShanghaiTime: newUint64(0), CancunTime: newUint64(0), TerminalTotalDifficulty: big.NewInt(0), - PragueTime: newUint64(0), + PragueTime: newUint64(uint64(time.Now().Add(time.Hour * 300).Unix())), BlobScheduleConfig: &BlobScheduleConfig{ Cancun: DefaultCancunBlobConfig, Prague: DefaultPragueBlobConfig, ``` --- core/blockchain.go | 28 ++++++++++++++++++++++++++++ params/config.go | 17 +++++++++++++++++ params/forks/forks.go | 34 +++++++++++++++++++++++++++++++++- 3 files changed, 78 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index c4caec66ed..6667f64911 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "io" + "math" "math/big" "runtime" "slices" @@ -100,6 +101,10 @@ var ( errInvalidNewChain = errors.New("invalid new chain") ) +var ( + forkReadyInterval = 3 * time.Minute +) + const ( bodyCacheLimit = 256 blockCacheLimit = 256 @@ -275,6 +280,8 @@ type BlockChain struct { processor Processor // Block transaction processor interface vmConfig vm.Config logger *tracing.Hooks + + lastForkReadyAlert time.Time // Last time there was a fork readiness print out } // NewBlockChain returns a fully initialised block chain using information @@ -1884,6 +1891,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness trieDiffNodes, trieBufNodes, _ := bc.triedb.Size() stats.report(chain, it.index, snapDiffItems, snapBufItems, trieDiffNodes, trieBufNodes, setHead) + // Print confirmation that a future fork is scheduled, but not yet active. + bc.logForkReadiness(block) + if !setHead { // After merge we expect few side chains. Simply count // all blocks the CL gives us for GC processing time @@ -1917,6 +1927,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness "root", block.Root()) } } + stats.ignored += it.remaining() return witness, it.index, err } @@ -2514,6 +2525,23 @@ func (bc *BlockChain) reportBlock(block *types.Block, res *ProcessResult, err er log.Error(summarizeBadBlock(block, receipts, bc.Config(), err)) } +// logForkReadiness will write a log when a future fork is scheduled, but not +// active. This is useful so operators know their client is ready for the fork. +func (bc *BlockChain) logForkReadiness(block *types.Block) { + c := bc.Config() + current, last := c.LatestFork(block.Time()), c.LatestFork(math.MaxUint64) + t := c.Timestamp(last) + if t == nil { + return + } + at := time.Unix(int64(*t), 0) + if current < last && time.Now().After(bc.lastForkReadyAlert.Add(forkReadyInterval)) { + log.Info("Ready for fork activation", "fork", last, "date", at.Format(time.RFC822), + "remaining", time.Until(at).Round(time.Second), "timestamp", at.Unix()) + bc.lastForkReadyAlert = time.Now() + } +} + // summarizeBadBlock returns a string summarizing the bad block and other // relevant information. func summarizeBadBlock(block *types.Block, receipts []*types.Receipt, config *params.ChainConfig, err error) string { diff --git a/params/config.go b/params/config.go index 03b797863c..67aa6b2225 100644 --- a/params/config.go +++ b/params/config.go @@ -909,6 +909,23 @@ func (c *ChainConfig) LatestFork(time uint64) forks.Fork { } } +// Timestamp returns the timestamp associated with the fork or returns nil if +// the fork isn't defined or isn't a time-based fork. +func (c *ChainConfig) Timestamp(fork forks.Fork) *uint64 { + switch { + case fork == forks.Osaka: + return c.OsakaTime + case fork == forks.Prague: + return c.PragueTime + case fork == forks.Cancun: + return c.CancunTime + case fork == forks.Shanghai: + return c.ShanghaiTime + default: + return nil + } +} + // isForkBlockIncompatible returns true if a fork scheduled at block s1 cannot be // rescheduled to block s2 because head is already past the fork. func isForkBlockIncompatible(s1, s2, head *big.Int) bool { diff --git a/params/forks/forks.go b/params/forks/forks.go index 02f6e5b612..5c9612a625 100644 --- a/params/forks/forks.go +++ b/params/forks/forks.go @@ -20,7 +20,7 @@ package forks type Fork int const ( - Frontier = iota + Frontier Fork = iota FrontierThawing Homestead DAO @@ -41,3 +41,35 @@ const ( Prague Osaka ) + +// String implements fmt.Stringer. +func (f Fork) String() string { + s, ok := forkToString[f] + if !ok { + return "Unknown fork" + } + return s +} + +var forkToString = map[Fork]string{ + Frontier: "Frontier", + FrontierThawing: "Frontier Thawing", + Homestead: "Homestead", + DAO: "DAO", + TangerineWhistle: "Tangerine Whistle", + SpuriousDragon: "Spurious Dragon", + Byzantium: "Byzantium", + Constantinople: "Constantinople", + Petersburg: "Petersburg", + Istanbul: "Istanbul", + MuirGlacier: "Muir Glacier", + Berlin: "Berlin", + London: "London", + ArrowGlacier: "Arrow Glacier", + GrayGlacier: "Gray Glacier", + Paris: "Paris", + Shanghai: "Shanghai", + Cancun: "Cancun", + Prague: "Prague", + Osaka: "Osaka", +}