From 76e3dc6b5be4ab2d1bb75b0b2b0237703a9b8e8a Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 9 Jul 2026 05:44:34 +0800 Subject: [PATCH] core: improve chain reset head (#35252) This PR does a few things: - reject `debug_setHead` if the target is even before the pivot block (if non-nil) - reject `debug_setHead` if in path mode, the target is not recoverable - decouple the chain rewinding and state recovery in path mode and recover the state in one shot --------- Co-authored-by: jonny rhea <5555162+jrhea@users.noreply.github.com> --- core/blockchain.go | 60 +++++++++++++++++++++------------- core/blockchain_reader.go | 4 +-- core/rawdb/accessors_chain.go | 4 +-- core/rawdb/schema.go | 2 +- eth/api_backend.go | 19 +++++++++++ eth/downloader/beaconsync.go | 4 +-- eth/downloader/downloader.go | 4 +-- eth/downloader/syncmode.go | 8 ++--- triedb/pathdb/database.go | 18 +++++----- triedb/pathdb/history_state.go | 27 --------------- 10 files changed, 78 insertions(+), 72 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index d6edd90133..798d3f1ae6 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -913,7 +913,7 @@ func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*typ // noState represents if the target state requested for search // is unavailable and impossible to be recovered. - noState = !bc.HasState(root) && !bc.stateRecoverable(root) + noState = !bc.HasState(root) && !bc.StateRecoverable(root) start = time.Now() // Timestamp the rewinding is restarted logged = time.Now() // Timestamp last progress log was printed @@ -940,7 +940,7 @@ func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*typ } // Check if the associated state is available or recoverable if // the requested root has already been crossed. - if beyondRoot && (bc.HasState(head.Root) || bc.stateRecoverable(head.Root)) { + if beyondRoot && (bc.HasState(head.Root) || bc.StateRecoverable(head.Root)) { break } // If pivot block is reached, return the genesis block as the @@ -966,14 +966,11 @@ func (bc *BlockChain) rewindPathHead(head *types.Header, root common.Hash) (*typ return head, rootNumber } } - // Recover if the target state if it's not available yet. - if !bc.HasState(head.Root) { - if err := bc.triedb.Recover(head.Root); err != nil { - log.Error("Failed to rollback state, resetting to genesis", "err", err) - return bc.genesisBlock.Header(), rootNumber - } - } - log.Info("Rewound to block with state", "number", head.Number, "hash", head.Hash()) + // Note, the state of the located head may not be physically present yet if + // it's only recoverable. The actual recovery is intentionally deferred once + // the new head is finalized, so that a deep rewind rolls the state back in + // one shot. + log.Info("Rewound to block with available state", "number", head.Number, "hash", head.Hash()) return head, rootNumber } @@ -1034,17 +1031,9 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha bc.currentBlock.Store(newHeadBlock) headBlockGauge.Update(int64(newHeadBlock.Number.Uint64())) - // The head state is missing, which is only possible in the path-based - // scheme. This situation occurs when the chain head is rewound below - // the pivot point. In this scenario, there is no possible recovery - // approach except for rerunning a snap sync. Do nothing here until the - // state syncer picks it up. - if !bc.HasState(newHeadBlock.Root) { - if newHeadBlock.Number.Uint64() != 0 { - log.Crit("Chain is stateless at a non-genesis block") - } - log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number, "hash", newHeadBlock.Hash()) - } + // Note, the located head state might not be physically present yet; in + // the path-based scheme a recoverable state is materialized in a single + // shot once the rewind is finalized. } // Rewind the snap block in a simpleton way to the target head if currentSnapBlock := bc.CurrentSnapBlock(); currentSnapBlock != nil && header.Number.Uint64() < currentSnapBlock.Number.Uint64() { @@ -1113,6 +1102,31 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha bc.hc.SetHead(head, updateFn, delFn) } } + // In the path-based scheme, the rewind loop above only locates the new head + // without materializing its state, so the potentially deep rollback is done + // here in a single shot. This rolls back the whole rewound range at once, + // performing a single fsync rather than one per block, which is critical when + // rewinding a large number of blocks. + if newHeadBlock := bc.CurrentBlock(); !bc.HasState(newHeadBlock.Root) { + switch { + case bc.StateRecoverable(newHeadBlock.Root): + if err := bc.triedb.Recover(newHeadBlock.Root); err != nil { + // The state was confirmed recoverable just above, so a failure here + // can only stem from an unexpected I/O error. There is no safe way to + // continue with a half-rolled-back state, hence crash hard. + log.Crit("Failed to recover state", "number", newHeadBlock.Number, "hash", newHeadBlock.Hash(), "err", err) + } + case newHeadBlock.Number.Uint64() != 0: + // rewindHead only returns a non-genesis head when its state is present + // or recoverable, so this branch should be unreachable. + log.Crit("Chain is stateless at a non-genesis block", "number", newHeadBlock.Number, "hash", newHeadBlock.Hash()) + default: + // The chain head was rewound below the snap-sync pivot to a stateless + // genesis. There is no recovery approach except rerunning a snap sync; + // do nothing here until the state syncer picks it up. + log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number, "hash", newHeadBlock.Hash()) + } + } // Clear out any stale content from the caches bc.bodyCache.Purge() bc.bodyRLPCache.Purge() @@ -2402,7 +2416,7 @@ func (bc *BlockChain) insertSideChain(ctx context.Context, block *types.Block, i ) parent := it.previous() for parent != nil && !bc.HasState(parent.Root) { - if bc.stateRecoverable(parent.Root) { + if bc.StateRecoverable(parent.Root) { if err := bc.triedb.Recover(parent.Root); err != nil { return nil, 0, err } @@ -2464,7 +2478,7 @@ func (bc *BlockChain) recoverAncestors(ctx context.Context, block *types.Block, parent = block ) for parent != nil && !bc.HasState(parent.Root()) { - if bc.stateRecoverable(parent.Root()) { + if bc.StateRecoverable(parent.Root()) { if err := bc.triedb.Recover(parent.Root()); err != nil { return common.Hash{}, err } diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index a540bbc11d..217a384752 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -395,11 +395,11 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool { return bc.HasState(block.Root()) } -// stateRecoverable checks if the specified state is recoverable. +// StateRecoverable checks if the specified state is recoverable. // Note, this function assumes the state is not present, because // state is not treated as recoverable if it's available, thus // false will be returned in this case. -func (bc *BlockChain) stateRecoverable(root common.Hash) bool { +func (bc *BlockChain) StateRecoverable(root common.Hash) bool { if bc.triedb.Scheme() == rawdb.HashScheme { return false } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index d8825b6b8f..e82a15877e 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -176,8 +176,8 @@ func WriteFinalizedBlockHash(db ethdb.KeyValueWriter, hash common.Hash) { // ReadLastPivotNumber retrieves the number of the last pivot block. If the node // has never attempted snap sync, the last pivot will always be nil. The marker -// is written during snap sync and never cleared, so that a rollback past the -// pivot can re-enable snap sync. +// is written during snap sync and never cleared, so that a rewind below the +// pivot can be detected. func ReadLastPivotNumber(db ethdb.KeyValueReader) *uint64 { data, _ := db.Get(lastPivotKey) if len(data) == 0 { diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 54c76143b4..ec72294354 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -46,7 +46,7 @@ var ( // persistentStateIDKey tracks the id of latest stored state(for path-based only). persistentStateIDKey = []byte("LastStateID") - // lastPivotKey tracks the last pivot block used by fast sync (to reenable on sethead). + // lastPivotKey tracks the last pivot block used by snap sync (to reject sethead below it). lastPivotKey = []byte("LastPivot") // fastTrieProgressKey tracks the number of trie entries imported during fast sync. diff --git a/eth/api_backend.go b/eth/api_backend.go index d527d4756e..c4e2508a3d 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -19,6 +19,7 @@ package eth import ( "context" "errors" + "fmt" "math/big" "time" @@ -64,6 +65,24 @@ func (b *EthAPIBackend) CurrentBlock() *types.Header { } func (b *EthAPIBackend) SetHead(number uint64) error { + // Reject rewinding to a point before the snap-sync pivot. The earliest + // recoverable state is the pivot block, so a target below it would simply + // reset the chain to genesis, which is almost never the intent behind a + // manual debug_setHead. + if pivot := rawdb.ReadLastPivotNumber(b.eth.ChainDb()); pivot != nil && number < *pivot { + return fmt.Errorf("rewind target %d is before the snap-sync pivot %d", number, *pivot) + } + // In path mode the deepest reachable state is bounded by the amount of state + // histories retained. If the reverse diffs for the target have already been + // pruned, its state is no longer recoverable. + bc := b.eth.blockchain + if bc.TrieDB().Scheme() == rawdb.PathScheme { + if header := bc.GetHeaderByNumber(number); header != nil { + if !bc.HasState(header.Root) && !bc.StateRecoverable(header.Root) { + return errors.New("rewind target is not recoverable") + } + } + } b.eth.handler.downloader.Cancel() return b.eth.blockchain.SetHead(number) } diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index 246fe7637b..2905b664ed 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -333,8 +333,8 @@ func (d *Downloader) fetchHeaders(from uint64) error { return errNoPivotHeader } // Write out the pivot into the database so a rollback beyond - // it will reenable snap sync and update the state root that - // the state syncer will be downloading + // it can be detected, and update the state root that the + // state syncer will be downloading rawdb.WriteLastPivotNumber(d.stateDB, d.pivotHeader.Number.Uint64()) } } diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 60b04e945b..876115ea97 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -542,8 +542,8 @@ func (d *Downloader) syncToHead() (err error) { if pivotNumber <= origin { origin = pivotNumber - 1 } - // Write out the pivot into the database so a rollback beyond it will - // reenable snap sync + // Write out the pivot into the database so a rollback beyond it + // can be detected rawdb.WriteLastPivotNumber(d.stateDB, pivotNumber) } } diff --git a/eth/downloader/syncmode.go b/eth/downloader/syncmode.go index 036119ce3d..a7792eb5fc 100644 --- a/eth/downloader/syncmode.go +++ b/eth/downloader/syncmode.go @@ -40,8 +40,8 @@ func newSyncModer(mode ethconfig.SyncMode, chain BlockChain, disk ethdb.KeyValue // The database seems empty as the current block is the genesis. Yet the snap // block is ahead, so snap sync was enabled for this node at a certain point. // The scenarios where this can happen is - // * if the user manually (or via a bad block) rolled back a snap sync node - // below the sync point. + // * if an internal reset (a fork config change or corruption recovery) + // rolled a snap sync node back below the sync point. // * the last snap sync is not finished while user specifies a full sync this // time. But we don't have any recent state for full sync. // In these cases however it's safe to reenable snap sync. @@ -87,8 +87,8 @@ func (m *syncModer) get(report bool) ethconfig.SyncMode { if report { logger = log.Info } - // We are probably in full sync, but we might have rewound to before the - // snap sync pivot, check if we should re-enable snap sync. + // We are probably in full sync, but an internal reset may have rewound the + // chain below the snap sync pivot, check if we should re-enable snap sync. head := m.chain.CurrentBlock() if pivot := rawdb.ReadLastPivotNumber(m.disk); pivot != nil { if head.Number.Uint64() < *pivot { diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index e52949c93e..996d3f3ce1 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -543,15 +543,15 @@ func (db *Database) Recoverable(root common.Hash) bool { if db.stateFreezer == nil { return false } - // Ensure the requested state is a canonical state and all state - // histories in range [id+1, dl.ID] are present and complete. - return checkStateHistories(db.stateFreezer, *id+1, dl.stateID()-*id, func(m *meta) error { - if m.parent != root { - return errors.New("unexpected state history") - } - root = m.root - return nil - }) == nil + blob := rawdb.ReadStateHistoryMeta(db.stateFreezer, *id+1) + if len(blob) == 0 { + return false // pruned from the tail or otherwise unavailable + } + var m meta + if err := m.decode(blob); err != nil { + return false + } + return m.parent == root } // Close closes the trie database and the held freezer. diff --git a/triedb/pathdb/history_state.go b/triedb/pathdb/history_state.go index 23428b1a54..9d3bf23e22 100644 --- a/triedb/pathdb/history_state.go +++ b/triedb/pathdb/history_state.go @@ -612,30 +612,3 @@ func writeStateHistory(writer ethdb.AncientWriter, dl *diffLayer) error { return nil } - -// checkStateHistories retrieves a batch of metadata objects with the specified -// range and performs the callback on each item. -func checkStateHistories(reader ethdb.AncientReader, start, count uint64, check func(*meta) error) error { - for count > 0 { - number := count - if number > 10000 { - number = 10000 // split the big read into small chunks - } - blobs, err := rawdb.ReadStateHistoryMetaList(reader, start, number) - if err != nil { - return err - } - for _, blob := range blobs { - var dec meta - if err := dec.decode(blob); err != nil { - return err - } - if err := check(&dec); err != nil { - return err - } - } - count -= uint64(len(blobs)) - start += uint64(len(blobs)) - } - return nil -}