diff --git a/core/blockchain.go b/core/blockchain.go index 90ee3a5fe3..6a07d91789 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -656,10 +656,10 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha pivot := rawdb.ReadLastPivotNumber(bc.db) frozen, _ := bc.db.Ancients() - // Rewind the blockchain, ensuring we don't end up with a stateless head - // block. Note, depth equality is permitted to allow using SetHead as a - // chain reparation mechanism without deleting any data! updateFn := func(db ethdb.KeyValueWriter, header *types.Header) (*types.Header, bool) { + // Rewind the blockchain, ensuring we don't end up with a stateless head + // block. Note, depth equality is permitted to allow using SetHead as a + // chain reparation mechanism without deleting any data! if currentBlock := bc.CurrentBlock(); currentBlock != nil && header.Number.Uint64() <= currentBlock.Number.Uint64() { newHeadBlock := bc.GetBlock(header.Hash(), header.Number.Uint64()) if newHeadBlock == nil { @@ -708,20 +708,21 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha } rawdb.WriteHeadBlockHash(db, newHeadBlock.Hash()) - // The genesis state is missing, which is only possible in the path-based - // scheme. This situation occurs when the chain head is rewound below the - // pivot point. In this scenario, there is no possible recovery approach - // except for rerunning a snap sync. Do nothing here until the state syncer - // picks it up. - if newHeadBlock.NumberU64() == 0 && !bc.HasState(newHeadBlock.Root()) { - log.Info("Genesis state is missing, wait state sync") - } // Degrade the chain markers if they are explicitly reverted. // In theory we should update all in-memory markers in the // last step, however the direction of SetHead is from high // to low, so it's safe to update in-memory markers directly. bc.currentBlock.Store(newHeadBlock.Header()) headBlockGauge.Update(int64(newHeadBlock.NumberU64())) + + // The head state is missing, which is only possible in the path-based + // scheme. This situation occurs when the chain head is rewound below + // the pivot point. In this scenario, there is no possible recovery + // approach except for rerunning a snap sync. Do nothing here until the + // state syncer picks it up. + if !bc.HasState(newHeadBlock.Root()) { + log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number(), "hash", newHeadBlock.Hash()) + } } // Rewind the snap block in a simpleton way to the target head if currentSnapBlock := bc.CurrentSnapBlock(); currentSnapBlock != nil && header.Number.Uint64() < currentSnapBlock.Number.Uint64() { diff --git a/core/rawdb/accessors_sync.go b/core/rawdb/accessors_sync.go index 0366f930c5..2dc08b3b72 100644 --- a/core/rawdb/accessors_sync.go +++ b/core/rawdb/accessors_sync.go @@ -78,22 +78,23 @@ func DeleteSkeletonHeader(db ethdb.KeyValueWriter, number uint64) { } const ( - StateSyncing = uint8(1) // flags the state snap sync is not completed yet - StateSynced = uint8(2) // flags the state snap sync is completed + StateSyncUnknown = uint8(0) // flags the state snap sync is unknown + StateSyncRunning = uint8(1) // flags the state snap sync is not completed yet + StateSyncFinished = uint8(2) // flags the state snap sync is completed ) // ReadSnapSyncStatusFlag retrieves the state snap sync status flag. func ReadSnapSyncStatusFlag(db ethdb.KeyValueReader) uint8 { - blob, err := db.Get(syncStatusFlagKey) + blob, err := db.Get(snapSyncStatusFlagKey) if err != nil || len(blob) != 1 { - return 0 + return StateSyncUnknown } return blob[0] } // WriteSnapSyncStatusFlag stores the state snap sync status flag into database. func WriteSnapSyncStatusFlag(db ethdb.KeyValueWriter, flag uint8) { - if err := db.Put(syncStatusFlagKey, []byte{flag}); err != nil { + if err := db.Put(snapSyncStatusFlagKey, []byte{flag}); err != nil { log.Crit("Failed to store sync status flag", "err", err) } } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 123021cd6b..e97eeb2aa3 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -555,7 +555,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey, snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, - persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, syncStatusFlagKey, + persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey, } { if bytes.Equal(key, meta) { metadata.Add(size) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 02d47e7629..91e4fbb974 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -91,8 +91,8 @@ var ( // transitionStatusKey tracks the eth2 transition status. transitionStatusKey = []byte("eth2-transition") - // syncStatusFlagKey flags that status of state sync. - syncStatusFlagKey = []byte("sync-status") + // snapSyncStatusFlagKey flags that status of snap sync. + snapSyncStatusFlagKey = []byte("snapSync-status") // Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes). headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index ae94bfa70e..1aed986c88 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -93,7 +93,9 @@ func New(gasTip *big.Int, chain BlockChain, subpools []SubPool) (*TxPool, error) quit: make(chan chan error), } if chain.HasState(head.Root) { - pool.init(gasTip, head) + if err := pool.init(gasTip, head); err != nil { + return nil, err + } go pool.loop(head, chain) } else { go pool.lazyInit(gasTip, chain) @@ -102,17 +104,18 @@ func New(gasTip *big.Int, chain BlockChain, subpools []SubPool) (*TxPool, error) } // init performs the initialization for subpools. -func (p *TxPool) init(gasTip *big.Int, head *types.Header) { +func (p *TxPool) init(gasTip *big.Int, head *types.Header) error { for i, subpool := range p.subpools { if err := subpool.Init(gasTip, head, p.reserver(i, subpool)); err != nil { for j := i - 1; j >= 0; j-- { p.subpools[j].Close() } - // TODO(rjl493456442) can we shutdown the node gracefully? - log.Crit("Failed to initialize subpool", "err", err) + return err } } p.inited.Store(true) + log.Info("Initialized subpools", "head", head.Number, "hash", head.Hash()) + return nil } // lazyInit waits the signal that state sync is completed and initializes the subpools. @@ -131,7 +134,10 @@ func (p *TxPool) lazyInit(gasTip *big.Int, chain BlockChain) { if !chain.HasState(head.Root) { continue // shouldn't happen } - p.init(gasTip, head) + if err := p.init(gasTip, head); err != nil { + // TODO(rjl493456442) can we shutdown the node gracefully? + log.Crit("Failed to lazy init subpools", "err", err) + } go p.loop(head, chain) return @@ -279,7 +285,7 @@ func (p *TxPool) loop(head *types.Header, chain BlockChain) { // new transaction, and drops all transactions below this threshold. func (p *TxPool) SetGasTip(tip *big.Int) { if !p.inited.Load() { - log.Info("Skip tip adjustment as txpool hasn't been initialized") + log.Info("Skip tip adjustment as txpool hasn't been initialized", "provided", tip) return } for _, subpool := range p.subpools { diff --git a/eth/handler.go b/eth/handler.go index 6254b3a01a..7f134d7677 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -172,10 +172,11 @@ func newHandler(config *handlerConfig) (*handler, error) { head := h.chain.CurrentBlock() if head.Number.Uint64() > 0 && h.chain.HasState(head.Root) { // Print warning log if database is not empty to run snap sync. - log.Warn("Switch sync mode from snap sync to full sync") + log.Warn("Switch sync mode from snap sync to full sync", "reason", "snap sync complete") } else { // If snap sync was requested and our database is empty, grant it h.snapSync.Store(true) + log.Info("Enabled snap sync", "head", head.Number, "hash", head.Hash()) } } // If the sync succeeds, mark the local node as synced and enable all features diff --git a/eth/sync.go b/eth/sync.go index 03357254e7..c7ba7c93d6 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -212,6 +212,7 @@ func (cs *chainSyncer) modeAndLocalHead() (downloader.SyncMode, *big.Int) { if !cs.handler.chain.HasState(head.Root) { block := cs.handler.chain.CurrentSnapBlock() td := cs.handler.chain.GetTd(block.Hash(), block.Number.Uint64()) + log.Info("Reenabled snap sync as chain is stateless") return downloader.SnapSync, td } // Nope, we're really full syncing diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index 99259538f2..ad8db9045e 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -181,7 +181,7 @@ func New(diskdb ethdb.Database, config *Config) *Database { } } // Disable database in case node is still in the initial state sync stage. - if rawdb.ReadSnapSyncStatusFlag(diskdb) == rawdb.StateSyncing && !db.readOnly { + if rawdb.ReadSnapSyncStatusFlag(diskdb) == rawdb.StateSyncRunning && !db.readOnly { db.Deactivate() } log.Warn("Path-based state scheme is an experimental feature") @@ -260,8 +260,8 @@ func (db *Database) Deactivate() error { db.tree.bottom().markStale() // Write the initial sync flag to persist it across restarts. - rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSyncing) - log.Info("Disabled trie database") + rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSyncRunning) + log.Info("Disabled trie database due to ongoing sync") return nil } @@ -304,7 +304,7 @@ func (db *Database) Activate(root common.Hash) error { // Re-enable the database as the final step. db.disabled = false - rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSynced) + rawdb.WriteSnapSyncStatusFlag(db.diskdb, rawdb.StateSyncFinished) log.Info("Rebuilt trie database", "root", root) return nil }