triedb/pathdb: report nothing recoverable during state sync (#35400)

Restarting a node mid-snap-sync can crash-loop during startup chain
repair. On a restart mid-sync the pathdb is disabled (`waitSync`), so
repair rewinds the stateless head to genesis and asks `StateRecoverable`
if it can roll back there. `Recoverable` doesn't check `waitSync`, so it
reports genesis as recoverable, but `Recover` does and refuses with
`waiting for sync`, tripping a `log.Crit`. The flag is persisted, so the
node crash-loops on every restart.

CRIT Failed to recover state number=0 hash=37fcdc..6edf0b err="waiting
for sync"

The fix is to make `Recoverable` return false while a sync is running,
so it agrees with `Recover`. Repair then falls through to the existing
"wait state sync" path.
This commit is contained in:
Jonny Rhea 2026-07-23 01:26:01 -05:00 committed by GitHub
parent c767f825c5
commit 8e8003acd3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 36 additions and 0 deletions

View file

@ -524,6 +524,10 @@ func (db *Database) Recover(root common.Hash) error {
//
// The supplied root must be a valid trie hash value.
func (db *Database) Recoverable(root common.Hash) bool {
// Nothing is recoverable while a state sync is in progress.
if db.waitSync {
return false
}
// Ensure the requested state is a known state.
id := rawdb.ReadStateID(db.diskdb, root)
if id == nil {

View file

@ -657,6 +657,38 @@ func TestDatabaseRecoverable(t *testing.T) {
}
}
// TestRecoverableDisabled checks that a database disabled for state sync
// reports nothing as recoverable, matching Recover which refuses to roll
// back mid-sync. Regression test for a snap-sync restart crash loop: during
// chain repair Recoverable returned true while Recover returned
// errDatabaseWaitSync, so setHeadBeyondRoot tripped a log.Crit.
func TestRecoverableDisabled(t *testing.T) {
maxDiffLayers = 4
defer func() { maxDiffLayers = 128 }()
tester := newTester(t, &testerConfig{layers: 12})
defer tester.release()
// A state below the disk layer is recoverable while the database is active.
root := tester.roots[tester.bottomIndex()-1]
if !tester.db.Recoverable(root) {
t.Fatalf("state below the disk layer should be recoverable while active")
}
// Disable the database to simulate an interrupted, still-running snap sync.
if err := tester.db.Disable(); err != nil {
t.Fatalf("failed to disable database: %v", err)
}
// Recover refuses while the sync is in progress.
if err := tester.db.Recover(root); !errors.Is(err, errDatabaseWaitSync) {
t.Fatalf("Recover: want errDatabaseWaitSync, got %v", err)
}
// Recoverable must agree, otherwise chain repair enters a Recover it
// believes is safe and crashes on the waitSync error.
if tester.db.Recoverable(root) {
t.Fatalf("disabled database must not report any state as recoverable")
}
}
func TestExecuteRollback(t *testing.T) {
// Redefine the diff layer depth allowance for faster testing.
maxDiffLayers = 4