From 8e8003acd317e5b954d4348c3e256f6a14b16884 Mon Sep 17 00:00:00 2001 From: Jonny Rhea <5555162+jrhea@users.noreply.github.com> Date: Thu, 23 Jul 2026 01:26:01 -0500 Subject: [PATCH] triedb/pathdb: report nothing recoverable during state sync (#35400) Restarting a node mid-snap-sync can crash-loop during startup chain repair. On a restart mid-sync the pathdb is disabled (`waitSync`), so repair rewinds the stateless head to genesis and asks `StateRecoverable` if it can roll back there. `Recoverable` doesn't check `waitSync`, so it reports genesis as recoverable, but `Recover` does and refuses with `waiting for sync`, tripping a `log.Crit`. The flag is persisted, so the node crash-loops on every restart. CRIT Failed to recover state number=0 hash=37fcdc..6edf0b err="waiting for sync" The fix is to make `Recoverable` return false while a sync is running, so it agrees with `Recover`. Repair then falls through to the existing "wait state sync" path. --- triedb/pathdb/database.go | 4 ++++ triedb/pathdb/database_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 996d3f3ce1..d44e667e53 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -524,6 +524,10 @@ func (db *Database) Recover(root common.Hash) error { // // The supplied root must be a valid trie hash value. func (db *Database) Recoverable(root common.Hash) bool { + // Nothing is recoverable while a state sync is in progress. + if db.waitSync { + return false + } // Ensure the requested state is a known state. id := rawdb.ReadStateID(db.diskdb, root) if id == nil { diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index 41212dc9d0..2480d13f58 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -657,6 +657,38 @@ func TestDatabaseRecoverable(t *testing.T) { } } +// TestRecoverableDisabled checks that a database disabled for state sync +// reports nothing as recoverable, matching Recover which refuses to roll +// back mid-sync. Regression test for a snap-sync restart crash loop: during +// chain repair Recoverable returned true while Recover returned +// errDatabaseWaitSync, so setHeadBeyondRoot tripped a log.Crit. +func TestRecoverableDisabled(t *testing.T) { + maxDiffLayers = 4 + defer func() { maxDiffLayers = 128 }() + + tester := newTester(t, &testerConfig{layers: 12}) + defer tester.release() + + // A state below the disk layer is recoverable while the database is active. + root := tester.roots[tester.bottomIndex()-1] + if !tester.db.Recoverable(root) { + t.Fatalf("state below the disk layer should be recoverable while active") + } + // Disable the database to simulate an interrupted, still-running snap sync. + if err := tester.db.Disable(); err != nil { + t.Fatalf("failed to disable database: %v", err) + } + // Recover refuses while the sync is in progress. + if err := tester.db.Recover(root); !errors.Is(err, errDatabaseWaitSync) { + t.Fatalf("Recover: want errDatabaseWaitSync, got %v", err) + } + // Recoverable must agree, otherwise chain repair enters a Recover it + // believes is safe and crashes on the waitSync error. + if tester.db.Recoverable(root) { + t.Fatalf("disabled database must not report any state as recoverable") + } +} + func TestExecuteRollback(t *testing.T) { // Redefine the diff layer depth allowance for faster testing. maxDiffLayers = 4