diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go index 44f041d82e..2359fb18f1 100644 --- a/core/rawdb/accessors_state.go +++ b/core/rawdb/accessors_state.go @@ -119,13 +119,6 @@ func WriteStateID(db ethdb.KeyValueWriter, root common.Hash, id uint64) { } } -// DeleteStateID deletes the specified state lookup from the database. -func DeleteStateID(db ethdb.KeyValueWriter, root common.Hash) { - if err := db.Delete(stateIDKey(root)); err != nil { - log.Crit("Failed to delete state ID", "err", err) - } -} - // ReadPersistentStateID retrieves the id of the persistent state from the database. func ReadPersistentStateID(db ethdb.KeyValueReader) uint64 { data, _ := db.Get(persistentStateIDKey) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index b1e2e75784..f438c64aa2 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -334,7 +334,7 @@ func (db *Database) repairHistory() error { } // Truncate the extra state histories above in freezer in case it's not // aligned with the disk layer. It might happen after a unclean shutdown. - pruned, err := truncateFromHead(db.diskdb, db.stateFreezer, id) + pruned, err := truncateFromHead(db.stateFreezer, id) if err != nil { log.Crit("Failed to truncate extra state histories", "err", err) } @@ -590,7 +590,7 @@ func (db *Database) Recover(root common.Hash) error { if err := db.diskdb.SyncKeyValue(); err != nil { return err } - _, err := truncateFromHead(db.diskdb, db.stateFreezer, dl.stateID()) + _, err := truncateFromHead(db.stateFreezer, dl.stateID()) if err != nil { return err } @@ -615,14 +615,14 @@ func (db *Database) Recoverable(root common.Hash) bool { return false } // This is a temporary workaround for the unavailability of the freezer in - // dev mode. As a consequence, the Pathdb loses the ability for deep reorg + // dev mode. As a consequence, the database loses the ability for deep reorg // in certain cases. // TODO(rjl493456442): Implement the in-memory ancient store. if db.stateFreezer == nil { return false } // Ensure the requested state is a canonical state and all state - // histories in range [id+1, disklayer.ID] are present and complete. + // histories in range [id+1, dl.ID] are present and complete. return checkStateHistories(db.stateFreezer, *id+1, dl.stateID()-*id, func(m *meta) error { if m.parent != root { return errors.New("unexpected state history") diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index f1248b02bd..13df6251e8 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -378,7 +378,7 @@ func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) { log.Debug("Skip tail truncation", "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit) return true, nil } - pruned, err := truncateFromTail(dl.db.diskdb, dl.db.stateFreezer, newFirst-1) + pruned, err := truncateFromTail(dl.db.stateFreezer, newFirst-1) if err != nil { return false, err } diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go new file mode 100644 index 0000000000..bbedd52f34 --- /dev/null +++ b/triedb/pathdb/history.go @@ -0,0 +1,87 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see id mappings are left in the database and wait + // for overwriting. + return int(ohead - nhead), nil +} + +// truncateFromTail removes excess elements from the end of the freezer based +// on the given parameters. It returns the number of items that were removed. +func truncateFromTail(store ethdb.AncientStore, ntail uint64) (int, error) { + ohead, err := store.Ancients() + if err != nil { + return 0, err + } + otail, err := store.Tail() + if err != nil { + return 0, err + } + // Ensure that the truncation target falls within the valid range. + if otail > ntail || ntail > ohead { + return 0, fmt.Errorf("%w, tail: %d, head: %d, target: %d", errTailTruncationOutOfRange, otail, ohead, ntail) + } + // Short circuit if nothing to truncate. + if otail == ntail { + return 0, nil + } + otail, err = store.TruncateTail(ntail) + if err != nil { + return 0, err + } + // Associated root->id mappings are left in the database. + return int(ntail - otail), nil +} diff --git a/triedb/pathdb/history_reader.go b/triedb/pathdb/history_reader.go index d0ecdf035f..a11297b3f6 100644 --- a/triedb/pathdb/history_reader.go +++ b/triedb/pathdb/history_reader.go @@ -320,11 +320,12 @@ func (r *historyReader) read(state stateIdentQuery, stateID uint64, lastID uint6 tail, err := r.freezer.Tail() if err != nil { return nil, err - } - // stateID == tail is allowed, as the first history object preserved - // is tail+1 + } // firstID = tail+1 + + // stateID+1 == firstID is allowed, as all the subsequent state histories + // are present with no gap inside. if stateID < tail { - return nil, errors.New("historical state has been pruned") + return nil, fmt.Errorf("historical state has been pruned, first: %d, state: %d", tail+1, stateID) } // To serve the request, all state histories from stateID+1 to lastID diff --git a/triedb/pathdb/history_reader_test.go b/triedb/pathdb/history_reader_test.go index e271b271a9..9028a886ce 100644 --- a/triedb/pathdb/history_reader_test.go +++ b/triedb/pathdb/history_reader_test.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/internal/testrand" ) func waitIndexing(db *Database) { @@ -36,11 +37,29 @@ func waitIndexing(db *Database) { } } -func checkHistoricState(env *tester, root common.Hash, hr *historyReader) error { - // Short circuit if the historical state is no longer available - if rawdb.ReadStateID(env.db.diskdb, root) == nil { +func stateAvail(id uint64, env *tester) bool { + if env.db.config.StateHistory == 0 { + return true + } + dl := env.db.tree.bottom() + if dl.stateID() <= env.db.config.StateHistory { + return true + } + firstID := dl.stateID() - env.db.config.StateHistory + 1 + + return id+1 >= firstID +} + +func checkHistoricalState(env *tester, root common.Hash, id uint64, hr *historyReader) error { + if !stateAvail(id, env) { return nil } + + // Short circuit if the historical state is no longer available + if rawdb.ReadStateID(env.db.diskdb, root) == nil { + return fmt.Errorf("state not found %d %x", id, root) + } + var ( dl = env.db.tree.bottom() stateID = rawdb.ReadStateID(env.db.diskdb, root) @@ -124,22 +143,22 @@ func testHistoryReader(t *testing.T, historyLimit uint64) { defer func() { maxDiffLayers = 128 }() - //log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) + // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) env := newTester(t, historyLimit, false, 64, true, "") defer env.release() waitIndexing(env.db) var ( roots = env.roots - dRoot = env.db.tree.bottom().rootHash() + dl = env.db.tree.bottom() hr = newHistoryReader(env.db.diskdb, env.db.stateFreezer) ) - for _, root := range roots { - if root == dRoot { + for i, root := range roots { + if root == dl.rootHash() { break } - if err := checkHistoricState(env, root, hr); err != nil { + if err := checkHistoricalState(env, root, uint64(i+1), hr); err != nil { t.Fatal(err) } } @@ -148,12 +167,41 @@ func testHistoryReader(t *testing.T, historyLimit uint64) { env.extend(4) waitIndexing(env.db) - for _, root := range roots { - if root == dRoot { + for i, root := range roots { + if root == dl.rootHash() { break } - if err := checkHistoricState(env, root, hr); err != nil { + if err := checkHistoricalState(env, root, uint64(i+1), hr); err != nil { t.Fatal(err) } } } + +func TestHistoricalStateReader(t *testing.T) { + maxDiffLayers = 4 + defer func() { + maxDiffLayers = 128 + }() + + //log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) + env := newTester(t, 0, false, 64, true, "") + defer env.release() + waitIndexing(env.db) + + // non-canonical state + fakeRoot := testrand.Hash() + rawdb.WriteStateID(env.db.diskdb, fakeRoot, 10) + + _, err := env.db.HistoricReader(fakeRoot) + if err == nil { + t.Fatal("expected error") + } + t.Log(err) + + // canonical state + realRoot := env.roots[9] + _, err = env.db.HistoricReader(realRoot) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } +} diff --git a/triedb/pathdb/history_state.go b/triedb/pathdb/history_state.go index ab8e97b6c0..3bb69a7f4d 100644 --- a/triedb/pathdb/history_state.go +++ b/triedb/pathdb/history_state.go @@ -504,6 +504,20 @@ func (h *stateHistory) decode(accountData, storageData, accountIndexes, storageI return nil } +// readStateHistoryMeta reads the metadata of state history with the specified id. +func readStateHistoryMeta(reader ethdb.AncientReader, id uint64) (*meta, error) { + data := rawdb.ReadStateHistoryMeta(reader, id) + if len(data) == 0 { + return nil, fmt.Errorf("metadata is not found, %d", id) + } + var m meta + err := m.decode(data) + if err != nil { + return nil, err + } + return &m, nil +} + // readStateHistory reads a single state history records with the specified id. func readStateHistory(reader ethdb.AncientReader, id uint64) (*stateHistory, error) { mData, accountIndexes, storageIndexes, accountData, storageData, err := rawdb.ReadStateHistory(reader, id) @@ -568,8 +582,8 @@ func writeStateHistory(writer ethdb.AncientWriter, dl *diffLayer) error { return nil } -// checkStateHistories retrieves a batch of meta objects with the specified range -// and performs the callback on each item. +// checkStateHistories retrieves a batch of metadata objects with the specified +// range and performs the callback on each item. func checkStateHistories(reader ethdb.AncientReader, start, count uint64, check func(*meta) error) error { for count > 0 { number := count @@ -594,87 +608,3 @@ func checkStateHistories(reader ethdb.AncientReader, start, count uint64, check } return nil } - -// truncateFromHead removes the extra state histories from the head with the given -// parameters. It returns the number of items removed from the head. -func truncateFromHead(db ethdb.Batcher, store ethdb.AncientStore, nhead uint64) (int, error) { - ohead, err := store.Ancients() - if err != nil { - return 0, err - } - otail, err := store.Tail() - if err != nil { - return 0, err - } - // Ensure that the truncation target falls within the specified range. - if ohead < nhead || nhead < otail { - return 0, fmt.Errorf("out of range, tail: %d, head: %d, target: %d", otail, ohead, nhead) - } - // Short circuit if nothing to truncate. - if ohead == nhead { - return 0, nil - } - // Load the meta objects in range [nhead+1, ohead] - blobs, err := rawdb.ReadStateHistoryMetaList(store, nhead+1, ohead-nhead) - if err != nil { - return 0, err - } - batch := db.NewBatch() - for _, blob := range blobs { - var m meta - if err := m.decode(blob); err != nil { - return 0, err - } - rawdb.DeleteStateID(batch, m.root) - } - if err := batch.Write(); err != nil { - return 0, err - } - ohead, err = store.TruncateHead(nhead) - if err != nil { - return 0, err - } - return int(ohead - nhead), nil -} - -// truncateFromTail removes the extra state histories from the tail with the given -// parameters. It returns the number of items removed from the tail. -func truncateFromTail(db ethdb.Batcher, store ethdb.AncientStore, ntail uint64) (int, error) { - ohead, err := store.Ancients() - if err != nil { - return 0, err - } - otail, err := store.Tail() - if err != nil { - return 0, err - } - // Ensure that the truncation target falls within the specified range. - if otail > ntail || ntail > ohead { - return 0, fmt.Errorf("out of range, tail: %d, head: %d, target: %d", otail, ohead, ntail) - } - // Short circuit if nothing to truncate. - if otail == ntail { - return 0, nil - } - // Load the meta objects in range [otail+1, ntail] - blobs, err := rawdb.ReadStateHistoryMetaList(store, otail+1, ntail-otail) - if err != nil { - return 0, err - } - batch := db.NewBatch() - for _, blob := range blobs { - var m meta - if err := m.decode(blob); err != nil { - return 0, err - } - rawdb.DeleteStateID(batch, m.root) - } - if err := batch.Write(); err != nil { - return 0, err - } - otail, err = store.TruncateTail(ntail) - if err != nil { - return 0, err - } - return int(ntail - otail), nil -} diff --git a/triedb/pathdb/history_state_test.go b/triedb/pathdb/history_state_test.go index e154811367..4a777111ea 100644 --- a/triedb/pathdb/history_state_test.go +++ b/triedb/pathdb/history_state_test.go @@ -18,6 +18,7 @@ package pathdb import ( "bytes" + "errors" "fmt" "reflect" "testing" @@ -108,7 +109,7 @@ func testEncodeDecodeStateHistory(t *testing.T, rawStorageKey bool) { } } -func checkStateHistory(t *testing.T, db ethdb.KeyValueReader, freezer ethdb.AncientReader, id uint64, root common.Hash, exist bool) { +func checkStateHistory(t *testing.T, freezer ethdb.AncientReader, id uint64, exist bool) { blob := rawdb.ReadStateHistoryMeta(freezer, id) if exist && len(blob) == 0 { t.Fatalf("Failed to load trie history, %d", id) @@ -116,25 +117,17 @@ func checkStateHistory(t *testing.T, db ethdb.KeyValueReader, freezer ethdb.Anci if !exist && len(blob) != 0 { t.Fatalf("Unexpected trie history, %d", id) } - if exist && rawdb.ReadStateID(db, root) == nil { - t.Fatalf("Root->ID mapping is not found, %d", id) - } - if !exist && rawdb.ReadStateID(db, root) != nil { - t.Fatalf("Unexpected root->ID mapping, %d", id) - } } -func checkHistoriesInRange(t *testing.T, db ethdb.KeyValueReader, freezer ethdb.AncientReader, from, to uint64, roots []common.Hash, exist bool) { - for i, j := from, 0; i <= to; i, j = i+1, j+1 { - checkStateHistory(t, db, freezer, i, roots[j], exist) +func checkHistoriesInRange(t *testing.T, freezer ethdb.AncientReader, from, to uint64, exist bool) { + for i := from; i <= to; i = i + 1 { + checkStateHistory(t, freezer, i, exist) } } func TestTruncateHeadStateHistory(t *testing.T) { var ( - roots []common.Hash hs = makeStateHistories(10) - db = rawdb.NewMemoryDatabase() freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false) ) defer freezer.Close() @@ -142,27 +135,23 @@ func TestTruncateHeadStateHistory(t *testing.T) { for i := 0; i < len(hs); i++ { accountData, storageData, accountIndex, storageIndex := hs[i].encode() rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData) - rawdb.WriteStateID(db, hs[i].meta.root, uint64(i+1)) - roots = append(roots, hs[i].meta.root) } for size := len(hs); size > 0; size-- { - pruned, err := truncateFromHead(db, freezer, uint64(size-1)) + pruned, err := truncateFromHead(freezer, uint64(size-1)) if err != nil { t.Fatalf("Failed to truncate from head %v", err) } if pruned != 1 { t.Error("Unexpected pruned items", "want", 1, "got", pruned) } - checkHistoriesInRange(t, db, freezer, uint64(size), uint64(10), roots[size-1:], false) - checkHistoriesInRange(t, db, freezer, uint64(1), uint64(size-1), roots[:size-1], true) + checkHistoriesInRange(t, freezer, uint64(size), uint64(10), false) + checkHistoriesInRange(t, freezer, uint64(1), uint64(size-1), true) } } func TestTruncateTailStateHistory(t *testing.T) { var ( - roots []common.Hash hs = makeStateHistories(10) - db = rawdb.NewMemoryDatabase() freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false) ) defer freezer.Close() @@ -170,16 +159,14 @@ func TestTruncateTailStateHistory(t *testing.T) { for i := 0; i < len(hs); i++ { accountData, storageData, accountIndex, storageIndex := hs[i].encode() rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData) - rawdb.WriteStateID(db, hs[i].meta.root, uint64(i+1)) - roots = append(roots, hs[i].meta.root) } for newTail := 1; newTail < len(hs); newTail++ { - pruned, _ := truncateFromTail(db, freezer, uint64(newTail)) + pruned, _ := truncateFromTail(freezer, uint64(newTail)) if pruned != 1 { t.Error("Unexpected pruned items", "want", 1, "got", pruned) } - checkHistoriesInRange(t, db, freezer, uint64(1), uint64(newTail), roots[:newTail], false) - checkHistoriesInRange(t, db, freezer, uint64(newTail+1), uint64(10), roots[newTail:], true) + checkHistoriesInRange(t, freezer, uint64(1), uint64(newTail), false) + checkHistoriesInRange(t, freezer, uint64(newTail+1), uint64(10), true) } } @@ -191,21 +178,29 @@ func TestTruncateTailStateHistories(t *testing.T) { minUnpruned uint64 empty bool }{ + // history: id [10] { - 1, 9, 9, 10, false, + limit: 1, + expPruned: 9, + maxPruned: 9, minUnpruned: 10, empty: false, }, + // history: none { - 0, 10, 10, 0 /* no meaning */, true, + limit: 0, + expPruned: 10, + empty: true, }, + // history: id [1:10] { - 10, 0, 0, 1, false, + limit: 10, + expPruned: 0, + maxPruned: 0, + minUnpruned: 1, }, } for i, c := range cases { var ( - roots []common.Hash hs = makeStateHistories(10) - db = rawdb.NewMemoryDatabase() freezer, _ = rawdb.NewStateFreezer(t.TempDir()+fmt.Sprintf("%d", i), false, false) ) defer freezer.Close() @@ -213,19 +208,16 @@ func TestTruncateTailStateHistories(t *testing.T) { for i := 0; i < len(hs); i++ { accountData, storageData, accountIndex, storageIndex := hs[i].encode() rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData) - rawdb.WriteStateID(db, hs[i].meta.root, uint64(i+1)) - roots = append(roots, hs[i].meta.root) } - pruned, _ := truncateFromTail(db, freezer, uint64(10)-c.limit) + pruned, _ := truncateFromTail(freezer, uint64(10)-c.limit) if pruned != c.expPruned { t.Error("Unexpected pruned items", "want", c.expPruned, "got", pruned) } if c.empty { - checkHistoriesInRange(t, db, freezer, uint64(1), uint64(10), roots, false) + checkHistoriesInRange(t, freezer, uint64(1), uint64(10), false) } else { - tail := 10 - int(c.limit) - checkHistoriesInRange(t, db, freezer, uint64(1), c.maxPruned, roots[:tail], false) - checkHistoriesInRange(t, db, freezer, c.minUnpruned, uint64(10), roots[tail:], true) + checkHistoriesInRange(t, freezer, uint64(1), c.maxPruned, false) + checkHistoriesInRange(t, freezer, c.minUnpruned, uint64(10), true) } } } @@ -233,7 +225,6 @@ func TestTruncateTailStateHistories(t *testing.T) { func TestTruncateOutOfRange(t *testing.T) { var ( hs = makeStateHistories(10) - db = rawdb.NewMemoryDatabase() freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false) ) defer freezer.Close() @@ -241,9 +232,8 @@ func TestTruncateOutOfRange(t *testing.T) { for i := 0; i < len(hs); i++ { accountData, storageData, accountIndex, storageIndex := hs[i].encode() rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData) - rawdb.WriteStateID(db, hs[i].meta.root, uint64(i+1)) } - truncateFromTail(db, freezer, uint64(len(hs)/2)) + truncateFromTail(freezer, uint64(len(hs)/2)) // Ensure of-out-range truncations are rejected correctly. head, _ := freezer.Ancients() @@ -255,20 +245,20 @@ func TestTruncateOutOfRange(t *testing.T) { expErr error }{ {0, head, nil}, // nothing to delete - {0, head + 1, fmt.Errorf("out of range, tail: %d, head: %d, target: %d", tail, head, head+1)}, - {0, tail - 1, fmt.Errorf("out of range, tail: %d, head: %d, target: %d", tail, head, tail-1)}, + {0, head + 1, errHeadTruncationOutOfRange}, + {0, tail - 1, errHeadTruncationOutOfRange}, {1, tail, nil}, // nothing to delete - {1, head + 1, fmt.Errorf("out of range, tail: %d, head: %d, target: %d", tail, head, head+1)}, - {1, tail - 1, fmt.Errorf("out of range, tail: %d, head: %d, target: %d", tail, head, tail-1)}, + {1, head + 1, errTailTruncationOutOfRange}, + {1, tail - 1, errTailTruncationOutOfRange}, } for _, c := range cases { var gotErr error if c.mode == 0 { - _, gotErr = truncateFromHead(db, freezer, c.target) + _, gotErr = truncateFromHead(freezer, c.target) } else { - _, gotErr = truncateFromTail(db, freezer, c.target) + _, gotErr = truncateFromTail(freezer, c.target) } - if !reflect.DeepEqual(gotErr, c.expErr) { + if !errors.Is(gotErr, c.expErr) { t.Errorf("Unexpected error, want: %v, got: %v", c.expErr, gotErr) } } diff --git a/triedb/pathdb/reader.go b/triedb/pathdb/reader.go index 43d12a1678..842ac0972e 100644 --- a/triedb/pathdb/reader.go +++ b/triedb/pathdb/reader.go @@ -213,20 +213,24 @@ func (db *Database) HistoricReader(root common.Hash) (*HistoricalStateReader, er if !db.stateIndexer.inited() { return nil, errors.New("state histories haven't been fully indexed yet") } - // States at the current disk layer or above are directly accessible via - // db.StateReader. + // - States at the current disk layer or above are directly accessible + // via `db.StateReader`. // - // States older than the current disk layer (including the disk layer - // itself) are available through historic state access. - // - // Note: the requested state may refer to a stale historic state that has - // already been pruned. This function does not validate availability, as - // underlying states may be pruned dynamically. Validity is checked during - // each actual state retrieval. + // - States older than the current disk layer (including the disk layer + // itself) are available via `db.HistoricReader`. id := rawdb.ReadStateID(db.diskdb, root) if id == nil { return nil, fmt.Errorf("state %#x is not available", root) } + // Ensure the requested state is canonical, historical states on side chain + // are not accessible. + meta, err := readStateHistoryMeta(db.stateFreezer, *id+1) + if err != nil { + return nil, err // e.g., the referred state history has been pruned + } + if meta.parent != root { + return nil, fmt.Errorf("state %#x is not canonincal", root) + } return &HistoricalStateReader{ id: *id, db: db,