forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-24 05:36:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

triedb/pathdb, core: keep root->id mappings after truncation

Browse source
This commit is contained in:
Gary Rong 2025-08-27 11:50:18 +08:00
parent 6191f31508
commit 8984043821
8 changed files with 184 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
core/rawdb/accessors_state.go
View file

@ -119,13 +119,6 @@ func WriteStateID(db ethdb.KeyValueWriter, root common.Hash, id uint64) {
}
}
// DeleteStateID deletes the specified state lookup from the database.
func DeleteStateID(db ethdb.KeyValueWriter, root common.Hash) {
if err := db.Delete(stateIDKey(root)); err != nil {
log.Crit("Failed to delete state ID", "err", err)
}
}
// ReadPersistentStateID retrieves the id of the persistent state from the database.
func ReadPersistentStateID(db ethdb.KeyValueReader) uint64 {
data, _ := db.Get(persistentStateIDKey)

8
triedb/pathdb/database.go
View file

@ -334,7 +334,7 @@ func (db *Database) repairHistory() error {
}
// Truncate the extra state histories above in freezer in case it's not
// aligned with the disk layer. It might happen after a unclean shutdown.
pruned, err := truncateFromHead(db.diskdb, db.stateFreezer, id)
pruned, err := truncateFromHead(db.stateFreezer, id)
if err != nil {
log.Crit("Failed to truncate extra state histories", "err", err)
}
@ -590,7 +590,7 @@ func (db *Database) Recover(root common.Hash) error {
if err := db.diskdb.SyncKeyValue(); err != nil {
return err
}
_, err := truncateFromHead(db.diskdb, db.stateFreezer, dl.stateID())
_, err := truncateFromHead(db.stateFreezer, dl.stateID())
if err != nil {
return err
}
@ -615,14 +615,14 @@ func (db *Database) Recoverable(root common.Hash) bool {
return false
}
// This is a temporary workaround for the unavailability of the freezer in
// dev mode. As a consequence, the Pathdb loses the ability for deep reorg
// dev mode. As a consequence, the database loses the ability for deep reorg
// in certain cases.
// TODO(rjl493456442): Implement the in-memory ancient store.
if db.stateFreezer == nil {
return false
}
// Ensure the requested state is a canonical state and all state
// histories in range [id+1, disklayer.ID] are present and complete.
// histories in range [id+1, dl.ID] are present and complete.
return checkStateHistories(db.stateFreezer, *id+1, dl.stateID()-*id, func(m *meta) error {
if m.parent != root {
return errors.New("unexpected state history")

2
triedb/pathdb/disklayer.go
View file

@ -378,7 +378,7 @@ func (dl *diskLayer) writeStateHistory(diff *diffLayer) (bool, error) {
log.Debug("Skip tail truncation", "persistentID", persistentID, "tailID", tail+1, "headID", diff.stateID(), "limit", limit)
return true, nil
}
pruned, err := truncateFromTail(dl.db.diskdb, dl.db.stateFreezer, newFirst-1)
pruned, err := truncateFromTail(dl.db.stateFreezer, newFirst-1)
if err != nil {
return false, err
}

87
triedb/pathdb/history.go Normal file
View file

@ -0,0 +1,87 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/
package pathdb
import (
"errors"
"fmt"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
)
var (
errHeadTruncationOutOfRange = errors.New("history head truncation out of range")
errTailTruncationOutOfRange = errors.New("history tail truncation out of range")
)
// truncateFromHead removes excess elements from the head of the freezer based
// on the given parameters. It returns the number of items that were removed.
func truncateFromHead(store ethdb.AncientStore, nhead uint64) (int, error) {
ohead, err := store.Ancients()
if err != nil {
return 0, err
}
otail, err := store.Tail()
if err != nil {
return 0, err
}
log.Info("Truncating from head", "ohead", ohead, "tail", otail, "nhead", nhead)
// Ensure that the truncation target falls within the valid range.
if ohead < nhead || nhead < otail {
return 0, fmt.Errorf("%w, tail: %d, head: %d, target: %d", errHeadTruncationOutOfRange, otail, ohead, nhead)
}
// Short circuit if nothing to truncate.
if ohead == nhead {
return 0, nil
}
ohead, err = store.TruncateHead(nhead)
if err != nil {
return 0, err
}
// Associated root->id mappings are left in the database and wait
// for overwriting.
return int(ohead - nhead), nil
}
// truncateFromTail removes excess elements from the end of the freezer based
// on the given parameters. It returns the number of items that were removed.
func truncateFromTail(store ethdb.AncientStore, ntail uint64) (int, error) {
ohead, err := store.Ancients()
if err != nil {
return 0, err
}
otail, err := store.Tail()
if err != nil {
return 0, err
}
// Ensure that the truncation target falls within the valid range.
if otail > ntail || ntail > ohead {
return 0, fmt.Errorf("%w, tail: %d, head: %d, target: %d", errTailTruncationOutOfRange, otail, ohead, ntail)
}
// Short circuit if nothing to truncate.
if otail == ntail {
return 0, nil
}
otail, err = store.TruncateTail(ntail)
if err != nil {
return 0, err
}
// Associated root->id mappings are left in the database.
return int(ntail - otail), nil
}

36
triedb/pathdb/history_reader_test.go
View file

@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/internal/testrand"
)
func waitIndexing(db *Database) {
@ -36,7 +37,7 @@ func waitIndexing(db *Database) {
}
}
func checkHistoricState(env *tester, root common.Hash, hr *historyReader) error {
func checkHistoricalState(env *tester, root common.Hash, hr *historyReader) error {
// Short circuit if the historical state is no longer available
if rawdb.ReadStateID(env.db.diskdb, root) == nil {
return nil
@ -139,7 +140,7 @@ func testHistoryReader(t *testing.T, historyLimit uint64) {
if root == dRoot {
break
}
if err := checkHistoricState(env, root, hr); err != nil {
if err := checkHistoricalState(env, root, hr); err != nil {
t.Fatal(err)
}
}
@ -152,8 +153,37 @@ func testHistoryReader(t *testing.T, historyLimit uint64) {
if root == dRoot {
break
}
if err := checkHistoricState(env, root, hr); err != nil {
if err := checkHistoricalState(env, root, hr); err != nil {
t.Fatal(err)
}
}
}
func TestHistoricalStateReader(t *testing.T) {
maxDiffLayers = 4
defer func() {
maxDiffLayers = 128
}()
//log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
env := newTester(t, 0, false, 64, true, "")
defer env.release()
waitIndexing(env.db)
// non-canonical state
fakeRoot := testrand.Hash()
rawdb.WriteStateID(env.db.diskdb, fakeRoot, 10)
_, err := env.db.HistoricReader(fakeRoot)
if err == nil {
t.Fatal("expected error")
}
t.Log(err)
// canonical state
realRoot := env.roots[9]
_, err = env.db.HistoricReader(realRoot)
if err != nil {
t.Fatalf("Unexpected error: %v", err)
}
}

102
triedb/pathdb/history_state.go
View file

@ -504,6 +504,20 @@ func (h *stateHistory) decode(accountData, storageData, accountIndexes, storageI
return nil
}
// readStateHistoryMeta reads the metadata of state history with the specified id.
func readStateHistoryMeta(reader ethdb.AncientReader, id uint64) (*meta, error) {
data := rawdb.ReadStateHistoryMeta(reader, id)
if len(data) == 0 {
return nil, fmt.Errorf("metadata is not found, %d", id)
}
var m meta
err := m.decode(data)
if err != nil {
return nil, err
}
return &m, nil
}
// readStateHistory reads a single state history records with the specified id.
func readStateHistory(reader ethdb.AncientReader, id uint64) (*stateHistory, error) {
mData, accountIndexes, storageIndexes, accountData, storageData, err := rawdb.ReadStateHistory(reader, id)
@ -568,8 +582,8 @@ func writeStateHistory(writer ethdb.AncientWriter, dl *diffLayer) error {
return nil
}
// checkStateHistories retrieves a batch of meta objects with the specified range
// and performs the callback on each item.
// checkStateHistories retrieves a batch of metadata objects with the specified
// range and performs the callback on each item.
func checkStateHistories(reader ethdb.AncientReader, start, count uint64, check func(*meta) error) error {
for count > 0 {
number := count
@ -594,87 +608,3 @@ func checkStateHistories(reader ethdb.AncientReader, start, count uint64, check
}
return nil
}
// truncateFromHead removes the extra state histories from the head with the given
// parameters. It returns the number of items removed from the head.
func truncateFromHead(db ethdb.Batcher, store ethdb.AncientStore, nhead uint64) (int, error) {
ohead, err := store.Ancients()
if err != nil {
return 0, err
}
otail, err := store.Tail()
if err != nil {
return 0, err
}
// Ensure that the truncation target falls within the specified range.
if ohead < nhead || nhead < otail {
return 0, fmt.Errorf("out of range, tail: %d, head: %d, target: %d", otail, ohead, nhead)
}
// Short circuit if nothing to truncate.
if ohead == nhead {
return 0, nil
}
// Load the meta objects in range [nhead+1, ohead]
blobs, err := rawdb.ReadStateHistoryMetaList(store, nhead+1, ohead-nhead)
if err != nil {
return 0, err
}
batch := db.NewBatch()
for _, blob := range blobs {
var m meta
if err := m.decode(blob); err != nil {
return 0, err
}
rawdb.DeleteStateID(batch, m.root)
}
if err := batch.Write(); err != nil {
return 0, err
}
ohead, err = store.TruncateHead(nhead)
if err != nil {
return 0, err
}
return int(ohead - nhead), nil
}
// truncateFromTail removes the extra state histories from the tail with the given
// parameters. It returns the number of items removed from the tail.
func truncateFromTail(db ethdb.Batcher, store ethdb.AncientStore, ntail uint64) (int, error) {
ohead, err := store.Ancients()
if err != nil {
return 0, err
}
otail, err := store.Tail()
if err != nil {
return 0, err
}
// Ensure that the truncation target falls within the specified range.
if otail > ntail || ntail > ohead {
return 0, fmt.Errorf("out of range, tail: %d, head: %d, target: %d", otail, ohead, ntail)
}
// Short circuit if nothing to truncate.
if otail == ntail {
return 0, nil
}
// Load the meta objects in range [otail+1, ntail]
blobs, err := rawdb.ReadStateHistoryMetaList(store, otail+1, ntail-otail)
if err != nil {
return 0, err
}
batch := db.NewBatch()
for _, blob := range blobs {
var m meta
if err := m.decode(blob); err != nil {
return 0, err
}
rawdb.DeleteStateID(batch, m.root)
}
if err := batch.Write(); err != nil {
return 0, err
}
otail, err = store.TruncateTail(ntail)
if err != nil {
return 0, err
}
return int(ntail - otail), nil
}

71
triedb/pathdb/history_state_test.go
View file

@ -108,7 +108,7 @@ func testEncodeDecodeStateHistory(t *testing.T, rawStorageKey bool) {
}
}
func checkStateHistory(t *testing.T, db ethdb.KeyValueReader, freezer ethdb.AncientReader, id uint64, root common.Hash, exist bool) {
func checkStateHistory(t *testing.T, freezer ethdb.AncientReader, id uint64, exist bool) {
blob := rawdb.ReadStateHistoryMeta(freezer, id)
if exist && len(blob) == 0 {
t.Fatalf("Failed to load trie history, %d", id)
@ -116,25 +116,17 @@ func checkStateHistory(t *testing.T, db ethdb.KeyValueReader, freezer ethdb.Anci
if !exist && len(blob) != 0 {
t.Fatalf("Unexpected trie history, %d", id)
}
if exist && rawdb.ReadStateID(db, root) == nil {
t.Fatalf("Root->ID mapping is not found, %d", id)
}
if !exist && rawdb.ReadStateID(db, root) != nil {
t.Fatalf("Unexpected root->ID mapping, %d", id)
}
}
func checkHistoriesInRange(t *testing.T, db ethdb.KeyValueReader, freezer ethdb.AncientReader, from, to uint64, roots []common.Hash, exist bool) {
for i, j := from, 0; i <= to; i, j = i+1, j+1 {
checkStateHistory(t, db, freezer, i, roots[j], exist)
func checkHistoriesInRange(t *testing.T, freezer ethdb.AncientReader, from, to uint64, exist bool) {
for i := from; i <= to; i = i + 1 {
checkStateHistory(t, freezer, i, exist)
}
}
func TestTruncateHeadStateHistory(t *testing.T) {
var (
roots []common.Hash
hs = makeStateHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false)
)
defer freezer.Close()
@ -142,27 +134,23 @@ func TestTruncateHeadStateHistory(t *testing.T) {
for i := 0; i < len(hs); i++ {
accountData, storageData, accountIndex, storageIndex := hs[i].encode()
rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData)
rawdb.WriteStateID(db, hs[i].meta.root, uint64(i+1))
roots = append(roots, hs[i].meta.root)
}
for size := len(hs); size > 0; size-- {
pruned, err := truncateFromHead(db, freezer, uint64(size-1))
pruned, err := truncateFromHead(freezer, uint64(size-1))
if err != nil {
t.Fatalf("Failed to truncate from head %v", err)
}
if pruned != 1 {
t.Error("Unexpected pruned items", "want", 1, "got", pruned)
}
checkHistoriesInRange(t, db, freezer, uint64(size), uint64(10), roots[size-1:], false)
checkHistoriesInRange(t, db, freezer, uint64(1), uint64(size-1), roots[:size-1], true)
checkHistoriesInRange(t, freezer, uint64(size), uint64(10), false)
checkHistoriesInRange(t, freezer, uint64(1), uint64(size-1), true)
}
}
func TestTruncateTailStateHistory(t *testing.T) {
var (
roots []common.Hash
hs = makeStateHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false)
)
defer freezer.Close()
@ -170,16 +158,14 @@ func TestTruncateTailStateHistory(t *testing.T) {
for i := 0; i < len(hs); i++ {
accountData, storageData, accountIndex, storageIndex := hs[i].encode()
rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData)
rawdb.WriteStateID(db, hs[i].meta.root, uint64(i+1))
roots = append(roots, hs[i].meta.root)
}
for newTail := 1; newTail < len(hs); newTail++ {
pruned, _ := truncateFromTail(db, freezer, uint64(newTail))
pruned, _ := truncateFromTail(freezer, uint64(newTail))
if pruned != 1 {
t.Error("Unexpected pruned items", "want", 1, "got", pruned)
}
checkHistoriesInRange(t, db, freezer, uint64(1), uint64(newTail), roots[:newTail], false)
checkHistoriesInRange(t, db, freezer, uint64(newTail+1), uint64(10), roots[newTail:], true)
checkHistoriesInRange(t, freezer, uint64(1), uint64(newTail), false)
checkHistoriesInRange(t, freezer, uint64(newTail+1), uint64(10), true)
}
}
@ -191,21 +177,29 @@ func TestTruncateTailStateHistories(t *testing.T) {
minUnpruned uint64
empty bool
}{
// history: id [10]
{
1, 9, 9, 10, false,
limit: 1,
expPruned: 9,
maxPruned: 9, minUnpruned: 10, empty: false,
},
// history: none
{
0, 10, 10, 0 /* no meaning */, true,
limit: 0,
expPruned: 10,
empty: true,
},
// history: id [1:10]
{
10, 0, 0, 1, false,
limit: 10,
expPruned: 0,
maxPruned: 0,
minUnpruned: 1,
},
}
for i, c := range cases {
var (
roots []common.Hash
hs = makeStateHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir()+fmt.Sprintf("%d", i), false, false)
)
defer freezer.Close()
@ -213,19 +207,16 @@ func TestTruncateTailStateHistories(t *testing.T) {
for i := 0; i < len(hs); i++ {
accountData, storageData, accountIndex, storageIndex := hs[i].encode()
rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData)
rawdb.WriteStateID(db, hs[i].meta.root, uint64(i+1))
roots = append(roots, hs[i].meta.root)
}
pruned, _ := truncateFromTail(db, freezer, uint64(10)-c.limit)
pruned, _ := truncateFromTail(freezer, uint64(10)-c.limit)
if pruned != c.expPruned {
t.Error("Unexpected pruned items", "want", c.expPruned, "got", pruned)
}
if c.empty {
checkHistoriesInRange(t, db, freezer, uint64(1), uint64(10), roots, false)
checkHistoriesInRange(t, freezer, uint64(1), uint64(10), false)
} else {
tail := 10 - int(c.limit)
checkHistoriesInRange(t, db, freezer, uint64(1), c.maxPruned, roots[:tail], false)
checkHistoriesInRange(t, db, freezer, c.minUnpruned, uint64(10), roots[tail:], true)
checkHistoriesInRange(t, freezer, uint64(1), c.maxPruned, false)
checkHistoriesInRange(t, freezer, c.minUnpruned, uint64(10), true)
}
}
}
@ -233,7 +224,6 @@ func TestTruncateTailStateHistories(t *testing.T) {
func TestTruncateOutOfRange(t *testing.T) {
var (
hs = makeStateHistories(10)
db = rawdb.NewMemoryDatabase()
freezer, _ = rawdb.NewStateFreezer(t.TempDir(), false, false)
)
defer freezer.Close()
@ -241,9 +231,8 @@ func TestTruncateOutOfRange(t *testing.T) {
for i := 0; i < len(hs); i++ {
accountData, storageData, accountIndex, storageIndex := hs[i].encode()
rawdb.WriteStateHistory(freezer, uint64(i+1), hs[i].meta.encode(), accountIndex, storageIndex, accountData, storageData)
rawdb.WriteStateID(db, hs[i].meta.root, uint64(i+1))
}
truncateFromTail(db, freezer, uint64(len(hs)/2))
truncateFromTail(freezer, uint64(len(hs)/2))
// Ensure of-out-range truncations are rejected correctly.
head, _ := freezer.Ancients()
@ -264,9 +253,9 @@ func TestTruncateOutOfRange(t *testing.T) {
for _, c := range cases {
var gotErr error
if c.mode == 0 {
_, gotErr = truncateFromHead(db, freezer, c.target)
_, gotErr = truncateFromHead(freezer, c.target)
} else {
_, gotErr = truncateFromTail(db, freezer, c.target)
_, gotErr = truncateFromTail(freezer, c.target)
}
if !reflect.DeepEqual(gotErr, c.expErr) {
t.Errorf("Unexpected error, want: %v, got: %v", c.expErr, gotErr)

22
triedb/pathdb/reader.go
View file

@ -213,20 +213,24 @@ func (db *Database) HistoricReader(root common.Hash) (*HistoricalStateReader, er
if !db.stateIndexer.inited() {
return nil, errors.New("state histories haven't been fully indexed yet")
}
// States at the current disk layer or above are directly accessible via
// db.StateReader.
// - States at the current disk layer or above are directly accessible
// via `db.StateReader`.
//
// States older than the current disk layer (including the disk layer
// itself) are available through historic state access.
//
// Note: the requested state may refer to a stale historic state that has
// already been pruned. This function does not validate availability, as
// underlying states may be pruned dynamically. Validity is checked during
// each actual state retrieval.
// - States older than the current disk layer (including the disk layer
// itself) are available via `db.HistoricReader`.
id := rawdb.ReadStateID(db.diskdb, root)
if id == nil {
return nil, fmt.Errorf("state %#x is not available", root)
}
// Ensure the requested state is canonical, historical states on side chain
// are not accessible.
meta, err := readStateHistoryMeta(db.stateFreezer, *id)
if err != nil {
return nil, err // e.g., the referred state history has been pruned
}
if meta.root != root {
return nil, fmt.Errorf("state %#x is not canonincal", root)
}
return &HistoricalStateReader{
id: *id,
db: db,