trie, cmd/geth: add archive verify command, Walk(), and archiver improvements

Add Trie.Walk() for exhaustive traversal that resolves expired nodes
with hash verification. Add `archive verify` subcommand that walks
the full state (account + storage tries) to validate all archived
data can be correctly resurrected.

Delete both the journal KV entry and file after archiving to force
geth to restart with a bare disk layer, rewinding the chain head to
the persistent disk state and re-executing blocks.

Also adds markSubtreeDirty() to resolveExpiredNodeData() so that all
nodes in a resolved expired subtree are captured in the NodeSet during
commit — preventing them from being lost between diff layers and the
raw DB.
This commit is contained in:
Guillaume Ballet 2026-02-13 09:13:23 +01:00
parent 8d2125e4fd
commit 5119945e25
15 changed files with 1032 additions and 282 deletions

View file

@ -17,8 +17,10 @@
package main package main
import ( import (
"encoding/binary"
"errors" "errors"
"fmt" "fmt"
"os"
"path/filepath" "path/filepath"
"slices" "slices"
"time" "time"
@ -26,10 +28,14 @@ import (
"github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/archive" "github.com/ethereum/go-ethereum/trie/archive"
"github.com/ethereum/go-ethereum/triedb/database"
"github.com/urfave/cli/v2" "github.com/urfave/cli/v2"
) )
@ -51,14 +57,73 @@ var (
} }
// Commands // Commands
archiveCheckNodeFlag = &cli.StringFlag{
Name: "owner",
Usage: "Owner hash (hex) for the trie node to check",
}
archiveCheckPathFlag = &cli.StringFlag{
Name: "path",
Usage: "Path (hex nibbles) of the trie node to check",
}
archiveCommand = &cli.Command{ archiveCommand = &cli.Command{
Name: "archive", Name: "archive",
Usage: "Archive state trie nodes to reduce database size", Usage: "Archive state trie nodes to reduce database size",
Subcommands: []*cli.Command{ Subcommands: []*cli.Command{
archiveGenerateCmd, archiveGenerateCmd,
archiveVerifyCmd,
archiveDeleteJournalCmd,
archiveCheckNodeCmd,
}, },
} }
archiveCheckNodeCmd = &cli.Command{
Name: "check-node",
Usage: "Check if a specific trie node exists in the raw DB",
Action: archiveCheckNode,
Flags: slices.Concat([]cli.Flag{
archiveCheckNodeFlag,
archiveCheckPathFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
}
archiveDeleteJournalCmd = &cli.Command{
Name: "delete-journal",
Usage: "Delete the pathdb journal to force a clean restart",
Action: archiveDeleteJournal,
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Description: `
Deletes the pathdb journal (TrieJournal key and merkle.journal file) from the
database. This forces geth to restart with a bare disk layer, discarding any
in-memory diff layers that may be inconsistent with archived state.
Use this after running 'archive generate' if geth was started in between and
recreated the journal.
Examples:
geth archive delete-journal --datadir /path/to/datadir
geth archive delete-journal --hoodi
`,
}
archiveVerifyCmd = &cli.Command{
Name: "verify",
Usage: "Verify all archived nodes can be correctly resurrected",
Action: archiveVerify,
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Description: `
Walks the entire state trie, resolving every expired node from the archive
file and verifying that the reconstructed subtree hash matches the original.
Also walks all storage tries referenced by accounts.
The database is opened read-only. No modifications are made.
Examples:
geth archive verify --datadir /path/to/datadir
geth archive verify --hoodi
`,
}
archiveGenerateCmd = &cli.Command{ archiveGenerateCmd = &cli.Command{
Name: "generate", Name: "generate",
Usage: "Generate archive files from height-3 subtrees", Usage: "Generate archive files from height-3 subtrees",
@ -77,22 +142,56 @@ that references the archive file offset and size.
Height is measured from leaves: leaves=0, parents=1, etc. A height-3 node Height is measured from leaves: leaves=0, parents=1, etc. A height-3 node
has leaves at most 3 levels below it. has leaves at most 3 levels below it.
The archiver reads trie nodes directly from the persistent database layer,
bypassing any in-memory diff layers. This ensures consistency between the
data it reads and the data it modifies.
Examples: Examples:
# Archive from head state # Archive from the persistent disk state
geth archive generate --datadir /path/to/datadir geth archive generate --datadir /path/to/datadir
# Dry run to see what would be archived # Dry run to see what would be archived
geth archive generate --dry-run --datadir /path/to/datadir geth archive generate --dry-run --datadir /path/to/datadir
# Archive from a specific state root
geth archive generate 0x1234...abcd --datadir /path/to/datadir
# Custom output and compaction interval # Custom output and compaction interval
geth archive generate --output /path/to/archive --compaction-interval 500 geth archive generate --output /path/to/archive --compaction-interval 500
`, `,
} }
) )
// rawDBNodeReader implements database.NodeReader by reading trie nodes directly
// from the raw key-value database, bypassing pathdb's in-memory diff layers.
// This ensures the archiver sees the same trie state it modifies.
type rawDBNodeReader struct {
db ethdb.KeyValueReader
}
func (r *rawDBNodeReader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
var blob []byte
if owner == (common.Hash{}) {
blob = rawdb.ReadAccountTrieNode(r.db, path)
} else {
blob = rawdb.ReadStorageTrieNode(r.db, owner, path)
}
// Skip hash verification: the raw DB may contain expiredNode markers
// (blob[0] == 0x00) which have different hashes than the original nodes.
return blob, nil
}
// rawDBNodeDatabase implements database.NodeDatabase using direct raw DB reads.
type rawDBNodeDatabase struct {
db ethdb.KeyValueReader
root common.Hash
}
func (d *rawDBNodeDatabase) NodeReader(stateRoot common.Hash) (database.NodeReader, error) {
// Only allow reading the persistent disk root state
if stateRoot != d.root {
return nil, fmt.Errorf("raw DB reader only supports disk root %x, got %x", d.root, stateRoot)
}
return &rawDBNodeReader{db: d.db}, nil
}
func archiveGenerate(ctx *cli.Context) error { func archiveGenerate(ctx *cli.Context) error {
// 1. Setup node and databases // 1. Setup node and databases
stack, _ := makeConfigNode(ctx) stack, _ := makeConfigNode(ctx)
@ -109,27 +208,25 @@ func archiveGenerate(ctx *cli.Context) error {
return fmt.Errorf("archive generation requires path-based state scheme, got: %s", scheme) return fmt.Errorf("archive generation requires path-based state scheme, got: %s", scheme)
} }
triedb := utils.MakeTrieDatabase(ctx, stack, chaindb, false, false, false) // 2. Determine the persistent disk state root.
defer triedb.Close() //
// The archiver reads and writes directly to the raw key-value database,
// 2. Determine state root // bypassing pathdb's in-memory diff layers. This avoids the inconsistency
var root common.Hash // where diff layers shadow expiredNode markers written to disk.
if ctx.NArg() > 0 { //
root = common.HexToHash(ctx.Args().First()) // The disk root is computed by hashing the account trie root node stored
log.Info("Using specified state root", "root", root) // in the raw database. This root corresponds to the last state that was
} else { // fully persisted (i.e., PersistentStateID), which matches the canonical
headBlock := rawdb.ReadHeadBlock(chaindb) // chain head.
if headBlock == nil { rootBlob := rawdb.ReadAccountTrieNode(chaindb, nil)
return errors.New("no head block found - specify a state root or sync the chain first") if len(rootBlob) == 0 {
}
root = headBlock.Root()
log.Info("Using head block state", "number", headBlock.NumberU64(), "root", root)
}
// Verify the state exists
if !rawdb.HasAccountTrieNode(chaindb, nil) {
return errors.New("state trie not found in database") return errors.New("state trie not found in database")
} }
root := crypto.Keccak256Hash(rootBlob)
log.Info("Using persistent disk state root", "root", root)
// Create a raw DB node reader that bypasses pathdb layers
nodeDB := &rawDBNodeDatabase{db: chaindb, root: root}
// 3. Open archive writer (unless dry-run) // 3. Open archive writer (unless dry-run)
var writer *archive.ArchiveWriter var writer *archive.ArchiveWriter
@ -153,7 +250,7 @@ func archiveGenerate(ctx *cli.Context) error {
// 4. Create and run archiver // 4. Create and run archiver
archiver := trie.NewArchiver( archiver := trie.NewArchiver(
chaindb, chaindb,
triedb, nodeDB,
writer, writer,
ctx.Uint64(archiveCompactionIntervalFlag.Name), ctx.Uint64(archiveCompactionIntervalFlag.Name),
dryRun, dryRun,
@ -174,6 +271,27 @@ func archiveGenerate(ctx *cli.Context) error {
} }
} }
if !dryRun {
// Delete the pathdb journal. The archiver modified the raw DB
// underneath the diff layers, so the journal's buffered state is
// inconsistent. Deleting forces geth to restart with a bare disk
// layer and rewind the chain head to the disk state.
if err := chaindb.Delete([]byte("TrieJournal")); err != nil {
log.Warn("Failed to delete pathdb journal key", "err", err)
}
log.Info("Deleted pathdb journal to force clean restart")
// Delete journal file(s) - check both legacy and current locations
for _, dir := range []string{"triedb", ""} {
for _, name := range []string{"merkle.journal", "verkle.journal"} {
journalFile := filepath.Join(stack.ResolvePath(dir), name)
if err := os.Remove(journalFile); err == nil {
log.Info("Deleted journal file", "path", journalFile)
}
}
}
}
// 6. Print summary // 6. Print summary
var archiveSize uint64 var archiveSize uint64
if writer != nil { if writer != nil {
@ -194,6 +312,258 @@ func archiveGenerate(ctx *cli.Context) error {
return nil return nil
} }
func archiveVerify(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
// Open database read-only
chaindb := utils.MakeChainDatabase(ctx, stack, true)
defer chaindb.Close()
scheme := cycleCheckScheme(ctx, chaindb)
if scheme != rawdb.PathScheme {
return fmt.Errorf("archive verify requires path-based state scheme, got: %s", scheme)
}
// Set archive data dir so ArchivedNodeResolver can find the file
// ResolvePath("") returns the node's data directory (e.g. .ethereum/hoodi/geth),
// but ArchivedNodeResolver expects the instance directory (.ethereum/hoodi)
// since it appends "geth/nodearchive" itself.
archive.ArchiveDataDir = filepath.Dir(stack.ResolvePath(""))
// Compute disk root
rootBlob := rawdb.ReadAccountTrieNode(chaindb, nil)
if len(rootBlob) == 0 {
return errors.New("state trie not found in database")
}
root := crypto.Keccak256Hash(rootBlob)
log.Info("Verifying archived nodes", "root", root)
nodeDB := &rawDBNodeDatabase{db: chaindb, root: root}
// Open account trie
accountTrie, err := trie.New(trie.StateTrieID(root), nodeDB)
if err != nil {
return fmt.Errorf("failed to open account trie: %w", err)
}
var (
totalAccounts int
totalStorageTries int
totalLeaves int
totalExpired int
totalErrors int
start = time.Now()
lastLog = time.Now()
)
// Walk the account trie — this resolves all expired nodes and verifies hashes
accountStats, err := accountTrie.Walk(func(path []byte, value []byte) error {
totalAccounts++
if time.Since(lastLog) > 30*time.Second {
log.Info("Verification progress",
"accounts", totalAccounts,
"storageTries", totalStorageTries,
"leaves", totalLeaves,
"expired", totalExpired,
"errors", totalErrors)
lastLog = time.Now()
}
// Decode account to check for storage trie
var acc types.StateAccount
if err := rlp.DecodeBytes(value, &acc); err != nil {
log.Warn("Failed to decode account", "err", err)
totalErrors++
return nil // continue walking
}
if acc.Root == types.EmptyRootHash {
return nil
}
// Open and walk storage trie.
// path is hex-nibble encoded (with a 16 terminator from the trie key),
// so convert nibble pairs back to the 32-byte account hash.
nibbles := path
if len(nibbles) > 0 && nibbles[len(nibbles)-1] == 16 {
nibbles = nibbles[:len(nibbles)-1]
}
keyBytes := make([]byte, len(nibbles)/2)
for i := 0; i < len(nibbles); i += 2 {
keyBytes[i/2] = nibbles[i]<<4 | nibbles[i+1]
}
accountHash := common.BytesToHash(keyBytes)
storageID := trie.StorageTrieID(root, accountHash, acc.Root)
storageTrie, err := trie.New(storageID, nodeDB)
if err != nil {
log.Warn("Failed to open storage trie", "account", accountHash, "err", err)
totalErrors++
return nil
}
storageStats, err := storageTrie.Walk(func(spath []byte, svalue []byte) error {
return nil
})
if err != nil {
log.Warn("Storage trie walk failed", "account", accountHash, "err", err)
totalErrors++
return nil
}
totalStorageTries++
totalLeaves += storageStats.Leaves
totalExpired += storageStats.ExpiredResolved
return nil
})
if err != nil {
return fmt.Errorf("account trie walk failed: %w", err)
}
totalLeaves += accountStats.Leaves
totalExpired += accountStats.ExpiredResolved
log.Info("Archive verification complete",
"accounts", totalAccounts,
"storageTries", totalStorageTries,
"totalLeaves", totalLeaves,
"expiredResolved", totalExpired,
"errors", totalErrors,
"elapsed", common.PrettyDuration(time.Since(start)))
if totalErrors > 0 {
return fmt.Errorf("verification completed with %d errors", totalErrors)
}
return nil
}
func archiveDeleteJournal(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chaindb := utils.MakeChainDatabase(ctx, stack, false)
defer chaindb.Close()
// Delete the pathdb journal KV key
if err := chaindb.Delete([]byte("TrieJournal")); err != nil {
log.Warn("Failed to delete pathdb journal key", "err", err)
} else {
log.Info("Deleted pathdb journal key (TrieJournal)")
}
// Delete the journal file(s) - check both legacy and current locations
for _, dir := range []string{"triedb", ""} {
for _, name := range []string{"merkle.journal", "verkle.journal"} {
journalFile := filepath.Join(stack.ResolvePath(dir), name)
if err := os.Remove(journalFile); err == nil {
log.Info("Deleted journal file", "path", journalFile)
} else if !os.IsNotExist(err) {
log.Warn("Failed to delete journal file", "path", journalFile, "err", err)
}
}
}
return nil
}
func archiveCheckNode(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chaindb := utils.MakeChainDatabase(ctx, stack, true)
defer chaindb.Close()
ownerHex := ctx.String(archiveCheckNodeFlag.Name)
pathHex := ctx.String(archiveCheckPathFlag.Name)
if ownerHex == "" {
return errors.New("--owner flag is required")
}
owner := common.HexToHash(ownerHex)
// Parse path: hex nibbles like "08" → []byte{0, 8}
var path []byte
for _, c := range pathHex {
var nibble byte
switch {
case c >= '0' && c <= '9':
nibble = byte(c - '0')
case c >= 'a' && c <= 'f':
nibble = byte(c-'a') + 10
case c >= 'A' && c <= 'F':
nibble = byte(c-'A') + 10
default:
return fmt.Errorf("invalid hex char in path: %c", c)
}
path = append(path, nibble)
}
log.Info("Checking node in raw DB", "owner", owner, "path", fmt.Sprintf("%x", path))
// Read the node directly from the raw DB
isAccount := owner == (common.Hash{})
// Check the target path and all prefixes up to root
for i := len(path); i >= 0; i-- {
subpath := path[:i]
var blob []byte
if isAccount {
blob = rawdb.ReadAccountTrieNode(chaindb, subpath)
} else {
blob = rawdb.ReadStorageTrieNode(chaindb, owner, subpath)
}
status := "MISSING"
details := ""
if len(blob) > 0 {
if blob[0] == 0x00 {
status = "EXPIRED"
if len(blob) == 17 {
offset := binary.BigEndian.Uint64(blob[1:9])
size := binary.BigEndian.Uint64(blob[9:17])
details = fmt.Sprintf("offset=%d size=%d", offset, size)
}
} else {
status = fmt.Sprintf("PRESENT (%d bytes, first=0x%02x)", len(blob), blob[0])
}
}
label := "prefix"
if i == len(path) {
label = "TARGET"
}
if i == 0 {
label = "ROOT"
}
log.Info("Node check",
"label", label,
"path", fmt.Sprintf("%x", subpath),
"pathLen", i,
"status", status,
"details", details)
}
// Also check a few child paths to see what's below the target
for nibble := byte(0); nibble < 16; nibble++ {
childPath := append(append([]byte{}, path...), nibble)
var blob []byte
if isAccount {
blob = rawdb.ReadAccountTrieNode(chaindb, childPath)
} else {
blob = rawdb.ReadStorageTrieNode(chaindb, owner, childPath)
}
if len(blob) > 0 {
status := fmt.Sprintf("PRESENT (%d bytes, first=0x%02x)", len(blob), blob[0])
if blob[0] == 0x00 && len(blob) == 17 {
offset := binary.BigEndian.Uint64(blob[1:9])
size := binary.BigEndian.Uint64(blob[9:17])
status = fmt.Sprintf("EXPIRED offset=%d size=%d", offset, size)
}
log.Info("Child node", "path", fmt.Sprintf("%x", childPath), "status", status)
}
}
return nil
}
// cycleCheckScheme returns the state scheme for the database. // cycleCheckScheme returns the state scheme for the database.
// It's a helper to check what scheme is in use. // It's a helper to check what scheme is in use.
func cycleCheckScheme(ctx *cli.Context, db ethdb.Database) string { func cycleCheckScheme(ctx *cli.Context, db ethdb.Database) string {

View file

@ -38,6 +38,7 @@ import (
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie/archive"
"github.com/gofrs/flock" "github.com/gofrs/flock"
) )
@ -85,6 +86,7 @@ func New(conf *Config) (*Node, error) {
return nil, err return nil, err
} }
conf.DataDir = absdatadir conf.DataDir = absdatadir
archive.ArchiveDataDir = absdatadir
} }
if conf.Logger == nil { if conf.Logger == nil {
conf.Logger = log.New() conf.Logger = log.New()

View file

@ -22,6 +22,7 @@ import (
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
) )
@ -45,13 +46,16 @@ type Record struct {
Value []byte Value []byte
} }
// ArchiveDataDir is the data directory where the archive file is stored.
var ArchiveDataDir string
// ArchivedNodeResolver takes a buffer containing the archive data // ArchivedNodeResolver takes a buffer containing the archive data
// held by an expiring node (an offset and a size) and returns a // held by an expiring node (an offset and a size) and returns a
// list of records, which is a list of serialized leaf nodes. The // list of records, which is a list of serialized leaf nodes. The
// caller knows the context (MPT, binary trie) and is responsible // caller knows the context (MPT, binary trie) and is responsible
// for decoding the nodes. // for decoding the nodes.
func ArchivedNodeResolver(offset, size uint64) ([]*Record, error) { func ArchivedNodeResolver(offset, size uint64) ([]*Record, error) {
file, err := os.Open("nodearchive") file, err := os.Open(filepath.Join(ArchiveDataDir, "geth", "nodearchive"))
if err != nil { if err != nil {
return nil, fmt.Errorf("error opening archive file: %w", err) return nil, fmt.Errorf("error opening archive file: %w", err)
} }

View file

@ -19,6 +19,7 @@ package trie
import ( import (
"encoding/binary" "encoding/binary"
"fmt" "fmt"
"time"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/rawdb"
@ -39,6 +40,7 @@ type subtreeInfo struct {
height int // Height of subtree (from leaves) height int // Height of subtree (from leaves)
leaves []*archive.Record // All leaf records (relative path + encoded node) leaves []*archive.Record // All leaf records (relative path + encoded node)
nodePaths [][]byte // Paths of all nodes to delete nodePaths [][]byte // Paths of all nodes to delete
rootHash common.Hash // Hash of the original subtree root (for verification)
} }
// Archiver handles the archival process of trie nodes. // Archiver handles the archival process of trie nodes.
@ -79,7 +81,7 @@ func NewArchiver(db ethdb.Database, triedb database.NodeDatabase,
} }
// ProcessState archives subtrees from the given state root. // ProcessState archives subtrees from the given state root.
// It processes the account trie first, then all storage tries. // It processes storage tries first, then the account trie.
func (a *Archiver) ProcessState(root common.Hash) error { func (a *Archiver) ProcessState(root common.Hash) error {
a.stateRoot = root a.stateRoot = root
@ -141,7 +143,12 @@ func (a *Archiver) processTrie(owner common.Hash, t *Trie) error {
subtrees := a.findHeight3Subtrees(t.root, nil, owner) subtrees := a.findHeight3Subtrees(t.root, nil, owner)
log.Info("Found subtrees to archive", "owner", owner, "count", len(subtrees)) log.Info("Found subtrees to archive", "owner", owner, "count", len(subtrees))
for _, info := range subtrees { lastLog := time.Now()
for i, info := range subtrees {
if time.Since(lastLog) > 30*time.Second {
log.Info("Archiving subtrees", "owner", owner, "progress", fmt.Sprintf("%d/%d", i, len(subtrees)), "archived", a.subtreesArchived)
lastLog = time.Now()
}
if err := a.archiveSubtree(info); err != nil { if err := a.archiveSubtree(info); err != nil {
log.Warn("Failed to archive subtree", "path", common.Bytes2Hex(info.path), "err", err) log.Warn("Failed to archive subtree", "path", common.Bytes2Hex(info.path), "err", err)
continue continue
@ -159,18 +166,52 @@ func (a *Archiver) processTrie(owner common.Hash, t *Trie) error {
// findHeight3Subtrees recursively finds all subtrees with height == 3. // findHeight3Subtrees recursively finds all subtrees with height == 3.
// Height is measured from leaves: leaves=0, their parents=1, etc. // Height is measured from leaves: leaves=0, their parents=1, etc.
func (a *Archiver) findHeight3Subtrees(n node, path []byte, owner common.Hash) []*subtreeInfo { func (a *Archiver) findHeight3Subtrees(n node, path []byte, owner common.Hash) []*subtreeInfo {
info := a.computeSubtreeInfo(n, path, owner) info, err := a.computeSubtreeInfo(n, path, owner)
if err != nil {
// computeSubtreeInfo failed (e.g. unresolvable hashNode within the
// subtree). We cannot archive this node as-is, but deeper children
// may still form valid height-3 subtrees. Recurse into them.
log.Debug("computeSubtreeInfo failed, trying children", "path", common.Bytes2Hex(path), "err", err)
return a.findSubtreesInChildren(n, path, owner)
}
if info == nil { if info == nil {
return nil return nil
} }
// If this subtree has height 3, it's a candidate for archival // If this subtree has height 3, it's a candidate for archival
if info.height == 3 { if info.height == 3 {
// Capture the original subtree root hash for verification.
// The hash is available from the node that was passed in:
// - hashNode: the hash IS the node
// - fullNode/shortNode: loaded from DB, flags.hash is set
switch nn := n.(type) {
case hashNode:
info.rootHash = common.BytesToHash(nn)
case *fullNode:
if nn.flags.hash != nil {
info.rootHash = common.BytesToHash(nn.flags.hash)
}
case *shortNode:
if nn.flags.hash != nil {
info.rootHash = common.BytesToHash(nn.flags.hash)
}
}
return []*subtreeInfo{info} return []*subtreeInfo{info}
} }
// If height > 3, recurse into children to find height-3 subtrees // If height > 3, recurse into children to find height-3 subtrees
if info.height > 3 { if info.height > 3 {
return a.findSubtreesInChildren(n, path, owner)
}
// Height < 3: no archivable subtrees here
return nil
}
// findSubtreesInChildren recurses into the children of a node to find
// height-3 subtrees. Used both by the normal height > 3 path and as a
// fallback when computeSubtreeInfo fails for a node.
func (a *Archiver) findSubtreesInChildren(n node, path []byte, owner common.Hash) []*subtreeInfo {
var results []*subtreeInfo var results []*subtreeInfo
switch n := n.(type) { switch n := n.(type) {
case *fullNode: case *fullNode:
@ -191,22 +232,20 @@ func (a *Archiver) findHeight3Subtrees(n node, path []byte, owner common.Hash) [
} }
} }
return results return results
}
// Height < 3: no archivable subtrees here
return nil
} }
// computeSubtreeInfo computes height and collects leaves for a subtree. // computeSubtreeInfo computes height and collects leaves for a subtree.
// Returns nil if the node is nil or an error occurs during resolution. // Returns (nil, nil) if the node is nil, already expired, or has no leaves.
func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *subtreeInfo { // Returns (nil, error) if any constituent node could not be resolved — the
// caller MUST NOT archive a subtree when an error is returned, as the leaf
// set would be incomplete.
func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) (*subtreeInfo, error) {
switch n := n.(type) { switch n := n.(type) {
case nil: case nil:
return nil return nil, nil
case valueNode: case valueNode:
// Leaf: height 0 // Leaf: height 0
// Encode the leaf as a shortNode for archive storage
return &subtreeInfo{ return &subtreeInfo{
path: copyBytes(path), path: copyBytes(path),
owner: owner, owner: owner,
@ -216,13 +255,16 @@ func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *s
Value: []byte(n), Value: []byte(n),
}}, }},
nodePaths: [][]byte{copyBytes(path)}, nodePaths: [][]byte{copyBytes(path)},
} }, nil
case *shortNode: case *shortNode:
childPath := append(append([]byte{}, path...), n.Key...) childPath := append(append([]byte{}, path...), n.Key...)
childInfo := a.computeSubtreeInfo(n.Val, childPath, owner) childInfo, err := a.computeSubtreeInfo(n.Val, childPath, owner)
if err != nil {
return nil, fmt.Errorf("shortNode key=%x: %w", n.Key, err)
}
if childInfo == nil { if childInfo == nil {
return nil return nil, nil
} }
// Adjust relative paths in leaves to include this node's key // Adjust relative paths in leaves to include this node's key
@ -236,7 +278,7 @@ func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *s
height: childInfo.height + 1, height: childInfo.height + 1,
leaves: childInfo.leaves, leaves: childInfo.leaves,
nodePaths: append([][]byte{copyBytes(path)}, childInfo.nodePaths...), nodePaths: append([][]byte{copyBytes(path)}, childInfo.nodePaths...),
} }, nil
case *fullNode: case *fullNode:
var ( var (
@ -247,7 +289,10 @@ func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *s
for i, child := range n.Children[:16] { for i, child := range n.Children[:16] {
if child != nil { if child != nil {
childPath := append(append([]byte{}, path...), byte(i)) childPath := append(append([]byte{}, path...), byte(i))
childInfo := a.computeSubtreeInfo(child, childPath, owner) childInfo, err := a.computeSubtreeInfo(child, childPath, owner)
if err != nil {
return nil, fmt.Errorf("fullNode child[%x]: %w", i, err)
}
if childInfo != nil { if childInfo != nil {
if childInfo.height+1 > maxHeight { if childInfo.height+1 > maxHeight {
maxHeight = childInfo.height + 1 maxHeight = childInfo.height + 1
@ -263,7 +308,7 @@ func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *s
} }
if len(allLeaves) == 0 { if len(allLeaves) == 0 {
return nil return nil, nil
} }
return &subtreeInfo{ return &subtreeInfo{
@ -272,21 +317,20 @@ func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *s
height: maxHeight, height: maxHeight,
leaves: allLeaves, leaves: allLeaves,
nodePaths: allPaths, nodePaths: allPaths,
} }, nil
case hashNode: case hashNode:
resolved, err := a.resolveNode(n, path, owner) resolved, err := a.resolveNode(n, path, owner)
if err != nil { if err != nil {
log.Debug("Failed to resolve hashNode", "path", common.Bytes2Hex(path), "err", err) return nil, fmt.Errorf("failed to resolve hashNode at path %s: %w", common.Bytes2Hex(path), err)
return nil
} }
return a.computeSubtreeInfo(resolved, path, owner) return a.computeSubtreeInfo(resolved, path, owner)
case *expiredNode: case *expiredNode:
// Already archived, skip // Already archived, skip
return nil return nil, nil
} }
return nil return nil, nil
} }
// archiveSubtree writes leaves to archive and replaces subtree with expiredNode. // archiveSubtree writes leaves to archive and replaces subtree with expiredNode.
@ -312,7 +356,25 @@ func (a *Archiver) archiveSubtree(info *subtreeInfo) error {
return fmt.Errorf("failed to sync archive: %w", err) return fmt.Errorf("failed to sync archive: %w", err)
} }
// 3. Batch database operations // 3. Verify archive round-trip: reconstruct trie from records and
// check that the hash matches the original subtree root. This
// catches any data corruption before we delete the original nodes.
if info.rootHash != (common.Hash{}) {
reconstructed, err := archiveRecordsToNode(info.leaves)
if err != nil {
return fmt.Errorf("archive verification failed: cannot reconstruct trie from records: %w", err)
}
h := newHasher(false)
gotHash := common.BytesToHash(h.hash(reconstructed, true))
returnHasherToPool(h)
if gotHash != info.rootHash {
return fmt.Errorf("archive verification failed: hash mismatch at path %s owner %s: got %s want %s (leaves=%d offset=%d size=%d)",
common.Bytes2Hex(info.path), info.owner, gotHash, info.rootHash,
len(info.leaves), offset, size)
}
}
// 4. Batch database operations
batch := a.db.NewBatch() batch := a.db.NewBatch()
// Delete all nodes in subtree (except the root which we'll overwrite) // Delete all nodes in subtree (except the root which we'll overwrite)

View file

@ -79,6 +79,8 @@ func (c *committer) commit(path []byte, n node, parallel bool) node {
return cn return cn
case hashNode: case hashNode:
return cn return cn
case *expiredNode:
return cn
default: default:
// nil, valuenode shouldn't be committed // nil, valuenode shouldn't be committed
panic(fmt.Sprintf("%T: invalid node: %v", n, n)) panic(fmt.Sprintf("%T: invalid node: %v", n, n))

View file

@ -17,6 +17,7 @@
package trie package trie
import ( import (
"bytes"
"encoding/binary" "encoding/binary"
"fmt" "fmt"
@ -33,11 +34,12 @@ const expiredNodeMarker = 0x00
type expiredNode struct { type expiredNode struct {
offset uint64 offset uint64
size uint64 size uint64
cachedHash hashNode
archiveResolver archive.ResolverFn archiveResolver archive.ResolverFn
} }
func (n *expiredNode) cache() (hashNode, bool) { func (n *expiredNode) cache() (hashNode, bool) {
return nil, true return n.cachedHash, n.cachedHash == nil
} }
func (n *expiredNode) encode(w rlp.EncoderBuffer) { func (n *expiredNode) encode(w rlp.EncoderBuffer) {
@ -62,36 +64,101 @@ func (n *expiredNode) SetArchiveResolver(resolver archive.ResolverFn) {
n.archiveResolver = resolver n.archiveResolver = resolver
} }
// resolveExpiredNodeData resolves an expired node from the archive, verifies
// the reconstructed subtree hash, and stamps the cached hash onto the root.
// Returns an error if the archive data is corrupted (hash mismatch).
func resolveExpiredNodeData(n *expiredNode) (node, error) {
records, err := archive.ArchivedNodeResolver(n.offset, n.size)
if err != nil {
return nil, fmt.Errorf("failed to resolve expired node: %w", err)
}
resolved, err := archiveRecordsToNode(records)
if err != nil {
return nil, fmt.Errorf("failed to rebuild expired node from archive: %w", err)
}
// Verify hash integrity: if the original hash is known, check that the
// reconstructed subtree produces the same hash. A mismatch means the
// archive is corrupted (e.g. missing leaves due to unresolvable hashNodes
// during archival) and any data from it is unreliable.
if n.cachedHash != nil {
h := newHasher(false)
gotHash := h.hash(resolved, true)
returnHasherToPool(h)
if !bytes.Equal(gotHash, n.cachedHash) {
return nil, fmt.Errorf("expired node hash mismatch at offset=%d size=%d: archive data is corrupted (expected %x got %x, %d records)",
n.offset, n.size, []byte(n.cachedHash), gotHash, len(records))
}
// Stamp the original hash onto the resolved subtree root so the
// hasher returns it directly instead of re-computing.
switch nn := resolved.(type) {
case *fullNode:
nn.flags.hash = n.cachedHash
case *shortNode:
nn.flags.hash = n.cachedHash
}
}
// Mark the entire resolved subtree as dirty. This is critical for
// correctness with pathdb's diff layer model: when a trie with expired
// nodes is modified and committed, the committer only captures dirty
// nodes into the NodeSet (which becomes the diff layer). Without this
// marking, resolved-but-unmodified sibling nodes within the subtree
// would exist nowhere — not in any diff layer (they're clean) and not
// in the raw DB (the archiver deleted them). Subsequent trie accesses
// from higher diff layers would fall through to the disk layer, find
// nothing, and produce MissingNodeError.
//
// For read-only tries (only get operations, no commit), this dirty
// marking is harmless — the nodes are discarded when the trie is GC'd.
markSubtreeDirty(resolved)
return resolved, nil
}
// markSubtreeDirty recursively marks all fullNode and shortNode in the
// subtree as dirty, preserving any cached hashes. This ensures the
// committer will capture them in the NodeSet during trie commit.
func markSubtreeDirty(n node) {
switch n := n.(type) {
case *fullNode:
n.flags.dirty = true
for _, child := range n.Children[:16] {
if child != nil {
markSubtreeDirty(child)
}
}
case *shortNode:
n.flags.dirty = true
markSubtreeDirty(n.Val)
}
// valueNode, hashNode, nil: no flags to mark
}
func archiveRecordsToNode(records []*archive.Record) (node, error) { func archiveRecordsToNode(records []*archive.Record) (node, error) {
if len(records) == 0 { if len(records) == 0 {
return nil, archive.EmptyArchiveRecord return nil, archive.EmptyArchiveRecord
} }
if len(records) == 1 {
return buildLeafFromRecord(records[0])
}
var newnode fullNode // Build the trie incrementally from nil to produce the canonical
// MPT structure. Starting with a fullNode would be wrong when the
// original subtree root was a shortNode (shared prefix).
var root node
for i, record := range records { for i, record := range records {
if err := validateRecordPath(record.Path); err != nil { if err := validateRecordPath(record.Path); err != nil {
return nil, err return nil, err
} }
// we are not in the case of a single leaf node, so each
// path should be at least 2 nibbles (terminator included)
if len(record.Path) < 2 || !hasTerm(record.Path) {
return nil, fmt.Errorf("invalid record path for non-leaf node #%d: %v", i, record.Path)
}
key, err := normalizeRecordKey(record.Path) key, err := normalizeRecordKey(record.Path)
if err != nil { if err != nil {
return nil, err return nil, err
} }
child, err := insertTrieNode(newnode.Children[key[0]], key[1:], valueNode(record.Value)) if len(key) < 1 {
return nil, fmt.Errorf("empty key in record #%d", i)
}
root, err = insertTrieNode(root, key, valueNode(record.Value))
if err != nil { if err != nil {
return nil, err return nil, err
} }
newnode.Children[key[0]] = child
} }
return &newnode, nil return root, nil
} }
func validateRecordPath(path []byte) error { func validateRecordPath(path []byte) error {
@ -106,14 +173,6 @@ func validateRecordPath(path []byte) error {
return nil return nil
} }
func buildLeafFromRecord(record *archive.Record) (node, error) {
key, err := normalizeRecordKey(record.Path)
if err != nil {
return nil, err
}
return &shortNode{Key: key, Val: valueNode(record.Value)}, nil
}
// normalizeRecordKey ensures the record path is a hex-nibble key suitable for // normalizeRecordKey ensures the record path is a hex-nibble key suitable for
// leaf insertion by guaranteeing a single terminator nibble and preserving any // leaf insertion by guaranteeing a single terminator nibble and preserving any
// already-terminated path. Empty paths are normalized to a sole terminator. // already-terminated path. Empty paths are normalized to a sole terminator.

View file

@ -19,12 +19,45 @@ package trie
import ( import (
"bytes" "bytes"
"errors" "errors"
"os"
"path/filepath"
"testing" "testing"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie/archive" "github.com/ethereum/go-ethereum/trie/archive"
) )
// setupTestArchive creates a temporary archive directory with an archive file
// containing the given records, and configures archive.ArchiveDataDir to point
// to it. It returns the offset and size of the written data, and a cleanup function.
func setupTestArchive(t *testing.T, records []*archive.Record) (offset, size uint64, cleanup func()) {
t.Helper()
tmpDir := t.TempDir()
gethDir := filepath.Join(tmpDir, "geth")
if err := os.MkdirAll(gethDir, 0755); err != nil {
t.Fatal(err)
}
writer, err := archive.NewArchiveWriter(filepath.Join(gethDir, "nodearchive"))
if err != nil {
t.Fatal(err)
}
offset, size, err = writer.WriteSubtree(records)
if err != nil {
writer.Close()
t.Fatal(err)
}
writer.Close()
oldDir := archive.ArchiveDataDir
archive.ArchiveDataDir = tmpDir
return offset, size, func() {
archive.ArchiveDataDir = oldDir
}
}
func TestExpiredNodeEncodeDecode(t *testing.T) { func TestExpiredNodeEncodeDecode(t *testing.T) {
testCases := []struct { testCases := []struct {
offset uint64 offset uint64
@ -120,34 +153,36 @@ func TestExpiredNodeInvalidLength(t *testing.T) {
} }
} }
func TestExpiredNodeNoResolver(t *testing.T) { func TestExpiredNodeNoArchiveFile(t *testing.T) {
// When no archive file exists, Get should return an error
tmpDir := t.TempDir()
gethDir := filepath.Join(tmpDir, "geth")
if err := os.MkdirAll(gethDir, 0755); err != nil {
t.Fatal(err)
}
oldDir := archive.ArchiveDataDir
archive.ArchiveDataDir = tmpDir
defer func() { archive.ArchiveDataDir = oldDir }()
tr := NewEmpty(nil) tr := NewEmpty(nil)
tr.root = &expiredNode{offset: 100} tr.root = &expiredNode{offset: 100, size: 50}
_, err := tr.Get([]byte("key")) _, err := tr.Get([]byte("key"))
if !errors.Is(err, archive.ErrNoResolver) { if err == nil {
t.Errorf("expected archive.ErrNoResolver, got %v", err) t.Error("expected error when archive file doesn't exist")
} }
} }
func TestExpiredNodeWithResolver(t *testing.T) { func TestExpiredNodeWithResolver(t *testing.T) {
records := []*archive.Record{
{Path: []byte{0x01, 0x02, 16}, Value: []byte("testvalue")},
}
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil) tr := NewEmpty(nil)
tr.root = &expiredNode{offset: offset, size: size}
leafNode := &shortNode{
Key: hexToCompact(keybytesToHex([]byte{0x12})),
Val: valueNode([]byte("testvalue")),
}
encodedLeaf := nodeToBytes(leafNode)
resolver := func(offset, size uint64) ([]*archive.Record, error) {
if offset == 100 {
return []*archive.Record{{Value: encodedLeaf}}, nil
}
return nil, errors.New("unknown offset")
}
tr.SetArchiveResolver(resolver)
tr.root = &expiredNode{offset: 100, size: uint64(len(encodedLeaf)), archiveResolver: resolver}
val, err := tr.Get([]byte{0x12}) val, err := tr.Get([]byte{0x12})
if err != nil { if err != nil {
@ -159,14 +194,10 @@ func TestExpiredNodeWithResolver(t *testing.T) {
} }
func TestExpiredNodeCopy(t *testing.T) { func TestExpiredNodeCopy(t *testing.T) {
resolver := func(offset, size uint64) ([]*archive.Record, error) {
return nil, nil
}
original := &expiredNode{ original := &expiredNode{
offset: 12345, offset: 12345,
size: 6789, size: 6789,
archiveResolver: resolver, archiveResolver: archive.ArchivedNodeResolver,
} }
copied := copyNode(original) copied := copyNode(original)
@ -201,18 +232,9 @@ func TestArchiveRecordsToNodeEmpty(t *testing.T) {
} }
func TestArchiveRecordsToNodeMultiple(t *testing.T) { func TestArchiveRecordsToNodeMultiple(t *testing.T) {
leaf1 := &shortNode{
Key: hexToCompact(keybytesToHex([]byte{0x10})),
Val: valueNode([]byte("value1")),
}
leaf2 := &shortNode{
Key: hexToCompact(keybytesToHex([]byte{0x20})),
Val: valueNode([]byte("value2")),
}
records := []*archive.Record{ records := []*archive.Record{
{Path: []byte{0x01}, Value: nodeToBytes(leaf1)}, {Path: []byte{0x01, 16}, Value: []byte("value1")},
{Path: []byte{0x02}, Value: nodeToBytes(leaf2)}, {Path: []byte{0x02, 16}, Value: []byte("value2")},
} }
node, err := archiveRecordsToNode(records) node, err := archiveRecordsToNode(records)
@ -234,27 +256,17 @@ func TestArchiveRecordsToNodeMultiple(t *testing.T) {
} }
func TestExpiredNodeGetMultipleRecords(t *testing.T) { func TestExpiredNodeGetMultipleRecords(t *testing.T) {
leaf1 := &shortNode{ records := []*archive.Record{
Key: hexToCompact([]byte{0x02, 0x03, 0x04, 16}), {Path: []byte{0x01, 0x02, 16}, Value: []byte("value1")},
Val: valueNode([]byte("value1")), {Path: []byte{0x04, 0x05, 16}, Value: []byte("value2")},
}
leaf2 := &shortNode{
Key: hexToCompact([]byte{0x05, 0x06, 0x07, 16}),
Val: valueNode([]byte("value2")),
}
resolver := func(offset, size uint64) ([]*archive.Record, error) {
return []*archive.Record{
{Path: []byte{0x01}, Value: nodeToBytes(leaf1)},
{Path: []byte{0x04}, Value: nodeToBytes(leaf2)},
}, nil
} }
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil) tr := NewEmpty(nil)
tr.SetArchiveResolver(resolver) tr.root = &expiredNode{offset: offset, size: size}
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
val, err := tr.Get([]byte{0x12, 0x34}) val, err := tr.Get([]byte{0x12})
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@ -263,10 +275,9 @@ func TestExpiredNodeGetMultipleRecords(t *testing.T) {
} }
tr2 := NewEmpty(nil) tr2 := NewEmpty(nil)
tr2.SetArchiveResolver(resolver) tr2.root = &expiredNode{offset: offset, size: size}
tr2.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
val2, err := tr2.Get([]byte{0x45, 0x67}) val2, err := tr2.Get([]byte{0x45})
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
@ -276,20 +287,14 @@ func TestExpiredNodeGetMultipleRecords(t *testing.T) {
} }
func TestExpiredNodeGetKeyNotFound(t *testing.T) { func TestExpiredNodeGetKeyNotFound(t *testing.T) {
leaf := &shortNode{ records := []*archive.Record{
Key: hexToCompact(keybytesToHex([]byte{0x12})), {Path: []byte{0x01, 0x02, 16}, Value: []byte("value1")},
Val: valueNode([]byte("value1")),
}
resolver := func(offset, size uint64) ([]*archive.Record, error) {
return []*archive.Record{
{Path: []byte{0x01}, Value: nodeToBytes(leaf)},
}, nil
} }
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil) tr := NewEmpty(nil)
tr.SetArchiveResolver(resolver) tr.root = &expiredNode{offset: offset, size: size}
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
val, err := tr.Get([]byte{0xff, 0xff}) val, err := tr.Get([]byte{0xff, 0xff})
if err != nil { if err != nil {
@ -301,20 +306,14 @@ func TestExpiredNodeGetKeyNotFound(t *testing.T) {
} }
func TestExpiredNodeGetPathMismatch(t *testing.T) { func TestExpiredNodeGetPathMismatch(t *testing.T) {
leaf := &shortNode{ records := []*archive.Record{
Key: hexToCompact(keybytesToHex([]byte{0x12})), {Path: []byte{0x01, 0x02, 16}, Value: []byte("testvalue")},
Val: valueNode([]byte("testvalue")),
}
resolver := func(offset, size uint64) ([]*archive.Record, error) {
return []*archive.Record{
{Path: []byte{0x01}, Value: nodeToBytes(leaf)},
}, nil
} }
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil) tr := NewEmpty(nil)
tr.SetArchiveResolver(resolver) tr.root = &expiredNode{offset: offset, size: size}
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
val, err := tr.Get([]byte{0x19}) val, err := tr.Get([]byte{0x19})
if err != nil { if err != nil {
@ -326,20 +325,14 @@ func TestExpiredNodeGetPathMismatch(t *testing.T) {
} }
func TestExpiredNodeInsert(t *testing.T) { func TestExpiredNodeInsert(t *testing.T) {
leaf := &shortNode{ records := []*archive.Record{
Key: hexToCompact(keybytesToHex([]byte{0x12})), {Path: []byte{0x01, 0x02, 16}, Value: []byte("existing")},
Val: valueNode([]byte("existing")),
}
resolver := func(offset, size uint64) ([]*archive.Record, error) {
return []*archive.Record{
{Path: []byte{}, Value: nodeToBytes(leaf)},
}, nil
} }
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil) tr := NewEmpty(nil)
tr.SetArchiveResolver(resolver) tr.root = &expiredNode{offset: offset, size: size}
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
err := tr.Update([]byte{0x45}, []byte("newvalue")) err := tr.Update([]byte{0x45}, []byte("newvalue"))
if err != nil { if err != nil {
@ -356,20 +349,14 @@ func TestExpiredNodeInsert(t *testing.T) {
} }
func TestExpiredNodeUpdate(t *testing.T) { func TestExpiredNodeUpdate(t *testing.T) {
leaf := &shortNode{ records := []*archive.Record{
Key: hexToCompact(keybytesToHex([]byte{0x12})), {Path: []byte{0x01, 0x02, 16}, Value: []byte("oldvalue")},
Val: valueNode([]byte("oldvalue")),
}
resolver := func(offset, size uint64) ([]*archive.Record, error) {
return []*archive.Record{
{Path: []byte{}, Value: nodeToBytes(leaf)},
}, nil
} }
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil) tr := NewEmpty(nil)
tr.SetArchiveResolver(resolver) tr.root = &expiredNode{offset: offset, size: size}
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
err := tr.Update([]byte{0x12}, []byte("newvalue")) err := tr.Update([]byte{0x12}, []byte("newvalue"))
if err != nil { if err != nil {
@ -386,28 +373,15 @@ func TestExpiredNodeUpdate(t *testing.T) {
} }
func TestExpiredNodeDelete(t *testing.T) { func TestExpiredNodeDelete(t *testing.T) {
leaf1 := &shortNode{ records := []*archive.Record{
Key: hexToCompact([]byte{0x02, 16}), {Path: []byte{0x01, 0x02, 16}, Value: []byte("value1")},
Val: valueNode([]byte("value1")), {Path: []byte{0x04, 0x05, 16}, Value: []byte("value2")},
}
leaf2 := &shortNode{
Key: hexToCompact([]byte{0x05, 16}),
Val: valueNode([]byte("value2")),
}
branch := &fullNode{}
branch.Children[0x01] = leaf1
branch.Children[0x04] = leaf2
resolver := func(offset, size uint64) ([]*archive.Record, error) {
return []*archive.Record{
{Path: []byte{}, Value: nodeToBytes(branch)},
}, nil
} }
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil) tr := NewEmpty(nil)
tr.SetArchiveResolver(resolver) tr.root = &expiredNode{offset: offset, size: size}
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
err := tr.Delete([]byte{0x12}) err := tr.Delete([]byte{0x12})
if err != nil { if err != nil {
@ -432,22 +406,14 @@ func TestExpiredNodeDelete(t *testing.T) {
} }
func TestTrieCopyPreservesArchiveResolver(t *testing.T) { func TestTrieCopyPreservesArchiveResolver(t *testing.T) {
leaf := &shortNode{ records := []*archive.Record{
Key: hexToCompact(keybytesToHex([]byte{0x12})), {Path: []byte{0x01, 0x02, 16}, Value: []byte("testvalue")},
Val: valueNode([]byte("testvalue")),
}
resolverCalled := false
resolver := func(offset, size uint64) ([]*archive.Record, error) {
resolverCalled = true
return []*archive.Record{
{Path: []byte{}, Value: nodeToBytes(leaf)},
}, nil
} }
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil) tr := NewEmpty(nil)
tr.SetArchiveResolver(resolver) tr.root = &expiredNode{offset: offset, size: size}
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
trCopy := tr.Copy() trCopy := tr.Copy()
@ -455,36 +421,180 @@ func TestTrieCopyPreservesArchiveResolver(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }
if !resolverCalled {
t.Error("resolver was not called on copied trie")
}
if string(val) != "testvalue" { if string(val) != "testvalue" {
t.Errorf("value mismatch: got %q, want %q", val, "testvalue") t.Errorf("value mismatch: got %q, want %q", val, "testvalue")
} }
} }
func TestExpiredNodeGetNode(t *testing.T) { func TestWalkWithExpiredNodes(t *testing.T) {
leaf := &shortNode{ records := []*archive.Record{
Key: hexToCompact(keybytesToHex([]byte{0x12})), {Path: []byte{0x01, 0x02, 16}, Value: []byte("value1")},
Val: valueNode([]byte("testvalue")), {Path: []byte{0x04, 0x05, 16}, Value: []byte("value2")},
} {Path: []byte{0x07, 0x08, 16}, Value: []byte("value3")},
resolverCalled := false
resolver := func(offset, size uint64) ([]*archive.Record, error) {
resolverCalled = true
return []*archive.Record{
{Path: []byte{}, Value: nodeToBytes(leaf)},
}, nil
} }
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil) tr := NewEmpty(nil)
tr.SetArchiveResolver(resolver) tr.root = &expiredNode{offset: offset, size: size}
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
var leaves []string
stats, err := tr.Walk(func(path []byte, value []byte) error {
leaves = append(leaves, string(value))
return nil
})
if err != nil {
t.Fatalf("Walk failed: %v", err)
}
if stats.Leaves != 3 {
t.Errorf("expected 3 leaves, got %d", stats.Leaves)
}
if stats.ExpiredResolved != 1 {
t.Errorf("expected 1 expired resolved, got %d", stats.ExpiredResolved)
}
// Verify all values were visited
expected := map[string]bool{"value1": true, "value2": true, "value3": true}
for _, leaf := range leaves {
if !expected[leaf] {
t.Errorf("unexpected leaf value: %q", leaf)
}
delete(expected, leaf)
}
if len(expected) > 0 {
t.Errorf("missing leaves: %v", expected)
}
}
func TestWalkEmptyTrie(t *testing.T) {
tr := NewEmpty(nil)
stats, err := tr.Walk(func(path []byte, value []byte) error {
t.Error("callback should not be called for empty trie")
return nil
})
if err != nil {
t.Fatalf("Walk failed: %v", err)
}
if stats.Leaves != 0 || stats.ExpiredResolved != 0 {
t.Errorf("expected zero stats for empty trie, got leaves=%d expired=%d", stats.Leaves, stats.ExpiredResolved)
}
}
func TestWalkCallbackError(t *testing.T) {
records := []*archive.Record{
{Path: []byte{0x01, 0x02, 16}, Value: []byte("value1")},
}
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil)
tr.root = &expiredNode{offset: offset, size: size}
testErr := errors.New("test error")
_, err := tr.Walk(func(path []byte, value []byte) error {
return testErr
})
if !errors.Is(err, testErr) {
t.Fatalf("expected test error, got %v", err)
}
}
// TestExpiredNodeResolvedSubtreeDirty verifies that when an expired node is
// resolved and a sibling leaf is modified, the commit captures ALL resolved
// nodes (not just the modified path). Without this fix, resolved-but-unmodified
// nodes would be lost: not in the diff layer (clean) and not in the raw DB
// (deleted by archiver).
func TestExpiredNodeResolvedSubtreeDirty(t *testing.T) {
// Use large values (>32 bytes) so leaf nodes are NOT embedded in
// their parent. This matches production storage tries where
// intermediate nodes are large enough to be stored independently.
bigVal1 := bytes.Repeat([]byte("A"), 40)
bigVal2 := bytes.Repeat([]byte("B"), 40)
// Create an archive with records under different branches.
records := []*archive.Record{
{Path: []byte{0x01, 0x02, 16}, Value: bigVal1},
{Path: []byte{0x04, 0x05, 16}, Value: bigVal2},
}
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil)
tr.root = &expiredNode{offset: offset, size: size}
// Insert a value that goes through one branch of the resolved subtree.
// This modifies path [1, ...] but leaves path [4, ...] unmodified.
if err := tr.Update([]byte{0x12}, bytes.Repeat([]byte("C"), 40)); err != nil {
t.Fatalf("Update failed: %v", err)
}
// Commit the trie. The NodeSet should be non-nil because we modified data.
_, nodes := tr.Commit(false)
if nodes == nil {
t.Fatal("expected non-nil NodeSet after modifying expired subtree")
}
// The resolved-but-unmodified sibling (path [4, 5]) should also be
// captured in the NodeSet, because markSubtreeDirty ensures all resolved
// nodes are dirty. Count the nodes to verify.
nodeCount := len(nodes.Nodes)
// We expect at least 3 nodes: the root, the modified branch, and the
// sibling branch. The exact count depends on trie structure.
if nodeCount < 3 {
t.Errorf("expected at least 3 nodes in NodeSet (root + modified + sibling), got %d", nodeCount)
}
}
// TestMarkSubtreeDirty verifies that markSubtreeDirty correctly sets the dirty
// flag on all nodes in a subtree while preserving cached hashes.
func TestMarkSubtreeDirty(t *testing.T) {
// Build a small trie structure
leaf1 := &shortNode{Key: []byte{1, 16}, Val: valueNode("v1")}
leaf2 := &shortNode{Key: []byte{2, 16}, Val: valueNode("v2")}
branch := &fullNode{}
branch.Children[1] = leaf1
branch.Children[2] = leaf2
// Set hash but not dirty (as if loaded from DB)
branch.flags = nodeFlag{hash: hashNode("testhash"), dirty: false}
leaf1.flags = nodeFlag{hash: hashNode("hash1"), dirty: false}
leaf2.flags = nodeFlag{hash: hashNode("hash2"), dirty: false}
markSubtreeDirty(branch)
// All nodes should be dirty
if !branch.flags.dirty {
t.Error("branch should be dirty")
}
if !leaf1.flags.dirty {
t.Error("leaf1 should be dirty")
}
if !leaf2.flags.dirty {
t.Error("leaf2 should be dirty")
}
// Hashes should be preserved
if !bytes.Equal(branch.flags.hash, hashNode("testhash")) {
t.Error("branch hash should be preserved")
}
if !bytes.Equal(leaf1.flags.hash, hashNode("hash1")) {
t.Error("leaf1 hash should be preserved")
}
if !bytes.Equal(leaf2.flags.hash, hashNode("hash2")) {
t.Error("leaf2 hash should be preserved")
}
}
func TestExpiredNodeGetNode(t *testing.T) {
records := []*archive.Record{
{Path: []byte{0x01, 0x02, 16}, Value: []byte("testvalue")},
}
offset, size, cleanup := setupTestArchive(t, records)
defer cleanup()
tr := NewEmpty(nil)
tr.root = &expiredNode{offset: offset, size: size}
_, _, err := tr.GetNode(hexToCompact([]byte{0x01, 0x02})) _, _, err := tr.GetNode(hexToCompact([]byte{0x01, 0x02}))
if !resolverCalled {
t.Error("resolver was not called during GetNode")
}
if err != nil && err.Error() != "non-consensus node" { if err != nil && err.Error() != "non-consensus node" {
t.Fatalf("unexpected error: %v", err) t.Fatalf("unexpected error: %v", err)
} }

View file

@ -18,6 +18,7 @@ package trie
import ( import (
"bytes" "bytes"
"encoding/binary"
"fmt" "fmt"
"sync" "sync"
@ -97,6 +98,22 @@ func (h *hasher) hash(n node, force bool) []byte {
// hash nodes don't have children, so they're left as were // hash nodes don't have children, so they're left as were
return n return n
case *expiredNode:
// Return the original subtree hash that was cached when the
// expired node was decoded. The parent node references this
// hash, so we must return the same value to keep the Merkle
// root consistent.
if n.cachedHash != nil {
return n.cachedHash
}
// Fallback: hash the marker blob (should not happen in practice
// because decodeNodeUnsafe always provides the hash).
var buf [1 + 2*8]byte // 17 bytes
buf[0] = expiredNodeMarker
binary.BigEndian.PutUint64(buf[1:], n.offset)
binary.BigEndian.PutUint64(buf[9:], n.size)
return h.hashData(buf[:])
default: default:
panic(fmt.Errorf("unexpected node type, %T", n)) panic(fmt.Errorf("unexpected node type, %T", n))
} }
@ -214,6 +231,12 @@ func (h *hasher) proofHash(original node) []byte {
return bytes.Clone(h.encodeShortNode(n)) return bytes.Clone(h.encodeShortNode(n))
case *fullNode: case *fullNode:
return bytes.Clone(h.encodeFullNode(n)) return bytes.Clone(h.encodeFullNode(n))
case *expiredNode:
var buf [1 + 2*8]byte
buf[0] = expiredNodeMarker
binary.BigEndian.PutUint64(buf[1:], n.offset)
binary.BigEndian.PutUint64(buf[9:], n.size)
return buf[:]
default: default:
panic(fmt.Errorf("unexpected node type, %T", original)) panic(fmt.Errorf("unexpected node type, %T", original))
} }

View file

@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie/archive" "github.com/ethereum/go-ethereum/trie/archive"
) )
@ -166,7 +167,8 @@ func decodeNodeUnsafe(hash, buf []byte) (node, error) {
} }
offset := binary.BigEndian.Uint64(buf[1:]) offset := binary.BigEndian.Uint64(buf[1:])
size := binary.BigEndian.Uint64(buf[1+archive.OffsetSize:]) size := binary.BigEndian.Uint64(buf[1+archive.OffsetSize:])
return &expiredNode{offset: offset, size: size, archiveResolver: archive.ArchivedNodeResolver}, nil log.Debug("Decoded expired node", "offset", offset, "size", size, "hash", common.BytesToHash(hash))
return &expiredNode{offset: offset, size: size, cachedHash: hashNode(hash), archiveResolver: archive.ArchivedNodeResolver}, nil
} }
elems, _, err := rlp.SplitList(buf) elems, _, err := rlp.SplitList(buf)
if err != nil { if err != nil {

View file

@ -25,6 +25,7 @@ import (
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie/archive"
) )
// Prove constructs a merkle proof for key. The result contains all encoded nodes // Prove constructs a merkle proof for key. The result contains all encoded nodes
@ -78,6 +79,16 @@ func (t *Trie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error {
// clean cache or the database, they are all in their own // clean cache or the database, they are all in their own
// copy and safe to use unsafe decoder. // copy and safe to use unsafe decoder.
tn = mustDecodeNodeUnsafe(n, blob) tn = mustDecodeNodeUnsafe(n, blob)
case *expiredNode:
records, err := archive.ArchivedNodeResolver(n.offset, n.size)
if err != nil {
return fmt.Errorf("failed to resolve expired node in proof: %w", err)
}
resolved, err := archiveRecordsToNode(records)
if err != nil {
return fmt.Errorf("failed to rebuild expired node in proof: %w", err)
}
tn = resolved
default: default:
panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
} }
@ -617,6 +628,8 @@ func get(tn node, key []byte, skipResolved bool) ([]byte, node) {
} }
case hashNode: case hashNode:
return key, n return key, n
case *expiredNode:
return key, n
case nil: case nil:
return key, nil return key, nil
case valueNode: case valueNode:

View file

@ -230,32 +230,12 @@ func (t *Trie) get(origNode node, key []byte, pos int) (value []byte, newnode no
value, newnode, _, err := t.get(child, key, pos) value, newnode, _, err := t.get(child, key, pos)
return value, newnode, true, err return value, newnode, true, err
case *expiredNode: case *expiredNode:
records, err := archive.ArchivedNodeResolver(n.offset, n.size) log.Debug("Resolving expired node in get()", "owner", t.owner, "offset", n.offset, "size", n.size, "pos", pos)
if err != nil { newnode, err := resolveExpiredNodeData(n)
return nil, n, false, fmt.Errorf("failed to resolve expired node: %w", err)
}
newnode, err := archiveRecordsToNode(records)
// alternative: don't rebuild, just find the value
// for _, record := range records {
// // make sure that the path up to the node matches
// if bytes.HasPrefix(key[pos:], record.Path) {
// resolved, err := decodeNodeUnsafe(nil, record.Value)
// if err != nil {
// fmt.Printf("%v %x\n", record.Path, record.Value)
// return nil, n, false, fmt.Errorf("failed to deserialize RLP node: %w", err)
// }
// if leaf, ok := resolved.(*shortNode); ok {
// // make sure that the key to the leaf also matches
// if bytes.Equal(key[pos+len(record.Path):], leaf.Key) {
// return leaf.Val.(valueNode), newnode, true, nil
// }
// }
// }
// }
if err != nil { if err != nil {
return nil, n, false, err return nil, n, false, err
} }
value, _, _, err = t.get(newnode, key, pos+1) value, _, _, err = t.get(newnode, key, pos)
return value, newnode, true, err return value, newnode, true, err
default: default:
panic(fmt.Sprintf("%T: invalid node: %v", origNode, origNode)) panic(fmt.Sprintf("%T: invalid node: %v", origNode, origNode))
@ -392,12 +372,11 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod
return item, newnode, resolved + 1, err return item, newnode, resolved + 1, err
case *expiredNode: case *expiredNode:
records, err := archive.ArchivedNodeResolver(n.offset, n.size) rn, err := resolveExpiredNodeData(n)
if err != nil { if err != nil {
return nil, n, 0, fmt.Errorf("failed to resolve expired node: %w", err) return nil, n, 0, err
} }
newnode, err := archiveRecordsToNode(records) item, newnode, resolvedCount, err := t.getNode(rn, path, pos)
item, newnode, resolvedCount, err := t.getNode(newnode, path, pos)
return item, newnode, resolvedCount + 1, err return item, newnode, resolvedCount + 1, err
default: default:
@ -524,16 +503,16 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
return true, nn, nil return true, nn, nil
case *expiredNode: case *expiredNode:
records, err := archive.ArchivedNodeResolver(n.offset, n.size) log.Info("Resolving expired node in insert()", "owner", t.owner, "offset", n.offset, "size", n.size)
rn, err := resolveExpiredNodeData(n)
if err != nil { if err != nil {
return false, nil, fmt.Errorf("failed to resolve expired node: %w", err) return false, nil, err
} }
nn, err := archiveRecordsToNode(records) dirty, nn, err := t.insert(rn, prefix, key, value)
if err != nil { if !dirty || err != nil {
return false, nil, fmt.Errorf("failed to rebuild expired node from archive: %w", err) return false, rn, err
} }
dirty, nn, err := t.insert(nn, prefix, key, value) return true, nn, nil
return dirty && err == nil, nn, err
default: default:
panic(fmt.Sprintf("%T: invalid node: %v", n, n)) panic(fmt.Sprintf("%T: invalid node: %v", n, n))
@ -697,16 +676,16 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
return true, nn, nil return true, nn, nil
case *expiredNode: case *expiredNode:
records, err := archive.ArchivedNodeResolver(n.offset, n.size) log.Info("Resolving expired node in delete()", "owner", t.owner, "offset", n.offset, "size", n.size)
rn, err := resolveExpiredNodeData(n)
if err != nil { if err != nil {
return false, nil, fmt.Errorf("failed to resolve expired node: %w", err) return false, nil, err
} }
nn, err := archiveRecordsToNode(records) dirty, nn, err := t.delete(rn, prefix, key)
if err != nil { if !dirty || err != nil {
return false, nil, fmt.Errorf("failed to rebuild expired node from archive: %w", err) return false, rn, err
} }
dirty, _, err := t.delete(nn, prefix, key) return true, nn, nil
return dirty && err == nil, nn, err
default: default:
panic(fmt.Sprintf("%T: invalid node: %v (%v)", n, n, key)) panic(fmt.Sprintf("%T: invalid node: %v (%v)", n, n, key))
@ -742,7 +721,8 @@ func copyNode(n node) node {
return &expiredNode{ return &expiredNode{
offset: n.offset, offset: n.offset,
size: n.size, size: n.size,
archiveResolver: archive.ArchivedNodeResolver, cachedHash: common.CopyBytes(n.cachedHash),
archiveResolver: n.archiveResolver,
} }
default: default:
panic(fmt.Sprintf("%T: unknown node type", n)) panic(fmt.Sprintf("%T: unknown node type", n))
@ -750,8 +730,11 @@ func copyNode(n node) node {
} }
func (t *Trie) resolve(n node, prefix []byte) (node, error) { func (t *Trie) resolve(n node, prefix []byte) (node, error) {
if n, ok := n.(hashNode); ok { switch n := n.(type) {
case hashNode:
return t.resolveAndTrack(n, prefix) return t.resolveAndTrack(n, prefix)
case *expiredNode:
return resolveExpiredNodeData(n)
} }
return n, nil return n, nil
} }
@ -862,6 +845,58 @@ func (t *Trie) Witness() map[string][]byte {
return t.prevalueTracer.Values() return t.prevalueTracer.Values()
} }
// WalkStats holds statistics from a Walk traversal.
type WalkStats struct {
Leaves int // Number of leaf nodes visited
ExpiredResolved int // Number of expired nodes resolved from archive
}
// Walk recursively traverses the trie, resolving all nodes including
// hashNodes and expiredNodes. It calls fn for each leaf found.
// This triggers hash verification for expired nodes via cachedHash.
func (t *Trie) Walk(fn func(path []byte, value []byte) error) (WalkStats, error) {
return t.walk(t.root, nil, fn)
}
func (t *Trie) walk(n node, path []byte, fn func([]byte, []byte) error) (WalkStats, error) {
switch n := n.(type) {
case *shortNode:
return t.walk(n.Val, append(append([]byte{}, path...), n.Key...), fn)
case *fullNode:
var stats WalkStats
for i, child := range n.Children[:16] {
if child != nil {
childStats, err := t.walk(child, append(append([]byte{}, path...), byte(i)), fn)
if err != nil {
return stats, err
}
stats.Leaves += childStats.Leaves
stats.ExpiredResolved += childStats.ExpiredResolved
}
}
return stats, nil
case hashNode:
resolved, err := t.resolveAndTrack(n, path)
if err != nil {
return WalkStats{}, err
}
return t.walk(resolved, path, fn)
case *expiredNode:
resolved, err := resolveExpiredNodeData(n)
if err != nil {
return WalkStats{}, err
}
childStats, err := t.walk(resolved, path, fn)
childStats.ExpiredResolved++
return childStats, err
case valueNode:
return WalkStats{Leaves: 1}, fn(path, []byte(n))
case nil:
return WalkStats{}, nil
}
return WalkStats{}, nil
}
// reset drops the referenced root node and cleans all internal state. // reset drops the referenced root node and cleans all internal state.
func (t *Trie) reset() { func (t *Trie) reset() {
t.root = nil t.root = nil

View file

@ -399,6 +399,28 @@ func (db *Database) Disk() ethdb.Database {
return db.disk return db.disk
} }
// DiffHead returns the root hash of the topmost diff layer in pathdb.
// If there are no diff layers or the backend is not pathdb, it returns
// the zero hash and false.
func (db *Database) DiffHead() (common.Hash, bool) {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return common.Hash{}, false
}
return pdb.DiffHead()
}
// DisableStateHistory closes and disables the state history freezer.
// This is used by the archiver to bypass state history writes during
// diff layer flushing when state history may have gaps.
func (db *Database) DisableStateHistory() {
pdb, ok := db.backend.(*pathdb.Database)
if !ok {
return
}
pdb.DisableStateHistory()
}
// SnapshotCompleted returns the indicator if the snapshot is completed. // SnapshotCompleted returns the indicator if the snapshot is completed.
func (db *Database) SnapshotCompleted() bool { func (db *Database) SnapshotCompleted() bool {
pdb, ok := db.backend.(*pathdb.Database) pdb, ok := db.backend.(*pathdb.Database)

View file

@ -318,6 +318,30 @@ func (db *Database) Update(root common.Hash, parentRoot common.Hash, block uint6
return db.tree.cap(root, maxDiffLayers) return db.tree.cap(root, maxDiffLayers)
} }
// DiffHead returns the root hash of the topmost diff layer. If there are no
// diff layers (only the disk layer), it returns the disk layer root and false.
func (db *Database) DiffHead() (common.Hash, bool) {
db.lock.RLock()
defer db.lock.RUnlock()
return db.tree.diffHead()
}
// DisableStateHistory closes and disables the state history freezer. This is
// used by the archiver to bypass state history writes during diff layer flushing,
// since the archiver only needs trie nodes committed to disk and state history
// may have gaps from unclean shutdowns that prevent sequential appends.
func (db *Database) DisableStateHistory() {
db.lock.Lock()
defer db.lock.Unlock()
if db.stateFreezer != nil {
db.stateFreezer.Close()
db.stateFreezer = nil
log.Info("Disabled state history freezer")
}
}
// Commit traverses downwards the layer tree from a specified layer with the // Commit traverses downwards the layer tree from a specified layer with the
// provided state root and all the layers below are flattened downwards. It // provided state root and all the layers below are flattened downwards. It
// can be used alone and mostly for test purposes. // can be used alone and mostly for test purposes.

View file

@ -278,9 +278,17 @@ func truncateFromHead(store ethdb.AncientStore, typ historyType, nhead uint64) (
return 0, err return 0, err
} }
// Ensure that the truncation target falls within the valid range. // Ensure that the truncation target falls within the valid range.
if ohead < nhead || nhead < otail { if nhead < otail {
return 0, fmt.Errorf("%w, %s, tail: %d, head: %d, target: %d", errHeadTruncationOutOfRange, typ, otail, ohead, nhead) return 0, fmt.Errorf("%w, %s, tail: %d, head: %d, target: %d", errHeadTruncationOutOfRange, typ, otail, ohead, nhead)
} }
// If the target is ahead of the current head, there's nothing to truncate.
// This can happen after unclean shutdowns where the state history was not
// fully written.
if ohead < nhead {
log.Warn("State history shorter than target, nothing to truncate",
"type", typ.String(), "head", ohead, "target", nhead)
return 0, nil
}
// Short circuit if nothing to truncate. // Short circuit if nothing to truncate.
if ohead == nhead { if ohead == nhead {
return 0, nil return 0, nil

View file

@ -31,6 +31,7 @@ import (
// of the referenced layer by themselves. // of the referenced layer by themselves.
type layerTree struct { type layerTree struct {
base *diskLayer base *diskLayer
head common.Hash // Root hash of the topmost layer (diff or disk)
layers map[common.Hash]layer layers map[common.Hash]layer
// descendants is a two-dimensional map where the keys represent // descendants is a two-dimensional map where the keys represent
@ -59,6 +60,7 @@ func (tree *layerTree) init(head layer) {
defer tree.lock.Unlock() defer tree.lock.Unlock()
current := head current := head
tree.head = head.rootHash()
tree.layers = make(map[common.Hash]layer) tree.layers = make(map[common.Hash]layer)
tree.descendants = make(map[common.Hash]map[common.Hash]struct{}) tree.descendants = make(map[common.Hash]map[common.Hash]struct{})
@ -76,6 +78,18 @@ func (tree *layerTree) init(head layer) {
tree.lookup = newLookup(head, tree.isDescendant) tree.lookup = newLookup(head, tree.isDescendant)
} }
// diffHead returns the root hash of the topmost diff layer. If there are no
// diff layers, returns the disk layer root and false.
func (tree *layerTree) diffHead() (common.Hash, bool) {
tree.lock.RLock()
defer tree.lock.RUnlock()
if _, ok := tree.layers[tree.head].(*diffLayer); ok {
return tree.head, true
}
return tree.base.rootHash(), false
}
// get retrieves a layer belonging to the given state root. // get retrieves a layer belonging to the given state root.
func (tree *layerTree) get(root common.Hash) layer { func (tree *layerTree) get(root common.Hash) layer {
tree.lock.RLock() tree.lock.RLock()