mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-07-17 18:30:45 +00:00
trie, cmd/geth: add archive verify command, Walk(), and archiver improvements
Add Trie.Walk() for exhaustive traversal that resolves expired nodes with hash verification. Add `archive verify` subcommand that walks the full state (account + storage tries) to validate all archived data can be correctly resurrected. Delete both the journal KV entry and file after archiving to force geth to restart with a bare disk layer, rewinding the chain head to the persistent disk state and re-executing blocks. Also adds markSubtreeDirty() to resolveExpiredNodeData() so that all nodes in a resolved expired subtree are captured in the NodeSet during commit — preventing them from being lost between diff layers and the raw DB.
This commit is contained in:
parent
8d2125e4fd
commit
5119945e25
15 changed files with 1032 additions and 282 deletions
|
|
@ -17,8 +17,10 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/binary"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"slices"
|
"slices"
|
||||||
"time"
|
"time"
|
||||||
|
|
@ -26,10 +28,14 @@ import (
|
||||||
"github.com/ethereum/go-ethereum/cmd/utils"
|
"github.com/ethereum/go-ethereum/cmd/utils"
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||||
|
"github.com/ethereum/go-ethereum/core/types"
|
||||||
|
"github.com/ethereum/go-ethereum/crypto"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/log"
|
"github.com/ethereum/go-ethereum/log"
|
||||||
|
"github.com/ethereum/go-ethereum/rlp"
|
||||||
"github.com/ethereum/go-ethereum/trie"
|
"github.com/ethereum/go-ethereum/trie"
|
||||||
"github.com/ethereum/go-ethereum/trie/archive"
|
"github.com/ethereum/go-ethereum/trie/archive"
|
||||||
|
"github.com/ethereum/go-ethereum/triedb/database"
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -51,14 +57,73 @@ var (
|
||||||
}
|
}
|
||||||
|
|
||||||
// Commands
|
// Commands
|
||||||
|
archiveCheckNodeFlag = &cli.StringFlag{
|
||||||
|
Name: "owner",
|
||||||
|
Usage: "Owner hash (hex) for the trie node to check",
|
||||||
|
}
|
||||||
|
archiveCheckPathFlag = &cli.StringFlag{
|
||||||
|
Name: "path",
|
||||||
|
Usage: "Path (hex nibbles) of the trie node to check",
|
||||||
|
}
|
||||||
|
|
||||||
archiveCommand = &cli.Command{
|
archiveCommand = &cli.Command{
|
||||||
Name: "archive",
|
Name: "archive",
|
||||||
Usage: "Archive state trie nodes to reduce database size",
|
Usage: "Archive state trie nodes to reduce database size",
|
||||||
Subcommands: []*cli.Command{
|
Subcommands: []*cli.Command{
|
||||||
archiveGenerateCmd,
|
archiveGenerateCmd,
|
||||||
|
archiveVerifyCmd,
|
||||||
|
archiveDeleteJournalCmd,
|
||||||
|
archiveCheckNodeCmd,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
archiveCheckNodeCmd = &cli.Command{
|
||||||
|
Name: "check-node",
|
||||||
|
Usage: "Check if a specific trie node exists in the raw DB",
|
||||||
|
Action: archiveCheckNode,
|
||||||
|
Flags: slices.Concat([]cli.Flag{
|
||||||
|
archiveCheckNodeFlag,
|
||||||
|
archiveCheckPathFlag,
|
||||||
|
}, utils.NetworkFlags, utils.DatabaseFlags),
|
||||||
|
}
|
||||||
|
|
||||||
|
archiveDeleteJournalCmd = &cli.Command{
|
||||||
|
Name: "delete-journal",
|
||||||
|
Usage: "Delete the pathdb journal to force a clean restart",
|
||||||
|
Action: archiveDeleteJournal,
|
||||||
|
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
|
||||||
|
Description: `
|
||||||
|
Deletes the pathdb journal (TrieJournal key and merkle.journal file) from the
|
||||||
|
database. This forces geth to restart with a bare disk layer, discarding any
|
||||||
|
in-memory diff layers that may be inconsistent with archived state.
|
||||||
|
|
||||||
|
Use this after running 'archive generate' if geth was started in between and
|
||||||
|
recreated the journal.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
geth archive delete-journal --datadir /path/to/datadir
|
||||||
|
geth archive delete-journal --hoodi
|
||||||
|
`,
|
||||||
|
}
|
||||||
|
|
||||||
|
archiveVerifyCmd = &cli.Command{
|
||||||
|
Name: "verify",
|
||||||
|
Usage: "Verify all archived nodes can be correctly resurrected",
|
||||||
|
Action: archiveVerify,
|
||||||
|
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
|
||||||
|
Description: `
|
||||||
|
Walks the entire state trie, resolving every expired node from the archive
|
||||||
|
file and verifying that the reconstructed subtree hash matches the original.
|
||||||
|
Also walks all storage tries referenced by accounts.
|
||||||
|
|
||||||
|
The database is opened read-only. No modifications are made.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
geth archive verify --datadir /path/to/datadir
|
||||||
|
geth archive verify --hoodi
|
||||||
|
`,
|
||||||
|
}
|
||||||
|
|
||||||
archiveGenerateCmd = &cli.Command{
|
archiveGenerateCmd = &cli.Command{
|
||||||
Name: "generate",
|
Name: "generate",
|
||||||
Usage: "Generate archive files from height-3 subtrees",
|
Usage: "Generate archive files from height-3 subtrees",
|
||||||
|
|
@ -77,22 +142,56 @@ that references the archive file offset and size.
|
||||||
Height is measured from leaves: leaves=0, parents=1, etc. A height-3 node
|
Height is measured from leaves: leaves=0, parents=1, etc. A height-3 node
|
||||||
has leaves at most 3 levels below it.
|
has leaves at most 3 levels below it.
|
||||||
|
|
||||||
|
The archiver reads trie nodes directly from the persistent database layer,
|
||||||
|
bypassing any in-memory diff layers. This ensures consistency between the
|
||||||
|
data it reads and the data it modifies.
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
# Archive from head state
|
# Archive from the persistent disk state
|
||||||
geth archive generate --datadir /path/to/datadir
|
geth archive generate --datadir /path/to/datadir
|
||||||
|
|
||||||
# Dry run to see what would be archived
|
# Dry run to see what would be archived
|
||||||
geth archive generate --dry-run --datadir /path/to/datadir
|
geth archive generate --dry-run --datadir /path/to/datadir
|
||||||
|
|
||||||
# Archive from a specific state root
|
|
||||||
geth archive generate 0x1234...abcd --datadir /path/to/datadir
|
|
||||||
|
|
||||||
# Custom output and compaction interval
|
# Custom output and compaction interval
|
||||||
geth archive generate --output /path/to/archive --compaction-interval 500
|
geth archive generate --output /path/to/archive --compaction-interval 500
|
||||||
`,
|
`,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// rawDBNodeReader implements database.NodeReader by reading trie nodes directly
|
||||||
|
// from the raw key-value database, bypassing pathdb's in-memory diff layers.
|
||||||
|
// This ensures the archiver sees the same trie state it modifies.
|
||||||
|
type rawDBNodeReader struct {
|
||||||
|
db ethdb.KeyValueReader
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *rawDBNodeReader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) {
|
||||||
|
var blob []byte
|
||||||
|
if owner == (common.Hash{}) {
|
||||||
|
blob = rawdb.ReadAccountTrieNode(r.db, path)
|
||||||
|
} else {
|
||||||
|
blob = rawdb.ReadStorageTrieNode(r.db, owner, path)
|
||||||
|
}
|
||||||
|
// Skip hash verification: the raw DB may contain expiredNode markers
|
||||||
|
// (blob[0] == 0x00) which have different hashes than the original nodes.
|
||||||
|
return blob, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// rawDBNodeDatabase implements database.NodeDatabase using direct raw DB reads.
|
||||||
|
type rawDBNodeDatabase struct {
|
||||||
|
db ethdb.KeyValueReader
|
||||||
|
root common.Hash
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *rawDBNodeDatabase) NodeReader(stateRoot common.Hash) (database.NodeReader, error) {
|
||||||
|
// Only allow reading the persistent disk root state
|
||||||
|
if stateRoot != d.root {
|
||||||
|
return nil, fmt.Errorf("raw DB reader only supports disk root %x, got %x", d.root, stateRoot)
|
||||||
|
}
|
||||||
|
return &rawDBNodeReader{db: d.db}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func archiveGenerate(ctx *cli.Context) error {
|
func archiveGenerate(ctx *cli.Context) error {
|
||||||
// 1. Setup node and databases
|
// 1. Setup node and databases
|
||||||
stack, _ := makeConfigNode(ctx)
|
stack, _ := makeConfigNode(ctx)
|
||||||
|
|
@ -109,27 +208,25 @@ func archiveGenerate(ctx *cli.Context) error {
|
||||||
return fmt.Errorf("archive generation requires path-based state scheme, got: %s", scheme)
|
return fmt.Errorf("archive generation requires path-based state scheme, got: %s", scheme)
|
||||||
}
|
}
|
||||||
|
|
||||||
triedb := utils.MakeTrieDatabase(ctx, stack, chaindb, false, false, false)
|
// 2. Determine the persistent disk state root.
|
||||||
defer triedb.Close()
|
//
|
||||||
|
// The archiver reads and writes directly to the raw key-value database,
|
||||||
// 2. Determine state root
|
// bypassing pathdb's in-memory diff layers. This avoids the inconsistency
|
||||||
var root common.Hash
|
// where diff layers shadow expiredNode markers written to disk.
|
||||||
if ctx.NArg() > 0 {
|
//
|
||||||
root = common.HexToHash(ctx.Args().First())
|
// The disk root is computed by hashing the account trie root node stored
|
||||||
log.Info("Using specified state root", "root", root)
|
// in the raw database. This root corresponds to the last state that was
|
||||||
} else {
|
// fully persisted (i.e., PersistentStateID), which matches the canonical
|
||||||
headBlock := rawdb.ReadHeadBlock(chaindb)
|
// chain head.
|
||||||
if headBlock == nil {
|
rootBlob := rawdb.ReadAccountTrieNode(chaindb, nil)
|
||||||
return errors.New("no head block found - specify a state root or sync the chain first")
|
if len(rootBlob) == 0 {
|
||||||
}
|
|
||||||
root = headBlock.Root()
|
|
||||||
log.Info("Using head block state", "number", headBlock.NumberU64(), "root", root)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Verify the state exists
|
|
||||||
if !rawdb.HasAccountTrieNode(chaindb, nil) {
|
|
||||||
return errors.New("state trie not found in database")
|
return errors.New("state trie not found in database")
|
||||||
}
|
}
|
||||||
|
root := crypto.Keccak256Hash(rootBlob)
|
||||||
|
log.Info("Using persistent disk state root", "root", root)
|
||||||
|
|
||||||
|
// Create a raw DB node reader that bypasses pathdb layers
|
||||||
|
nodeDB := &rawDBNodeDatabase{db: chaindb, root: root}
|
||||||
|
|
||||||
// 3. Open archive writer (unless dry-run)
|
// 3. Open archive writer (unless dry-run)
|
||||||
var writer *archive.ArchiveWriter
|
var writer *archive.ArchiveWriter
|
||||||
|
|
@ -153,7 +250,7 @@ func archiveGenerate(ctx *cli.Context) error {
|
||||||
// 4. Create and run archiver
|
// 4. Create and run archiver
|
||||||
archiver := trie.NewArchiver(
|
archiver := trie.NewArchiver(
|
||||||
chaindb,
|
chaindb,
|
||||||
triedb,
|
nodeDB,
|
||||||
writer,
|
writer,
|
||||||
ctx.Uint64(archiveCompactionIntervalFlag.Name),
|
ctx.Uint64(archiveCompactionIntervalFlag.Name),
|
||||||
dryRun,
|
dryRun,
|
||||||
|
|
@ -174,6 +271,27 @@ func archiveGenerate(ctx *cli.Context) error {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if !dryRun {
|
||||||
|
// Delete the pathdb journal. The archiver modified the raw DB
|
||||||
|
// underneath the diff layers, so the journal's buffered state is
|
||||||
|
// inconsistent. Deleting forces geth to restart with a bare disk
|
||||||
|
// layer and rewind the chain head to the disk state.
|
||||||
|
if err := chaindb.Delete([]byte("TrieJournal")); err != nil {
|
||||||
|
log.Warn("Failed to delete pathdb journal key", "err", err)
|
||||||
|
}
|
||||||
|
log.Info("Deleted pathdb journal to force clean restart")
|
||||||
|
|
||||||
|
// Delete journal file(s) - check both legacy and current locations
|
||||||
|
for _, dir := range []string{"triedb", ""} {
|
||||||
|
for _, name := range []string{"merkle.journal", "verkle.journal"} {
|
||||||
|
journalFile := filepath.Join(stack.ResolvePath(dir), name)
|
||||||
|
if err := os.Remove(journalFile); err == nil {
|
||||||
|
log.Info("Deleted journal file", "path", journalFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// 6. Print summary
|
// 6. Print summary
|
||||||
var archiveSize uint64
|
var archiveSize uint64
|
||||||
if writer != nil {
|
if writer != nil {
|
||||||
|
|
@ -194,6 +312,258 @@ func archiveGenerate(ctx *cli.Context) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func archiveVerify(ctx *cli.Context) error {
|
||||||
|
stack, _ := makeConfigNode(ctx)
|
||||||
|
defer stack.Close()
|
||||||
|
|
||||||
|
// Open database read-only
|
||||||
|
chaindb := utils.MakeChainDatabase(ctx, stack, true)
|
||||||
|
defer chaindb.Close()
|
||||||
|
|
||||||
|
scheme := cycleCheckScheme(ctx, chaindb)
|
||||||
|
if scheme != rawdb.PathScheme {
|
||||||
|
return fmt.Errorf("archive verify requires path-based state scheme, got: %s", scheme)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set archive data dir so ArchivedNodeResolver can find the file
|
||||||
|
// ResolvePath("") returns the node's data directory (e.g. .ethereum/hoodi/geth),
|
||||||
|
// but ArchivedNodeResolver expects the instance directory (.ethereum/hoodi)
|
||||||
|
// since it appends "geth/nodearchive" itself.
|
||||||
|
archive.ArchiveDataDir = filepath.Dir(stack.ResolvePath(""))
|
||||||
|
|
||||||
|
// Compute disk root
|
||||||
|
rootBlob := rawdb.ReadAccountTrieNode(chaindb, nil)
|
||||||
|
if len(rootBlob) == 0 {
|
||||||
|
return errors.New("state trie not found in database")
|
||||||
|
}
|
||||||
|
root := crypto.Keccak256Hash(rootBlob)
|
||||||
|
log.Info("Verifying archived nodes", "root", root)
|
||||||
|
|
||||||
|
nodeDB := &rawDBNodeDatabase{db: chaindb, root: root}
|
||||||
|
|
||||||
|
// Open account trie
|
||||||
|
accountTrie, err := trie.New(trie.StateTrieID(root), nodeDB)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to open account trie: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
totalAccounts int
|
||||||
|
totalStorageTries int
|
||||||
|
totalLeaves int
|
||||||
|
totalExpired int
|
||||||
|
totalErrors int
|
||||||
|
start = time.Now()
|
||||||
|
lastLog = time.Now()
|
||||||
|
)
|
||||||
|
|
||||||
|
// Walk the account trie — this resolves all expired nodes and verifies hashes
|
||||||
|
accountStats, err := accountTrie.Walk(func(path []byte, value []byte) error {
|
||||||
|
totalAccounts++
|
||||||
|
if time.Since(lastLog) > 30*time.Second {
|
||||||
|
log.Info("Verification progress",
|
||||||
|
"accounts", totalAccounts,
|
||||||
|
"storageTries", totalStorageTries,
|
||||||
|
"leaves", totalLeaves,
|
||||||
|
"expired", totalExpired,
|
||||||
|
"errors", totalErrors)
|
||||||
|
lastLog = time.Now()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Decode account to check for storage trie
|
||||||
|
var acc types.StateAccount
|
||||||
|
if err := rlp.DecodeBytes(value, &acc); err != nil {
|
||||||
|
log.Warn("Failed to decode account", "err", err)
|
||||||
|
totalErrors++
|
||||||
|
return nil // continue walking
|
||||||
|
}
|
||||||
|
if acc.Root == types.EmptyRootHash {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Open and walk storage trie.
|
||||||
|
// path is hex-nibble encoded (with a 16 terminator from the trie key),
|
||||||
|
// so convert nibble pairs back to the 32-byte account hash.
|
||||||
|
nibbles := path
|
||||||
|
if len(nibbles) > 0 && nibbles[len(nibbles)-1] == 16 {
|
||||||
|
nibbles = nibbles[:len(nibbles)-1]
|
||||||
|
}
|
||||||
|
keyBytes := make([]byte, len(nibbles)/2)
|
||||||
|
for i := 0; i < len(nibbles); i += 2 {
|
||||||
|
keyBytes[i/2] = nibbles[i]<<4 | nibbles[i+1]
|
||||||
|
}
|
||||||
|
accountHash := common.BytesToHash(keyBytes)
|
||||||
|
storageID := trie.StorageTrieID(root, accountHash, acc.Root)
|
||||||
|
storageTrie, err := trie.New(storageID, nodeDB)
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("Failed to open storage trie", "account", accountHash, "err", err)
|
||||||
|
totalErrors++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
storageStats, err := storageTrie.Walk(func(spath []byte, svalue []byte) error {
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Warn("Storage trie walk failed", "account", accountHash, "err", err)
|
||||||
|
totalErrors++
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
totalStorageTries++
|
||||||
|
totalLeaves += storageStats.Leaves
|
||||||
|
totalExpired += storageStats.ExpiredResolved
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("account trie walk failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
totalLeaves += accountStats.Leaves
|
||||||
|
totalExpired += accountStats.ExpiredResolved
|
||||||
|
|
||||||
|
log.Info("Archive verification complete",
|
||||||
|
"accounts", totalAccounts,
|
||||||
|
"storageTries", totalStorageTries,
|
||||||
|
"totalLeaves", totalLeaves,
|
||||||
|
"expiredResolved", totalExpired,
|
||||||
|
"errors", totalErrors,
|
||||||
|
"elapsed", common.PrettyDuration(time.Since(start)))
|
||||||
|
|
||||||
|
if totalErrors > 0 {
|
||||||
|
return fmt.Errorf("verification completed with %d errors", totalErrors)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveDeleteJournal(ctx *cli.Context) error {
|
||||||
|
stack, _ := makeConfigNode(ctx)
|
||||||
|
defer stack.Close()
|
||||||
|
|
||||||
|
chaindb := utils.MakeChainDatabase(ctx, stack, false)
|
||||||
|
defer chaindb.Close()
|
||||||
|
|
||||||
|
// Delete the pathdb journal KV key
|
||||||
|
if err := chaindb.Delete([]byte("TrieJournal")); err != nil {
|
||||||
|
log.Warn("Failed to delete pathdb journal key", "err", err)
|
||||||
|
} else {
|
||||||
|
log.Info("Deleted pathdb journal key (TrieJournal)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete the journal file(s) - check both legacy and current locations
|
||||||
|
for _, dir := range []string{"triedb", ""} {
|
||||||
|
for _, name := range []string{"merkle.journal", "verkle.journal"} {
|
||||||
|
journalFile := filepath.Join(stack.ResolvePath(dir), name)
|
||||||
|
if err := os.Remove(journalFile); err == nil {
|
||||||
|
log.Info("Deleted journal file", "path", journalFile)
|
||||||
|
} else if !os.IsNotExist(err) {
|
||||||
|
log.Warn("Failed to delete journal file", "path", journalFile, "err", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func archiveCheckNode(ctx *cli.Context) error {
|
||||||
|
stack, _ := makeConfigNode(ctx)
|
||||||
|
defer stack.Close()
|
||||||
|
|
||||||
|
chaindb := utils.MakeChainDatabase(ctx, stack, true)
|
||||||
|
defer chaindb.Close()
|
||||||
|
|
||||||
|
ownerHex := ctx.String(archiveCheckNodeFlag.Name)
|
||||||
|
pathHex := ctx.String(archiveCheckPathFlag.Name)
|
||||||
|
|
||||||
|
if ownerHex == "" {
|
||||||
|
return errors.New("--owner flag is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
owner := common.HexToHash(ownerHex)
|
||||||
|
|
||||||
|
// Parse path: hex nibbles like "08" → []byte{0, 8}
|
||||||
|
var path []byte
|
||||||
|
for _, c := range pathHex {
|
||||||
|
var nibble byte
|
||||||
|
switch {
|
||||||
|
case c >= '0' && c <= '9':
|
||||||
|
nibble = byte(c - '0')
|
||||||
|
case c >= 'a' && c <= 'f':
|
||||||
|
nibble = byte(c-'a') + 10
|
||||||
|
case c >= 'A' && c <= 'F':
|
||||||
|
nibble = byte(c-'A') + 10
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("invalid hex char in path: %c", c)
|
||||||
|
}
|
||||||
|
path = append(path, nibble)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Info("Checking node in raw DB", "owner", owner, "path", fmt.Sprintf("%x", path))
|
||||||
|
|
||||||
|
// Read the node directly from the raw DB
|
||||||
|
isAccount := owner == (common.Hash{})
|
||||||
|
|
||||||
|
// Check the target path and all prefixes up to root
|
||||||
|
for i := len(path); i >= 0; i-- {
|
||||||
|
subpath := path[:i]
|
||||||
|
var blob []byte
|
||||||
|
if isAccount {
|
||||||
|
blob = rawdb.ReadAccountTrieNode(chaindb, subpath)
|
||||||
|
} else {
|
||||||
|
blob = rawdb.ReadStorageTrieNode(chaindb, owner, subpath)
|
||||||
|
}
|
||||||
|
|
||||||
|
status := "MISSING"
|
||||||
|
details := ""
|
||||||
|
if len(blob) > 0 {
|
||||||
|
if blob[0] == 0x00 {
|
||||||
|
status = "EXPIRED"
|
||||||
|
if len(blob) == 17 {
|
||||||
|
offset := binary.BigEndian.Uint64(blob[1:9])
|
||||||
|
size := binary.BigEndian.Uint64(blob[9:17])
|
||||||
|
details = fmt.Sprintf("offset=%d size=%d", offset, size)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
status = fmt.Sprintf("PRESENT (%d bytes, first=0x%02x)", len(blob), blob[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
label := "prefix"
|
||||||
|
if i == len(path) {
|
||||||
|
label = "TARGET"
|
||||||
|
}
|
||||||
|
if i == 0 {
|
||||||
|
label = "ROOT"
|
||||||
|
}
|
||||||
|
log.Info("Node check",
|
||||||
|
"label", label,
|
||||||
|
"path", fmt.Sprintf("%x", subpath),
|
||||||
|
"pathLen", i,
|
||||||
|
"status", status,
|
||||||
|
"details", details)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also check a few child paths to see what's below the target
|
||||||
|
for nibble := byte(0); nibble < 16; nibble++ {
|
||||||
|
childPath := append(append([]byte{}, path...), nibble)
|
||||||
|
var blob []byte
|
||||||
|
if isAccount {
|
||||||
|
blob = rawdb.ReadAccountTrieNode(chaindb, childPath)
|
||||||
|
} else {
|
||||||
|
blob = rawdb.ReadStorageTrieNode(chaindb, owner, childPath)
|
||||||
|
}
|
||||||
|
if len(blob) > 0 {
|
||||||
|
status := fmt.Sprintf("PRESENT (%d bytes, first=0x%02x)", len(blob), blob[0])
|
||||||
|
if blob[0] == 0x00 && len(blob) == 17 {
|
||||||
|
offset := binary.BigEndian.Uint64(blob[1:9])
|
||||||
|
size := binary.BigEndian.Uint64(blob[9:17])
|
||||||
|
status = fmt.Sprintf("EXPIRED offset=%d size=%d", offset, size)
|
||||||
|
}
|
||||||
|
log.Info("Child node", "path", fmt.Sprintf("%x", childPath), "status", status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// cycleCheckScheme returns the state scheme for the database.
|
// cycleCheckScheme returns the state scheme for the database.
|
||||||
// It's a helper to check what scheme is in use.
|
// It's a helper to check what scheme is in use.
|
||||||
func cycleCheckScheme(ctx *cli.Context, db ethdb.Database) string {
|
func cycleCheckScheme(ctx *cli.Context, db ethdb.Database) string {
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,7 @@ import (
|
||||||
"github.com/ethereum/go-ethereum/log"
|
"github.com/ethereum/go-ethereum/log"
|
||||||
"github.com/ethereum/go-ethereum/p2p"
|
"github.com/ethereum/go-ethereum/p2p"
|
||||||
"github.com/ethereum/go-ethereum/rpc"
|
"github.com/ethereum/go-ethereum/rpc"
|
||||||
|
"github.com/ethereum/go-ethereum/trie/archive"
|
||||||
"github.com/gofrs/flock"
|
"github.com/gofrs/flock"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -85,6 +86,7 @@ func New(conf *Config) (*Node, error) {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
conf.DataDir = absdatadir
|
conf.DataDir = absdatadir
|
||||||
|
archive.ArchiveDataDir = absdatadir
|
||||||
}
|
}
|
||||||
if conf.Logger == nil {
|
if conf.Logger == nil {
|
||||||
conf.Logger = log.New()
|
conf.Logger = log.New()
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,7 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/rlp"
|
"github.com/ethereum/go-ethereum/rlp"
|
||||||
)
|
)
|
||||||
|
|
@ -45,13 +46,16 @@ type Record struct {
|
||||||
Value []byte
|
Value []byte
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ArchiveDataDir is the data directory where the archive file is stored.
|
||||||
|
var ArchiveDataDir string
|
||||||
|
|
||||||
// ArchivedNodeResolver takes a buffer containing the archive data
|
// ArchivedNodeResolver takes a buffer containing the archive data
|
||||||
// held by an expiring node (an offset and a size) and returns a
|
// held by an expiring node (an offset and a size) and returns a
|
||||||
// list of records, which is a list of serialized leaf nodes. The
|
// list of records, which is a list of serialized leaf nodes. The
|
||||||
// caller knows the context (MPT, binary trie) and is responsible
|
// caller knows the context (MPT, binary trie) and is responsible
|
||||||
// for decoding the nodes.
|
// for decoding the nodes.
|
||||||
func ArchivedNodeResolver(offset, size uint64) ([]*Record, error) {
|
func ArchivedNodeResolver(offset, size uint64) ([]*Record, error) {
|
||||||
file, err := os.Open("nodearchive")
|
file, err := os.Open(filepath.Join(ArchiveDataDir, "geth", "nodearchive"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("error opening archive file: %w", err)
|
return nil, fmt.Errorf("error opening archive file: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
140
trie/archiver.go
140
trie/archiver.go
|
|
@ -19,6 +19,7 @@ package trie
|
||||||
import (
|
import (
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||||
|
|
@ -39,6 +40,7 @@ type subtreeInfo struct {
|
||||||
height int // Height of subtree (from leaves)
|
height int // Height of subtree (from leaves)
|
||||||
leaves []*archive.Record // All leaf records (relative path + encoded node)
|
leaves []*archive.Record // All leaf records (relative path + encoded node)
|
||||||
nodePaths [][]byte // Paths of all nodes to delete
|
nodePaths [][]byte // Paths of all nodes to delete
|
||||||
|
rootHash common.Hash // Hash of the original subtree root (for verification)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Archiver handles the archival process of trie nodes.
|
// Archiver handles the archival process of trie nodes.
|
||||||
|
|
@ -79,7 +81,7 @@ func NewArchiver(db ethdb.Database, triedb database.NodeDatabase,
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProcessState archives subtrees from the given state root.
|
// ProcessState archives subtrees from the given state root.
|
||||||
// It processes the account trie first, then all storage tries.
|
// It processes storage tries first, then the account trie.
|
||||||
func (a *Archiver) ProcessState(root common.Hash) error {
|
func (a *Archiver) ProcessState(root common.Hash) error {
|
||||||
a.stateRoot = root
|
a.stateRoot = root
|
||||||
|
|
||||||
|
|
@ -141,7 +143,12 @@ func (a *Archiver) processTrie(owner common.Hash, t *Trie) error {
|
||||||
subtrees := a.findHeight3Subtrees(t.root, nil, owner)
|
subtrees := a.findHeight3Subtrees(t.root, nil, owner)
|
||||||
log.Info("Found subtrees to archive", "owner", owner, "count", len(subtrees))
|
log.Info("Found subtrees to archive", "owner", owner, "count", len(subtrees))
|
||||||
|
|
||||||
for _, info := range subtrees {
|
lastLog := time.Now()
|
||||||
|
for i, info := range subtrees {
|
||||||
|
if time.Since(lastLog) > 30*time.Second {
|
||||||
|
log.Info("Archiving subtrees", "owner", owner, "progress", fmt.Sprintf("%d/%d", i, len(subtrees)), "archived", a.subtreesArchived)
|
||||||
|
lastLog = time.Now()
|
||||||
|
}
|
||||||
if err := a.archiveSubtree(info); err != nil {
|
if err := a.archiveSubtree(info); err != nil {
|
||||||
log.Warn("Failed to archive subtree", "path", common.Bytes2Hex(info.path), "err", err)
|
log.Warn("Failed to archive subtree", "path", common.Bytes2Hex(info.path), "err", err)
|
||||||
continue
|
continue
|
||||||
|
|
@ -159,54 +166,86 @@ func (a *Archiver) processTrie(owner common.Hash, t *Trie) error {
|
||||||
// findHeight3Subtrees recursively finds all subtrees with height == 3.
|
// findHeight3Subtrees recursively finds all subtrees with height == 3.
|
||||||
// Height is measured from leaves: leaves=0, their parents=1, etc.
|
// Height is measured from leaves: leaves=0, their parents=1, etc.
|
||||||
func (a *Archiver) findHeight3Subtrees(n node, path []byte, owner common.Hash) []*subtreeInfo {
|
func (a *Archiver) findHeight3Subtrees(n node, path []byte, owner common.Hash) []*subtreeInfo {
|
||||||
info := a.computeSubtreeInfo(n, path, owner)
|
info, err := a.computeSubtreeInfo(n, path, owner)
|
||||||
|
if err != nil {
|
||||||
|
// computeSubtreeInfo failed (e.g. unresolvable hashNode within the
|
||||||
|
// subtree). We cannot archive this node as-is, but deeper children
|
||||||
|
// may still form valid height-3 subtrees. Recurse into them.
|
||||||
|
log.Debug("computeSubtreeInfo failed, trying children", "path", common.Bytes2Hex(path), "err", err)
|
||||||
|
return a.findSubtreesInChildren(n, path, owner)
|
||||||
|
}
|
||||||
if info == nil {
|
if info == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// If this subtree has height 3, it's a candidate for archival
|
// If this subtree has height 3, it's a candidate for archival
|
||||||
if info.height == 3 {
|
if info.height == 3 {
|
||||||
|
// Capture the original subtree root hash for verification.
|
||||||
|
// The hash is available from the node that was passed in:
|
||||||
|
// - hashNode: the hash IS the node
|
||||||
|
// - fullNode/shortNode: loaded from DB, flags.hash is set
|
||||||
|
switch nn := n.(type) {
|
||||||
|
case hashNode:
|
||||||
|
info.rootHash = common.BytesToHash(nn)
|
||||||
|
case *fullNode:
|
||||||
|
if nn.flags.hash != nil {
|
||||||
|
info.rootHash = common.BytesToHash(nn.flags.hash)
|
||||||
|
}
|
||||||
|
case *shortNode:
|
||||||
|
if nn.flags.hash != nil {
|
||||||
|
info.rootHash = common.BytesToHash(nn.flags.hash)
|
||||||
|
}
|
||||||
|
}
|
||||||
return []*subtreeInfo{info}
|
return []*subtreeInfo{info}
|
||||||
}
|
}
|
||||||
|
|
||||||
// If height > 3, recurse into children to find height-3 subtrees
|
// If height > 3, recurse into children to find height-3 subtrees
|
||||||
if info.height > 3 {
|
if info.height > 3 {
|
||||||
var results []*subtreeInfo
|
return a.findSubtreesInChildren(n, path, owner)
|
||||||
switch n := n.(type) {
|
|
||||||
case *fullNode:
|
|
||||||
for i, child := range n.Children[:16] {
|
|
||||||
if child != nil {
|
|
||||||
childPath := append(append([]byte{}, path...), byte(i))
|
|
||||||
results = append(results, a.findHeight3Subtrees(child, childPath, owner)...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case *shortNode:
|
|
||||||
childPath := append(append([]byte{}, path...), n.Key...)
|
|
||||||
results = append(results, a.findHeight3Subtrees(n.Val, childPath, owner)...)
|
|
||||||
case hashNode:
|
|
||||||
// Resolve and recurse
|
|
||||||
resolved, err := a.resolveNode(n, path, owner)
|
|
||||||
if err == nil {
|
|
||||||
results = append(results, a.findHeight3Subtrees(resolved, path, owner)...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return results
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Height < 3: no archivable subtrees here
|
// Height < 3: no archivable subtrees here
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// findSubtreesInChildren recurses into the children of a node to find
|
||||||
|
// height-3 subtrees. Used both by the normal height > 3 path and as a
|
||||||
|
// fallback when computeSubtreeInfo fails for a node.
|
||||||
|
func (a *Archiver) findSubtreesInChildren(n node, path []byte, owner common.Hash) []*subtreeInfo {
|
||||||
|
var results []*subtreeInfo
|
||||||
|
switch n := n.(type) {
|
||||||
|
case *fullNode:
|
||||||
|
for i, child := range n.Children[:16] {
|
||||||
|
if child != nil {
|
||||||
|
childPath := append(append([]byte{}, path...), byte(i))
|
||||||
|
results = append(results, a.findHeight3Subtrees(child, childPath, owner)...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case *shortNode:
|
||||||
|
childPath := append(append([]byte{}, path...), n.Key...)
|
||||||
|
results = append(results, a.findHeight3Subtrees(n.Val, childPath, owner)...)
|
||||||
|
case hashNode:
|
||||||
|
// Resolve and recurse
|
||||||
|
resolved, err := a.resolveNode(n, path, owner)
|
||||||
|
if err == nil {
|
||||||
|
results = append(results, a.findHeight3Subtrees(resolved, path, owner)...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return results
|
||||||
|
}
|
||||||
|
|
||||||
// computeSubtreeInfo computes height and collects leaves for a subtree.
|
// computeSubtreeInfo computes height and collects leaves for a subtree.
|
||||||
// Returns nil if the node is nil or an error occurs during resolution.
|
// Returns (nil, nil) if the node is nil, already expired, or has no leaves.
|
||||||
func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *subtreeInfo {
|
// Returns (nil, error) if any constituent node could not be resolved — the
|
||||||
|
// caller MUST NOT archive a subtree when an error is returned, as the leaf
|
||||||
|
// set would be incomplete.
|
||||||
|
func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) (*subtreeInfo, error) {
|
||||||
switch n := n.(type) {
|
switch n := n.(type) {
|
||||||
case nil:
|
case nil:
|
||||||
return nil
|
return nil, nil
|
||||||
|
|
||||||
case valueNode:
|
case valueNode:
|
||||||
// Leaf: height 0
|
// Leaf: height 0
|
||||||
// Encode the leaf as a shortNode for archive storage
|
|
||||||
return &subtreeInfo{
|
return &subtreeInfo{
|
||||||
path: copyBytes(path),
|
path: copyBytes(path),
|
||||||
owner: owner,
|
owner: owner,
|
||||||
|
|
@ -216,13 +255,16 @@ func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *s
|
||||||
Value: []byte(n),
|
Value: []byte(n),
|
||||||
}},
|
}},
|
||||||
nodePaths: [][]byte{copyBytes(path)},
|
nodePaths: [][]byte{copyBytes(path)},
|
||||||
}
|
}, nil
|
||||||
|
|
||||||
case *shortNode:
|
case *shortNode:
|
||||||
childPath := append(append([]byte{}, path...), n.Key...)
|
childPath := append(append([]byte{}, path...), n.Key...)
|
||||||
childInfo := a.computeSubtreeInfo(n.Val, childPath, owner)
|
childInfo, err := a.computeSubtreeInfo(n.Val, childPath, owner)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("shortNode key=%x: %w", n.Key, err)
|
||||||
|
}
|
||||||
if childInfo == nil {
|
if childInfo == nil {
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Adjust relative paths in leaves to include this node's key
|
// Adjust relative paths in leaves to include this node's key
|
||||||
|
|
@ -236,7 +278,7 @@ func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *s
|
||||||
height: childInfo.height + 1,
|
height: childInfo.height + 1,
|
||||||
leaves: childInfo.leaves,
|
leaves: childInfo.leaves,
|
||||||
nodePaths: append([][]byte{copyBytes(path)}, childInfo.nodePaths...),
|
nodePaths: append([][]byte{copyBytes(path)}, childInfo.nodePaths...),
|
||||||
}
|
}, nil
|
||||||
|
|
||||||
case *fullNode:
|
case *fullNode:
|
||||||
var (
|
var (
|
||||||
|
|
@ -247,7 +289,10 @@ func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *s
|
||||||
for i, child := range n.Children[:16] {
|
for i, child := range n.Children[:16] {
|
||||||
if child != nil {
|
if child != nil {
|
||||||
childPath := append(append([]byte{}, path...), byte(i))
|
childPath := append(append([]byte{}, path...), byte(i))
|
||||||
childInfo := a.computeSubtreeInfo(child, childPath, owner)
|
childInfo, err := a.computeSubtreeInfo(child, childPath, owner)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("fullNode child[%x]: %w", i, err)
|
||||||
|
}
|
||||||
if childInfo != nil {
|
if childInfo != nil {
|
||||||
if childInfo.height+1 > maxHeight {
|
if childInfo.height+1 > maxHeight {
|
||||||
maxHeight = childInfo.height + 1
|
maxHeight = childInfo.height + 1
|
||||||
|
|
@ -263,7 +308,7 @@ func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *s
|
||||||
}
|
}
|
||||||
|
|
||||||
if len(allLeaves) == 0 {
|
if len(allLeaves) == 0 {
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return &subtreeInfo{
|
return &subtreeInfo{
|
||||||
|
|
@ -272,21 +317,20 @@ func (a *Archiver) computeSubtreeInfo(n node, path []byte, owner common.Hash) *s
|
||||||
height: maxHeight,
|
height: maxHeight,
|
||||||
leaves: allLeaves,
|
leaves: allLeaves,
|
||||||
nodePaths: allPaths,
|
nodePaths: allPaths,
|
||||||
}
|
}, nil
|
||||||
|
|
||||||
case hashNode:
|
case hashNode:
|
||||||
resolved, err := a.resolveNode(n, path, owner)
|
resolved, err := a.resolveNode(n, path, owner)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Debug("Failed to resolve hashNode", "path", common.Bytes2Hex(path), "err", err)
|
return nil, fmt.Errorf("failed to resolve hashNode at path %s: %w", common.Bytes2Hex(path), err)
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
return a.computeSubtreeInfo(resolved, path, owner)
|
return a.computeSubtreeInfo(resolved, path, owner)
|
||||||
|
|
||||||
case *expiredNode:
|
case *expiredNode:
|
||||||
// Already archived, skip
|
// Already archived, skip
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// archiveSubtree writes leaves to archive and replaces subtree with expiredNode.
|
// archiveSubtree writes leaves to archive and replaces subtree with expiredNode.
|
||||||
|
|
@ -312,7 +356,25 @@ func (a *Archiver) archiveSubtree(info *subtreeInfo) error {
|
||||||
return fmt.Errorf("failed to sync archive: %w", err)
|
return fmt.Errorf("failed to sync archive: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
// 3. Batch database operations
|
// 3. Verify archive round-trip: reconstruct trie from records and
|
||||||
|
// check that the hash matches the original subtree root. This
|
||||||
|
// catches any data corruption before we delete the original nodes.
|
||||||
|
if info.rootHash != (common.Hash{}) {
|
||||||
|
reconstructed, err := archiveRecordsToNode(info.leaves)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("archive verification failed: cannot reconstruct trie from records: %w", err)
|
||||||
|
}
|
||||||
|
h := newHasher(false)
|
||||||
|
gotHash := common.BytesToHash(h.hash(reconstructed, true))
|
||||||
|
returnHasherToPool(h)
|
||||||
|
if gotHash != info.rootHash {
|
||||||
|
return fmt.Errorf("archive verification failed: hash mismatch at path %s owner %s: got %s want %s (leaves=%d offset=%d size=%d)",
|
||||||
|
common.Bytes2Hex(info.path), info.owner, gotHash, info.rootHash,
|
||||||
|
len(info.leaves), offset, size)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Batch database operations
|
||||||
batch := a.db.NewBatch()
|
batch := a.db.NewBatch()
|
||||||
|
|
||||||
// Delete all nodes in subtree (except the root which we'll overwrite)
|
// Delete all nodes in subtree (except the root which we'll overwrite)
|
||||||
|
|
|
||||||
|
|
@ -79,6 +79,8 @@ func (c *committer) commit(path []byte, n node, parallel bool) node {
|
||||||
return cn
|
return cn
|
||||||
case hashNode:
|
case hashNode:
|
||||||
return cn
|
return cn
|
||||||
|
case *expiredNode:
|
||||||
|
return cn
|
||||||
default:
|
default:
|
||||||
// nil, valuenode shouldn't be committed
|
// nil, valuenode shouldn't be committed
|
||||||
panic(fmt.Sprintf("%T: invalid node: %v", n, n))
|
panic(fmt.Sprintf("%T: invalid node: %v", n, n))
|
||||||
|
|
|
||||||
|
|
@ -17,6 +17,7 @@
|
||||||
package trie
|
package trie
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"encoding/binary"
|
"encoding/binary"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
|
|
@ -33,11 +34,12 @@ const expiredNodeMarker = 0x00
|
||||||
type expiredNode struct {
|
type expiredNode struct {
|
||||||
offset uint64
|
offset uint64
|
||||||
size uint64
|
size uint64
|
||||||
|
cachedHash hashNode
|
||||||
archiveResolver archive.ResolverFn
|
archiveResolver archive.ResolverFn
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *expiredNode) cache() (hashNode, bool) {
|
func (n *expiredNode) cache() (hashNode, bool) {
|
||||||
return nil, true
|
return n.cachedHash, n.cachedHash == nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *expiredNode) encode(w rlp.EncoderBuffer) {
|
func (n *expiredNode) encode(w rlp.EncoderBuffer) {
|
||||||
|
|
@ -62,36 +64,101 @@ func (n *expiredNode) SetArchiveResolver(resolver archive.ResolverFn) {
|
||||||
n.archiveResolver = resolver
|
n.archiveResolver = resolver
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// resolveExpiredNodeData resolves an expired node from the archive, verifies
|
||||||
|
// the reconstructed subtree hash, and stamps the cached hash onto the root.
|
||||||
|
// Returns an error if the archive data is corrupted (hash mismatch).
|
||||||
|
func resolveExpiredNodeData(n *expiredNode) (node, error) {
|
||||||
|
records, err := archive.ArchivedNodeResolver(n.offset, n.size)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to resolve expired node: %w", err)
|
||||||
|
}
|
||||||
|
resolved, err := archiveRecordsToNode(records)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to rebuild expired node from archive: %w", err)
|
||||||
|
}
|
||||||
|
// Verify hash integrity: if the original hash is known, check that the
|
||||||
|
// reconstructed subtree produces the same hash. A mismatch means the
|
||||||
|
// archive is corrupted (e.g. missing leaves due to unresolvable hashNodes
|
||||||
|
// during archival) and any data from it is unreliable.
|
||||||
|
if n.cachedHash != nil {
|
||||||
|
h := newHasher(false)
|
||||||
|
gotHash := h.hash(resolved, true)
|
||||||
|
returnHasherToPool(h)
|
||||||
|
if !bytes.Equal(gotHash, n.cachedHash) {
|
||||||
|
return nil, fmt.Errorf("expired node hash mismatch at offset=%d size=%d: archive data is corrupted (expected %x got %x, %d records)",
|
||||||
|
n.offset, n.size, []byte(n.cachedHash), gotHash, len(records))
|
||||||
|
}
|
||||||
|
// Stamp the original hash onto the resolved subtree root so the
|
||||||
|
// hasher returns it directly instead of re-computing.
|
||||||
|
switch nn := resolved.(type) {
|
||||||
|
case *fullNode:
|
||||||
|
nn.flags.hash = n.cachedHash
|
||||||
|
case *shortNode:
|
||||||
|
nn.flags.hash = n.cachedHash
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Mark the entire resolved subtree as dirty. This is critical for
|
||||||
|
// correctness with pathdb's diff layer model: when a trie with expired
|
||||||
|
// nodes is modified and committed, the committer only captures dirty
|
||||||
|
// nodes into the NodeSet (which becomes the diff layer). Without this
|
||||||
|
// marking, resolved-but-unmodified sibling nodes within the subtree
|
||||||
|
// would exist nowhere — not in any diff layer (they're clean) and not
|
||||||
|
// in the raw DB (the archiver deleted them). Subsequent trie accesses
|
||||||
|
// from higher diff layers would fall through to the disk layer, find
|
||||||
|
// nothing, and produce MissingNodeError.
|
||||||
|
//
|
||||||
|
// For read-only tries (only get operations, no commit), this dirty
|
||||||
|
// marking is harmless — the nodes are discarded when the trie is GC'd.
|
||||||
|
markSubtreeDirty(resolved)
|
||||||
|
return resolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// markSubtreeDirty recursively marks all fullNode and shortNode in the
|
||||||
|
// subtree as dirty, preserving any cached hashes. This ensures the
|
||||||
|
// committer will capture them in the NodeSet during trie commit.
|
||||||
|
func markSubtreeDirty(n node) {
|
||||||
|
switch n := n.(type) {
|
||||||
|
case *fullNode:
|
||||||
|
n.flags.dirty = true
|
||||||
|
for _, child := range n.Children[:16] {
|
||||||
|
if child != nil {
|
||||||
|
markSubtreeDirty(child)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case *shortNode:
|
||||||
|
n.flags.dirty = true
|
||||||
|
markSubtreeDirty(n.Val)
|
||||||
|
}
|
||||||
|
// valueNode, hashNode, nil: no flags to mark
|
||||||
|
}
|
||||||
|
|
||||||
func archiveRecordsToNode(records []*archive.Record) (node, error) {
|
func archiveRecordsToNode(records []*archive.Record) (node, error) {
|
||||||
if len(records) == 0 {
|
if len(records) == 0 {
|
||||||
return nil, archive.EmptyArchiveRecord
|
return nil, archive.EmptyArchiveRecord
|
||||||
}
|
}
|
||||||
if len(records) == 1 {
|
|
||||||
return buildLeafFromRecord(records[0])
|
|
||||||
}
|
|
||||||
|
|
||||||
var newnode fullNode
|
// Build the trie incrementally from nil to produce the canonical
|
||||||
|
// MPT structure. Starting with a fullNode would be wrong when the
|
||||||
|
// original subtree root was a shortNode (shared prefix).
|
||||||
|
var root node
|
||||||
for i, record := range records {
|
for i, record := range records {
|
||||||
if err := validateRecordPath(record.Path); err != nil {
|
if err := validateRecordPath(record.Path); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// we are not in the case of a single leaf node, so each
|
|
||||||
// path should be at least 2 nibbles (terminator included)
|
|
||||||
if len(record.Path) < 2 || !hasTerm(record.Path) {
|
|
||||||
return nil, fmt.Errorf("invalid record path for non-leaf node #%d: %v", i, record.Path)
|
|
||||||
}
|
|
||||||
key, err := normalizeRecordKey(record.Path)
|
key, err := normalizeRecordKey(record.Path)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
child, err := insertTrieNode(newnode.Children[key[0]], key[1:], valueNode(record.Value))
|
if len(key) < 1 {
|
||||||
|
return nil, fmt.Errorf("empty key in record #%d", i)
|
||||||
|
}
|
||||||
|
root, err = insertTrieNode(root, key, valueNode(record.Value))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
newnode.Children[key[0]] = child
|
|
||||||
}
|
}
|
||||||
return &newnode, nil
|
return root, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func validateRecordPath(path []byte) error {
|
func validateRecordPath(path []byte) error {
|
||||||
|
|
@ -106,14 +173,6 @@ func validateRecordPath(path []byte) error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func buildLeafFromRecord(record *archive.Record) (node, error) {
|
|
||||||
key, err := normalizeRecordKey(record.Path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &shortNode{Key: key, Val: valueNode(record.Value)}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// normalizeRecordKey ensures the record path is a hex-nibble key suitable for
|
// normalizeRecordKey ensures the record path is a hex-nibble key suitable for
|
||||||
// leaf insertion by guaranteeing a single terminator nibble and preserving any
|
// leaf insertion by guaranteeing a single terminator nibble and preserving any
|
||||||
// already-terminated path. Empty paths are normalized to a sole terminator.
|
// already-terminated path. Empty paths are normalized to a sole terminator.
|
||||||
|
|
|
||||||
|
|
@ -19,12 +19,45 @@ package trie
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/rlp"
|
"github.com/ethereum/go-ethereum/rlp"
|
||||||
"github.com/ethereum/go-ethereum/trie/archive"
|
"github.com/ethereum/go-ethereum/trie/archive"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// setupTestArchive creates a temporary archive directory with an archive file
|
||||||
|
// containing the given records, and configures archive.ArchiveDataDir to point
|
||||||
|
// to it. It returns the offset and size of the written data, and a cleanup function.
|
||||||
|
func setupTestArchive(t *testing.T, records []*archive.Record) (offset, size uint64, cleanup func()) {
|
||||||
|
t.Helper()
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
gethDir := filepath.Join(tmpDir, "geth")
|
||||||
|
if err := os.MkdirAll(gethDir, 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
writer, err := archive.NewArchiveWriter(filepath.Join(gethDir, "nodearchive"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
offset, size, err = writer.WriteSubtree(records)
|
||||||
|
if err != nil {
|
||||||
|
writer.Close()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
writer.Close()
|
||||||
|
|
||||||
|
oldDir := archive.ArchiveDataDir
|
||||||
|
archive.ArchiveDataDir = tmpDir
|
||||||
|
|
||||||
|
return offset, size, func() {
|
||||||
|
archive.ArchiveDataDir = oldDir
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestExpiredNodeEncodeDecode(t *testing.T) {
|
func TestExpiredNodeEncodeDecode(t *testing.T) {
|
||||||
testCases := []struct {
|
testCases := []struct {
|
||||||
offset uint64
|
offset uint64
|
||||||
|
|
@ -120,34 +153,36 @@ func TestExpiredNodeInvalidLength(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExpiredNodeNoResolver(t *testing.T) {
|
func TestExpiredNodeNoArchiveFile(t *testing.T) {
|
||||||
|
// When no archive file exists, Get should return an error
|
||||||
|
tmpDir := t.TempDir()
|
||||||
|
gethDir := filepath.Join(tmpDir, "geth")
|
||||||
|
if err := os.MkdirAll(gethDir, 0755); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
oldDir := archive.ArchiveDataDir
|
||||||
|
archive.ArchiveDataDir = tmpDir
|
||||||
|
defer func() { archive.ArchiveDataDir = oldDir }()
|
||||||
|
|
||||||
tr := NewEmpty(nil)
|
tr := NewEmpty(nil)
|
||||||
tr.root = &expiredNode{offset: 100}
|
tr.root = &expiredNode{offset: 100, size: 50}
|
||||||
|
|
||||||
_, err := tr.Get([]byte("key"))
|
_, err := tr.Get([]byte("key"))
|
||||||
if !errors.Is(err, archive.ErrNoResolver) {
|
if err == nil {
|
||||||
t.Errorf("expected archive.ErrNoResolver, got %v", err)
|
t.Error("expected error when archive file doesn't exist")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExpiredNodeWithResolver(t *testing.T) {
|
func TestExpiredNodeWithResolver(t *testing.T) {
|
||||||
|
records := []*archive.Record{
|
||||||
|
{Path: []byte{0x01, 0x02, 16}, Value: []byte("testvalue")},
|
||||||
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
tr := NewEmpty(nil)
|
tr := NewEmpty(nil)
|
||||||
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
leafNode := &shortNode{
|
|
||||||
Key: hexToCompact(keybytesToHex([]byte{0x12})),
|
|
||||||
Val: valueNode([]byte("testvalue")),
|
|
||||||
}
|
|
||||||
encodedLeaf := nodeToBytes(leafNode)
|
|
||||||
|
|
||||||
resolver := func(offset, size uint64) ([]*archive.Record, error) {
|
|
||||||
if offset == 100 {
|
|
||||||
return []*archive.Record{{Value: encodedLeaf}}, nil
|
|
||||||
}
|
|
||||||
return nil, errors.New("unknown offset")
|
|
||||||
}
|
|
||||||
|
|
||||||
tr.SetArchiveResolver(resolver)
|
|
||||||
tr.root = &expiredNode{offset: 100, size: uint64(len(encodedLeaf)), archiveResolver: resolver}
|
|
||||||
|
|
||||||
val, err := tr.Get([]byte{0x12})
|
val, err := tr.Get([]byte{0x12})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -159,14 +194,10 @@ func TestExpiredNodeWithResolver(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExpiredNodeCopy(t *testing.T) {
|
func TestExpiredNodeCopy(t *testing.T) {
|
||||||
resolver := func(offset, size uint64) ([]*archive.Record, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
original := &expiredNode{
|
original := &expiredNode{
|
||||||
offset: 12345,
|
offset: 12345,
|
||||||
size: 6789,
|
size: 6789,
|
||||||
archiveResolver: resolver,
|
archiveResolver: archive.ArchivedNodeResolver,
|
||||||
}
|
}
|
||||||
|
|
||||||
copied := copyNode(original)
|
copied := copyNode(original)
|
||||||
|
|
@ -201,18 +232,9 @@ func TestArchiveRecordsToNodeEmpty(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestArchiveRecordsToNodeMultiple(t *testing.T) {
|
func TestArchiveRecordsToNodeMultiple(t *testing.T) {
|
||||||
leaf1 := &shortNode{
|
|
||||||
Key: hexToCompact(keybytesToHex([]byte{0x10})),
|
|
||||||
Val: valueNode([]byte("value1")),
|
|
||||||
}
|
|
||||||
leaf2 := &shortNode{
|
|
||||||
Key: hexToCompact(keybytesToHex([]byte{0x20})),
|
|
||||||
Val: valueNode([]byte("value2")),
|
|
||||||
}
|
|
||||||
|
|
||||||
records := []*archive.Record{
|
records := []*archive.Record{
|
||||||
{Path: []byte{0x01}, Value: nodeToBytes(leaf1)},
|
{Path: []byte{0x01, 16}, Value: []byte("value1")},
|
||||||
{Path: []byte{0x02}, Value: nodeToBytes(leaf2)},
|
{Path: []byte{0x02, 16}, Value: []byte("value2")},
|
||||||
}
|
}
|
||||||
|
|
||||||
node, err := archiveRecordsToNode(records)
|
node, err := archiveRecordsToNode(records)
|
||||||
|
|
@ -234,27 +256,17 @@ func TestArchiveRecordsToNodeMultiple(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExpiredNodeGetMultipleRecords(t *testing.T) {
|
func TestExpiredNodeGetMultipleRecords(t *testing.T) {
|
||||||
leaf1 := &shortNode{
|
records := []*archive.Record{
|
||||||
Key: hexToCompact([]byte{0x02, 0x03, 0x04, 16}),
|
{Path: []byte{0x01, 0x02, 16}, Value: []byte("value1")},
|
||||||
Val: valueNode([]byte("value1")),
|
{Path: []byte{0x04, 0x05, 16}, Value: []byte("value2")},
|
||||||
}
|
|
||||||
leaf2 := &shortNode{
|
|
||||||
Key: hexToCompact([]byte{0x05, 0x06, 0x07, 16}),
|
|
||||||
Val: valueNode([]byte("value2")),
|
|
||||||
}
|
|
||||||
|
|
||||||
resolver := func(offset, size uint64) ([]*archive.Record, error) {
|
|
||||||
return []*archive.Record{
|
|
||||||
{Path: []byte{0x01}, Value: nodeToBytes(leaf1)},
|
|
||||||
{Path: []byte{0x04}, Value: nodeToBytes(leaf2)},
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
tr := NewEmpty(nil)
|
tr := NewEmpty(nil)
|
||||||
tr.SetArchiveResolver(resolver)
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
|
|
||||||
|
|
||||||
val, err := tr.Get([]byte{0x12, 0x34})
|
val, err := tr.Get([]byte{0x12})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -263,10 +275,9 @@ func TestExpiredNodeGetMultipleRecords(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
tr2 := NewEmpty(nil)
|
tr2 := NewEmpty(nil)
|
||||||
tr2.SetArchiveResolver(resolver)
|
tr2.root = &expiredNode{offset: offset, size: size}
|
||||||
tr2.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
|
|
||||||
|
|
||||||
val2, err := tr2.Get([]byte{0x45, 0x67})
|
val2, err := tr2.Get([]byte{0x45})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -276,20 +287,14 @@ func TestExpiredNodeGetMultipleRecords(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExpiredNodeGetKeyNotFound(t *testing.T) {
|
func TestExpiredNodeGetKeyNotFound(t *testing.T) {
|
||||||
leaf := &shortNode{
|
records := []*archive.Record{
|
||||||
Key: hexToCompact(keybytesToHex([]byte{0x12})),
|
{Path: []byte{0x01, 0x02, 16}, Value: []byte("value1")},
|
||||||
Val: valueNode([]byte("value1")),
|
|
||||||
}
|
|
||||||
|
|
||||||
resolver := func(offset, size uint64) ([]*archive.Record, error) {
|
|
||||||
return []*archive.Record{
|
|
||||||
{Path: []byte{0x01}, Value: nodeToBytes(leaf)},
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
tr := NewEmpty(nil)
|
tr := NewEmpty(nil)
|
||||||
tr.SetArchiveResolver(resolver)
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
|
|
||||||
|
|
||||||
val, err := tr.Get([]byte{0xff, 0xff})
|
val, err := tr.Get([]byte{0xff, 0xff})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -301,20 +306,14 @@ func TestExpiredNodeGetKeyNotFound(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExpiredNodeGetPathMismatch(t *testing.T) {
|
func TestExpiredNodeGetPathMismatch(t *testing.T) {
|
||||||
leaf := &shortNode{
|
records := []*archive.Record{
|
||||||
Key: hexToCompact(keybytesToHex([]byte{0x12})),
|
{Path: []byte{0x01, 0x02, 16}, Value: []byte("testvalue")},
|
||||||
Val: valueNode([]byte("testvalue")),
|
|
||||||
}
|
|
||||||
|
|
||||||
resolver := func(offset, size uint64) ([]*archive.Record, error) {
|
|
||||||
return []*archive.Record{
|
|
||||||
{Path: []byte{0x01}, Value: nodeToBytes(leaf)},
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
tr := NewEmpty(nil)
|
tr := NewEmpty(nil)
|
||||||
tr.SetArchiveResolver(resolver)
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
|
|
||||||
|
|
||||||
val, err := tr.Get([]byte{0x19})
|
val, err := tr.Get([]byte{0x19})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -326,20 +325,14 @@ func TestExpiredNodeGetPathMismatch(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExpiredNodeInsert(t *testing.T) {
|
func TestExpiredNodeInsert(t *testing.T) {
|
||||||
leaf := &shortNode{
|
records := []*archive.Record{
|
||||||
Key: hexToCompact(keybytesToHex([]byte{0x12})),
|
{Path: []byte{0x01, 0x02, 16}, Value: []byte("existing")},
|
||||||
Val: valueNode([]byte("existing")),
|
|
||||||
}
|
|
||||||
|
|
||||||
resolver := func(offset, size uint64) ([]*archive.Record, error) {
|
|
||||||
return []*archive.Record{
|
|
||||||
{Path: []byte{}, Value: nodeToBytes(leaf)},
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
tr := NewEmpty(nil)
|
tr := NewEmpty(nil)
|
||||||
tr.SetArchiveResolver(resolver)
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
|
|
||||||
|
|
||||||
err := tr.Update([]byte{0x45}, []byte("newvalue"))
|
err := tr.Update([]byte{0x45}, []byte("newvalue"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -356,20 +349,14 @@ func TestExpiredNodeInsert(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExpiredNodeUpdate(t *testing.T) {
|
func TestExpiredNodeUpdate(t *testing.T) {
|
||||||
leaf := &shortNode{
|
records := []*archive.Record{
|
||||||
Key: hexToCompact(keybytesToHex([]byte{0x12})),
|
{Path: []byte{0x01, 0x02, 16}, Value: []byte("oldvalue")},
|
||||||
Val: valueNode([]byte("oldvalue")),
|
|
||||||
}
|
|
||||||
|
|
||||||
resolver := func(offset, size uint64) ([]*archive.Record, error) {
|
|
||||||
return []*archive.Record{
|
|
||||||
{Path: []byte{}, Value: nodeToBytes(leaf)},
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
tr := NewEmpty(nil)
|
tr := NewEmpty(nil)
|
||||||
tr.SetArchiveResolver(resolver)
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
|
|
||||||
|
|
||||||
err := tr.Update([]byte{0x12}, []byte("newvalue"))
|
err := tr.Update([]byte{0x12}, []byte("newvalue"))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -386,28 +373,15 @@ func TestExpiredNodeUpdate(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExpiredNodeDelete(t *testing.T) {
|
func TestExpiredNodeDelete(t *testing.T) {
|
||||||
leaf1 := &shortNode{
|
records := []*archive.Record{
|
||||||
Key: hexToCompact([]byte{0x02, 16}),
|
{Path: []byte{0x01, 0x02, 16}, Value: []byte("value1")},
|
||||||
Val: valueNode([]byte("value1")),
|
{Path: []byte{0x04, 0x05, 16}, Value: []byte("value2")},
|
||||||
}
|
|
||||||
leaf2 := &shortNode{
|
|
||||||
Key: hexToCompact([]byte{0x05, 16}),
|
|
||||||
Val: valueNode([]byte("value2")),
|
|
||||||
}
|
|
||||||
|
|
||||||
branch := &fullNode{}
|
|
||||||
branch.Children[0x01] = leaf1
|
|
||||||
branch.Children[0x04] = leaf2
|
|
||||||
|
|
||||||
resolver := func(offset, size uint64) ([]*archive.Record, error) {
|
|
||||||
return []*archive.Record{
|
|
||||||
{Path: []byte{}, Value: nodeToBytes(branch)},
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
tr := NewEmpty(nil)
|
tr := NewEmpty(nil)
|
||||||
tr.SetArchiveResolver(resolver)
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
|
|
||||||
|
|
||||||
err := tr.Delete([]byte{0x12})
|
err := tr.Delete([]byte{0x12})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -432,22 +406,14 @@ func TestExpiredNodeDelete(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestTrieCopyPreservesArchiveResolver(t *testing.T) {
|
func TestTrieCopyPreservesArchiveResolver(t *testing.T) {
|
||||||
leaf := &shortNode{
|
records := []*archive.Record{
|
||||||
Key: hexToCompact(keybytesToHex([]byte{0x12})),
|
{Path: []byte{0x01, 0x02, 16}, Value: []byte("testvalue")},
|
||||||
Val: valueNode([]byte("testvalue")),
|
|
||||||
}
|
|
||||||
|
|
||||||
resolverCalled := false
|
|
||||||
resolver := func(offset, size uint64) ([]*archive.Record, error) {
|
|
||||||
resolverCalled = true
|
|
||||||
return []*archive.Record{
|
|
||||||
{Path: []byte{}, Value: nodeToBytes(leaf)},
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
tr := NewEmpty(nil)
|
tr := NewEmpty(nil)
|
||||||
tr.SetArchiveResolver(resolver)
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
|
|
||||||
|
|
||||||
trCopy := tr.Copy()
|
trCopy := tr.Copy()
|
||||||
|
|
||||||
|
|
@ -455,36 +421,180 @@ func TestTrieCopyPreservesArchiveResolver(t *testing.T) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
if !resolverCalled {
|
|
||||||
t.Error("resolver was not called on copied trie")
|
|
||||||
}
|
|
||||||
if string(val) != "testvalue" {
|
if string(val) != "testvalue" {
|
||||||
t.Errorf("value mismatch: got %q, want %q", val, "testvalue")
|
t.Errorf("value mismatch: got %q, want %q", val, "testvalue")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestExpiredNodeGetNode(t *testing.T) {
|
func TestWalkWithExpiredNodes(t *testing.T) {
|
||||||
leaf := &shortNode{
|
records := []*archive.Record{
|
||||||
Key: hexToCompact(keybytesToHex([]byte{0x12})),
|
{Path: []byte{0x01, 0x02, 16}, Value: []byte("value1")},
|
||||||
Val: valueNode([]byte("testvalue")),
|
{Path: []byte{0x04, 0x05, 16}, Value: []byte("value2")},
|
||||||
}
|
{Path: []byte{0x07, 0x08, 16}, Value: []byte("value3")},
|
||||||
|
|
||||||
resolverCalled := false
|
|
||||||
resolver := func(offset, size uint64) ([]*archive.Record, error) {
|
|
||||||
resolverCalled = true
|
|
||||||
return []*archive.Record{
|
|
||||||
{Path: []byte{}, Value: nodeToBytes(leaf)},
|
|
||||||
}, nil
|
|
||||||
}
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
tr := NewEmpty(nil)
|
tr := NewEmpty(nil)
|
||||||
tr.SetArchiveResolver(resolver)
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
tr.root = &expiredNode{offset: 100, size: 200, archiveResolver: resolver}
|
|
||||||
|
var leaves []string
|
||||||
|
stats, err := tr.Walk(func(path []byte, value []byte) error {
|
||||||
|
leaves = append(leaves, string(value))
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Walk failed: %v", err)
|
||||||
|
}
|
||||||
|
if stats.Leaves != 3 {
|
||||||
|
t.Errorf("expected 3 leaves, got %d", stats.Leaves)
|
||||||
|
}
|
||||||
|
if stats.ExpiredResolved != 1 {
|
||||||
|
t.Errorf("expected 1 expired resolved, got %d", stats.ExpiredResolved)
|
||||||
|
}
|
||||||
|
// Verify all values were visited
|
||||||
|
expected := map[string]bool{"value1": true, "value2": true, "value3": true}
|
||||||
|
for _, leaf := range leaves {
|
||||||
|
if !expected[leaf] {
|
||||||
|
t.Errorf("unexpected leaf value: %q", leaf)
|
||||||
|
}
|
||||||
|
delete(expected, leaf)
|
||||||
|
}
|
||||||
|
if len(expected) > 0 {
|
||||||
|
t.Errorf("missing leaves: %v", expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWalkEmptyTrie(t *testing.T) {
|
||||||
|
tr := NewEmpty(nil)
|
||||||
|
stats, err := tr.Walk(func(path []byte, value []byte) error {
|
||||||
|
t.Error("callback should not be called for empty trie")
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Walk failed: %v", err)
|
||||||
|
}
|
||||||
|
if stats.Leaves != 0 || stats.ExpiredResolved != 0 {
|
||||||
|
t.Errorf("expected zero stats for empty trie, got leaves=%d expired=%d", stats.Leaves, stats.ExpiredResolved)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWalkCallbackError(t *testing.T) {
|
||||||
|
records := []*archive.Record{
|
||||||
|
{Path: []byte{0x01, 0x02, 16}, Value: []byte("value1")},
|
||||||
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
tr := NewEmpty(nil)
|
||||||
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
|
|
||||||
|
testErr := errors.New("test error")
|
||||||
|
_, err := tr.Walk(func(path []byte, value []byte) error {
|
||||||
|
return testErr
|
||||||
|
})
|
||||||
|
if !errors.Is(err, testErr) {
|
||||||
|
t.Fatalf("expected test error, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestExpiredNodeResolvedSubtreeDirty verifies that when an expired node is
|
||||||
|
// resolved and a sibling leaf is modified, the commit captures ALL resolved
|
||||||
|
// nodes (not just the modified path). Without this fix, resolved-but-unmodified
|
||||||
|
// nodes would be lost: not in the diff layer (clean) and not in the raw DB
|
||||||
|
// (deleted by archiver).
|
||||||
|
func TestExpiredNodeResolvedSubtreeDirty(t *testing.T) {
|
||||||
|
// Use large values (>32 bytes) so leaf nodes are NOT embedded in
|
||||||
|
// their parent. This matches production storage tries where
|
||||||
|
// intermediate nodes are large enough to be stored independently.
|
||||||
|
bigVal1 := bytes.Repeat([]byte("A"), 40)
|
||||||
|
bigVal2 := bytes.Repeat([]byte("B"), 40)
|
||||||
|
|
||||||
|
// Create an archive with records under different branches.
|
||||||
|
records := []*archive.Record{
|
||||||
|
{Path: []byte{0x01, 0x02, 16}, Value: bigVal1},
|
||||||
|
{Path: []byte{0x04, 0x05, 16}, Value: bigVal2},
|
||||||
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
tr := NewEmpty(nil)
|
||||||
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
|
|
||||||
|
// Insert a value that goes through one branch of the resolved subtree.
|
||||||
|
// This modifies path [1, ...] but leaves path [4, ...] unmodified.
|
||||||
|
if err := tr.Update([]byte{0x12}, bytes.Repeat([]byte("C"), 40)); err != nil {
|
||||||
|
t.Fatalf("Update failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Commit the trie. The NodeSet should be non-nil because we modified data.
|
||||||
|
_, nodes := tr.Commit(false)
|
||||||
|
if nodes == nil {
|
||||||
|
t.Fatal("expected non-nil NodeSet after modifying expired subtree")
|
||||||
|
}
|
||||||
|
|
||||||
|
// The resolved-but-unmodified sibling (path [4, 5]) should also be
|
||||||
|
// captured in the NodeSet, because markSubtreeDirty ensures all resolved
|
||||||
|
// nodes are dirty. Count the nodes to verify.
|
||||||
|
nodeCount := len(nodes.Nodes)
|
||||||
|
// We expect at least 3 nodes: the root, the modified branch, and the
|
||||||
|
// sibling branch. The exact count depends on trie structure.
|
||||||
|
if nodeCount < 3 {
|
||||||
|
t.Errorf("expected at least 3 nodes in NodeSet (root + modified + sibling), got %d", nodeCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMarkSubtreeDirty verifies that markSubtreeDirty correctly sets the dirty
|
||||||
|
// flag on all nodes in a subtree while preserving cached hashes.
|
||||||
|
func TestMarkSubtreeDirty(t *testing.T) {
|
||||||
|
// Build a small trie structure
|
||||||
|
leaf1 := &shortNode{Key: []byte{1, 16}, Val: valueNode("v1")}
|
||||||
|
leaf2 := &shortNode{Key: []byte{2, 16}, Val: valueNode("v2")}
|
||||||
|
branch := &fullNode{}
|
||||||
|
branch.Children[1] = leaf1
|
||||||
|
branch.Children[2] = leaf2
|
||||||
|
|
||||||
|
// Set hash but not dirty (as if loaded from DB)
|
||||||
|
branch.flags = nodeFlag{hash: hashNode("testhash"), dirty: false}
|
||||||
|
leaf1.flags = nodeFlag{hash: hashNode("hash1"), dirty: false}
|
||||||
|
leaf2.flags = nodeFlag{hash: hashNode("hash2"), dirty: false}
|
||||||
|
|
||||||
|
markSubtreeDirty(branch)
|
||||||
|
|
||||||
|
// All nodes should be dirty
|
||||||
|
if !branch.flags.dirty {
|
||||||
|
t.Error("branch should be dirty")
|
||||||
|
}
|
||||||
|
if !leaf1.flags.dirty {
|
||||||
|
t.Error("leaf1 should be dirty")
|
||||||
|
}
|
||||||
|
if !leaf2.flags.dirty {
|
||||||
|
t.Error("leaf2 should be dirty")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hashes should be preserved
|
||||||
|
if !bytes.Equal(branch.flags.hash, hashNode("testhash")) {
|
||||||
|
t.Error("branch hash should be preserved")
|
||||||
|
}
|
||||||
|
if !bytes.Equal(leaf1.flags.hash, hashNode("hash1")) {
|
||||||
|
t.Error("leaf1 hash should be preserved")
|
||||||
|
}
|
||||||
|
if !bytes.Equal(leaf2.flags.hash, hashNode("hash2")) {
|
||||||
|
t.Error("leaf2 hash should be preserved")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExpiredNodeGetNode(t *testing.T) {
|
||||||
|
records := []*archive.Record{
|
||||||
|
{Path: []byte{0x01, 0x02, 16}, Value: []byte("testvalue")},
|
||||||
|
}
|
||||||
|
offset, size, cleanup := setupTestArchive(t, records)
|
||||||
|
defer cleanup()
|
||||||
|
|
||||||
|
tr := NewEmpty(nil)
|
||||||
|
tr.root = &expiredNode{offset: offset, size: size}
|
||||||
|
|
||||||
_, _, err := tr.GetNode(hexToCompact([]byte{0x01, 0x02}))
|
_, _, err := tr.GetNode(hexToCompact([]byte{0x01, 0x02}))
|
||||||
if !resolverCalled {
|
|
||||||
t.Error("resolver was not called during GetNode")
|
|
||||||
}
|
|
||||||
if err != nil && err.Error() != "non-consensus node" {
|
if err != nil && err.Error() != "non-consensus node" {
|
||||||
t.Fatalf("unexpected error: %v", err)
|
t.Fatalf("unexpected error: %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ package trie
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
|
"encoding/binary"
|
||||||
"fmt"
|
"fmt"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
|
@ -97,6 +98,22 @@ func (h *hasher) hash(n node, force bool) []byte {
|
||||||
// hash nodes don't have children, so they're left as were
|
// hash nodes don't have children, so they're left as were
|
||||||
return n
|
return n
|
||||||
|
|
||||||
|
case *expiredNode:
|
||||||
|
// Return the original subtree hash that was cached when the
|
||||||
|
// expired node was decoded. The parent node references this
|
||||||
|
// hash, so we must return the same value to keep the Merkle
|
||||||
|
// root consistent.
|
||||||
|
if n.cachedHash != nil {
|
||||||
|
return n.cachedHash
|
||||||
|
}
|
||||||
|
// Fallback: hash the marker blob (should not happen in practice
|
||||||
|
// because decodeNodeUnsafe always provides the hash).
|
||||||
|
var buf [1 + 2*8]byte // 17 bytes
|
||||||
|
buf[0] = expiredNodeMarker
|
||||||
|
binary.BigEndian.PutUint64(buf[1:], n.offset)
|
||||||
|
binary.BigEndian.PutUint64(buf[9:], n.size)
|
||||||
|
return h.hashData(buf[:])
|
||||||
|
|
||||||
default:
|
default:
|
||||||
panic(fmt.Errorf("unexpected node type, %T", n))
|
panic(fmt.Errorf("unexpected node type, %T", n))
|
||||||
}
|
}
|
||||||
|
|
@ -214,6 +231,12 @@ func (h *hasher) proofHash(original node) []byte {
|
||||||
return bytes.Clone(h.encodeShortNode(n))
|
return bytes.Clone(h.encodeShortNode(n))
|
||||||
case *fullNode:
|
case *fullNode:
|
||||||
return bytes.Clone(h.encodeFullNode(n))
|
return bytes.Clone(h.encodeFullNode(n))
|
||||||
|
case *expiredNode:
|
||||||
|
var buf [1 + 2*8]byte
|
||||||
|
buf[0] = expiredNodeMarker
|
||||||
|
binary.BigEndian.PutUint64(buf[1:], n.offset)
|
||||||
|
binary.BigEndian.PutUint64(buf[9:], n.size)
|
||||||
|
return buf[:]
|
||||||
default:
|
default:
|
||||||
panic(fmt.Errorf("unexpected node type, %T", original))
|
panic(fmt.Errorf("unexpected node type, %T", original))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ import (
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
"github.com/ethereum/go-ethereum/crypto"
|
"github.com/ethereum/go-ethereum/crypto"
|
||||||
|
"github.com/ethereum/go-ethereum/log"
|
||||||
"github.com/ethereum/go-ethereum/rlp"
|
"github.com/ethereum/go-ethereum/rlp"
|
||||||
"github.com/ethereum/go-ethereum/trie/archive"
|
"github.com/ethereum/go-ethereum/trie/archive"
|
||||||
)
|
)
|
||||||
|
|
@ -166,7 +167,8 @@ func decodeNodeUnsafe(hash, buf []byte) (node, error) {
|
||||||
}
|
}
|
||||||
offset := binary.BigEndian.Uint64(buf[1:])
|
offset := binary.BigEndian.Uint64(buf[1:])
|
||||||
size := binary.BigEndian.Uint64(buf[1+archive.OffsetSize:])
|
size := binary.BigEndian.Uint64(buf[1+archive.OffsetSize:])
|
||||||
return &expiredNode{offset: offset, size: size, archiveResolver: archive.ArchivedNodeResolver}, nil
|
log.Debug("Decoded expired node", "offset", offset, "size", size, "hash", common.BytesToHash(hash))
|
||||||
|
return &expiredNode{offset: offset, size: size, cachedHash: hashNode(hash), archiveResolver: archive.ArchivedNodeResolver}, nil
|
||||||
}
|
}
|
||||||
elems, _, err := rlp.SplitList(buf)
|
elems, _, err := rlp.SplitList(buf)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ import (
|
||||||
"github.com/ethereum/go-ethereum/crypto"
|
"github.com/ethereum/go-ethereum/crypto"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/log"
|
"github.com/ethereum/go-ethereum/log"
|
||||||
|
"github.com/ethereum/go-ethereum/trie/archive"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Prove constructs a merkle proof for key. The result contains all encoded nodes
|
// Prove constructs a merkle proof for key. The result contains all encoded nodes
|
||||||
|
|
@ -78,6 +79,16 @@ func (t *Trie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error {
|
||||||
// clean cache or the database, they are all in their own
|
// clean cache or the database, they are all in their own
|
||||||
// copy and safe to use unsafe decoder.
|
// copy and safe to use unsafe decoder.
|
||||||
tn = mustDecodeNodeUnsafe(n, blob)
|
tn = mustDecodeNodeUnsafe(n, blob)
|
||||||
|
case *expiredNode:
|
||||||
|
records, err := archive.ArchivedNodeResolver(n.offset, n.size)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to resolve expired node in proof: %w", err)
|
||||||
|
}
|
||||||
|
resolved, err := archiveRecordsToNode(records)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("failed to rebuild expired node in proof: %w", err)
|
||||||
|
}
|
||||||
|
tn = resolved
|
||||||
default:
|
default:
|
||||||
panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
|
panic(fmt.Sprintf("%T: invalid node: %v", tn, tn))
|
||||||
}
|
}
|
||||||
|
|
@ -617,6 +628,8 @@ func get(tn node, key []byte, skipResolved bool) ([]byte, node) {
|
||||||
}
|
}
|
||||||
case hashNode:
|
case hashNode:
|
||||||
return key, n
|
return key, n
|
||||||
|
case *expiredNode:
|
||||||
|
return key, n
|
||||||
case nil:
|
case nil:
|
||||||
return key, nil
|
return key, nil
|
||||||
case valueNode:
|
case valueNode:
|
||||||
|
|
|
||||||
121
trie/trie.go
121
trie/trie.go
|
|
@ -230,32 +230,12 @@ func (t *Trie) get(origNode node, key []byte, pos int) (value []byte, newnode no
|
||||||
value, newnode, _, err := t.get(child, key, pos)
|
value, newnode, _, err := t.get(child, key, pos)
|
||||||
return value, newnode, true, err
|
return value, newnode, true, err
|
||||||
case *expiredNode:
|
case *expiredNode:
|
||||||
records, err := archive.ArchivedNodeResolver(n.offset, n.size)
|
log.Debug("Resolving expired node in get()", "owner", t.owner, "offset", n.offset, "size", n.size, "pos", pos)
|
||||||
if err != nil {
|
newnode, err := resolveExpiredNodeData(n)
|
||||||
return nil, n, false, fmt.Errorf("failed to resolve expired node: %w", err)
|
|
||||||
}
|
|
||||||
newnode, err := archiveRecordsToNode(records)
|
|
||||||
// alternative: don't rebuild, just find the value
|
|
||||||
// for _, record := range records {
|
|
||||||
// // make sure that the path up to the node matches
|
|
||||||
// if bytes.HasPrefix(key[pos:], record.Path) {
|
|
||||||
// resolved, err := decodeNodeUnsafe(nil, record.Value)
|
|
||||||
// if err != nil {
|
|
||||||
// fmt.Printf("%v %x\n", record.Path, record.Value)
|
|
||||||
// return nil, n, false, fmt.Errorf("failed to deserialize RLP node: %w", err)
|
|
||||||
// }
|
|
||||||
// if leaf, ok := resolved.(*shortNode); ok {
|
|
||||||
// // make sure that the key to the leaf also matches
|
|
||||||
// if bytes.Equal(key[pos+len(record.Path):], leaf.Key) {
|
|
||||||
// return leaf.Val.(valueNode), newnode, true, nil
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
// }
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, n, false, err
|
return nil, n, false, err
|
||||||
}
|
}
|
||||||
value, _, _, err = t.get(newnode, key, pos+1)
|
value, _, _, err = t.get(newnode, key, pos)
|
||||||
return value, newnode, true, err
|
return value, newnode, true, err
|
||||||
default:
|
default:
|
||||||
panic(fmt.Sprintf("%T: invalid node: %v", origNode, origNode))
|
panic(fmt.Sprintf("%T: invalid node: %v", origNode, origNode))
|
||||||
|
|
@ -392,12 +372,11 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod
|
||||||
return item, newnode, resolved + 1, err
|
return item, newnode, resolved + 1, err
|
||||||
|
|
||||||
case *expiredNode:
|
case *expiredNode:
|
||||||
records, err := archive.ArchivedNodeResolver(n.offset, n.size)
|
rn, err := resolveExpiredNodeData(n)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, n, 0, fmt.Errorf("failed to resolve expired node: %w", err)
|
return nil, n, 0, err
|
||||||
}
|
}
|
||||||
newnode, err := archiveRecordsToNode(records)
|
item, newnode, resolvedCount, err := t.getNode(rn, path, pos)
|
||||||
item, newnode, resolvedCount, err := t.getNode(newnode, path, pos)
|
|
||||||
return item, newnode, resolvedCount + 1, err
|
return item, newnode, resolvedCount + 1, err
|
||||||
|
|
||||||
default:
|
default:
|
||||||
|
|
@ -524,16 +503,16 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
|
||||||
return true, nn, nil
|
return true, nn, nil
|
||||||
|
|
||||||
case *expiredNode:
|
case *expiredNode:
|
||||||
records, err := archive.ArchivedNodeResolver(n.offset, n.size)
|
log.Info("Resolving expired node in insert()", "owner", t.owner, "offset", n.offset, "size", n.size)
|
||||||
|
rn, err := resolveExpiredNodeData(n)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, nil, fmt.Errorf("failed to resolve expired node: %w", err)
|
return false, nil, err
|
||||||
}
|
}
|
||||||
nn, err := archiveRecordsToNode(records)
|
dirty, nn, err := t.insert(rn, prefix, key, value)
|
||||||
if err != nil {
|
if !dirty || err != nil {
|
||||||
return false, nil, fmt.Errorf("failed to rebuild expired node from archive: %w", err)
|
return false, rn, err
|
||||||
}
|
}
|
||||||
dirty, nn, err := t.insert(nn, prefix, key, value)
|
return true, nn, nil
|
||||||
return dirty && err == nil, nn, err
|
|
||||||
|
|
||||||
default:
|
default:
|
||||||
panic(fmt.Sprintf("%T: invalid node: %v", n, n))
|
panic(fmt.Sprintf("%T: invalid node: %v", n, n))
|
||||||
|
|
@ -697,16 +676,16 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
|
||||||
return true, nn, nil
|
return true, nn, nil
|
||||||
|
|
||||||
case *expiredNode:
|
case *expiredNode:
|
||||||
records, err := archive.ArchivedNodeResolver(n.offset, n.size)
|
log.Info("Resolving expired node in delete()", "owner", t.owner, "offset", n.offset, "size", n.size)
|
||||||
|
rn, err := resolveExpiredNodeData(n)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, nil, fmt.Errorf("failed to resolve expired node: %w", err)
|
return false, nil, err
|
||||||
}
|
}
|
||||||
nn, err := archiveRecordsToNode(records)
|
dirty, nn, err := t.delete(rn, prefix, key)
|
||||||
if err != nil {
|
if !dirty || err != nil {
|
||||||
return false, nil, fmt.Errorf("failed to rebuild expired node from archive: %w", err)
|
return false, rn, err
|
||||||
}
|
}
|
||||||
dirty, _, err := t.delete(nn, prefix, key)
|
return true, nn, nil
|
||||||
return dirty && err == nil, nn, err
|
|
||||||
|
|
||||||
default:
|
default:
|
||||||
panic(fmt.Sprintf("%T: invalid node: %v (%v)", n, n, key))
|
panic(fmt.Sprintf("%T: invalid node: %v (%v)", n, n, key))
|
||||||
|
|
@ -742,7 +721,8 @@ func copyNode(n node) node {
|
||||||
return &expiredNode{
|
return &expiredNode{
|
||||||
offset: n.offset,
|
offset: n.offset,
|
||||||
size: n.size,
|
size: n.size,
|
||||||
archiveResolver: archive.ArchivedNodeResolver,
|
cachedHash: common.CopyBytes(n.cachedHash),
|
||||||
|
archiveResolver: n.archiveResolver,
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
panic(fmt.Sprintf("%T: unknown node type", n))
|
panic(fmt.Sprintf("%T: unknown node type", n))
|
||||||
|
|
@ -750,8 +730,11 @@ func copyNode(n node) node {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *Trie) resolve(n node, prefix []byte) (node, error) {
|
func (t *Trie) resolve(n node, prefix []byte) (node, error) {
|
||||||
if n, ok := n.(hashNode); ok {
|
switch n := n.(type) {
|
||||||
|
case hashNode:
|
||||||
return t.resolveAndTrack(n, prefix)
|
return t.resolveAndTrack(n, prefix)
|
||||||
|
case *expiredNode:
|
||||||
|
return resolveExpiredNodeData(n)
|
||||||
}
|
}
|
||||||
return n, nil
|
return n, nil
|
||||||
}
|
}
|
||||||
|
|
@ -862,6 +845,58 @@ func (t *Trie) Witness() map[string][]byte {
|
||||||
return t.prevalueTracer.Values()
|
return t.prevalueTracer.Values()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WalkStats holds statistics from a Walk traversal.
|
||||||
|
type WalkStats struct {
|
||||||
|
Leaves int // Number of leaf nodes visited
|
||||||
|
ExpiredResolved int // Number of expired nodes resolved from archive
|
||||||
|
}
|
||||||
|
|
||||||
|
// Walk recursively traverses the trie, resolving all nodes including
|
||||||
|
// hashNodes and expiredNodes. It calls fn for each leaf found.
|
||||||
|
// This triggers hash verification for expired nodes via cachedHash.
|
||||||
|
func (t *Trie) Walk(fn func(path []byte, value []byte) error) (WalkStats, error) {
|
||||||
|
return t.walk(t.root, nil, fn)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *Trie) walk(n node, path []byte, fn func([]byte, []byte) error) (WalkStats, error) {
|
||||||
|
switch n := n.(type) {
|
||||||
|
case *shortNode:
|
||||||
|
return t.walk(n.Val, append(append([]byte{}, path...), n.Key...), fn)
|
||||||
|
case *fullNode:
|
||||||
|
var stats WalkStats
|
||||||
|
for i, child := range n.Children[:16] {
|
||||||
|
if child != nil {
|
||||||
|
childStats, err := t.walk(child, append(append([]byte{}, path...), byte(i)), fn)
|
||||||
|
if err != nil {
|
||||||
|
return stats, err
|
||||||
|
}
|
||||||
|
stats.Leaves += childStats.Leaves
|
||||||
|
stats.ExpiredResolved += childStats.ExpiredResolved
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return stats, nil
|
||||||
|
case hashNode:
|
||||||
|
resolved, err := t.resolveAndTrack(n, path)
|
||||||
|
if err != nil {
|
||||||
|
return WalkStats{}, err
|
||||||
|
}
|
||||||
|
return t.walk(resolved, path, fn)
|
||||||
|
case *expiredNode:
|
||||||
|
resolved, err := resolveExpiredNodeData(n)
|
||||||
|
if err != nil {
|
||||||
|
return WalkStats{}, err
|
||||||
|
}
|
||||||
|
childStats, err := t.walk(resolved, path, fn)
|
||||||
|
childStats.ExpiredResolved++
|
||||||
|
return childStats, err
|
||||||
|
case valueNode:
|
||||||
|
return WalkStats{Leaves: 1}, fn(path, []byte(n))
|
||||||
|
case nil:
|
||||||
|
return WalkStats{}, nil
|
||||||
|
}
|
||||||
|
return WalkStats{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// reset drops the referenced root node and cleans all internal state.
|
// reset drops the referenced root node and cleans all internal state.
|
||||||
func (t *Trie) reset() {
|
func (t *Trie) reset() {
|
||||||
t.root = nil
|
t.root = nil
|
||||||
|
|
|
||||||
|
|
@ -399,6 +399,28 @@ func (db *Database) Disk() ethdb.Database {
|
||||||
return db.disk
|
return db.disk
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DiffHead returns the root hash of the topmost diff layer in pathdb.
|
||||||
|
// If there are no diff layers or the backend is not pathdb, it returns
|
||||||
|
// the zero hash and false.
|
||||||
|
func (db *Database) DiffHead() (common.Hash, bool) {
|
||||||
|
pdb, ok := db.backend.(*pathdb.Database)
|
||||||
|
if !ok {
|
||||||
|
return common.Hash{}, false
|
||||||
|
}
|
||||||
|
return pdb.DiffHead()
|
||||||
|
}
|
||||||
|
|
||||||
|
// DisableStateHistory closes and disables the state history freezer.
|
||||||
|
// This is used by the archiver to bypass state history writes during
|
||||||
|
// diff layer flushing when state history may have gaps.
|
||||||
|
func (db *Database) DisableStateHistory() {
|
||||||
|
pdb, ok := db.backend.(*pathdb.Database)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
pdb.DisableStateHistory()
|
||||||
|
}
|
||||||
|
|
||||||
// SnapshotCompleted returns the indicator if the snapshot is completed.
|
// SnapshotCompleted returns the indicator if the snapshot is completed.
|
||||||
func (db *Database) SnapshotCompleted() bool {
|
func (db *Database) SnapshotCompleted() bool {
|
||||||
pdb, ok := db.backend.(*pathdb.Database)
|
pdb, ok := db.backend.(*pathdb.Database)
|
||||||
|
|
|
||||||
|
|
@ -318,6 +318,30 @@ func (db *Database) Update(root common.Hash, parentRoot common.Hash, block uint6
|
||||||
return db.tree.cap(root, maxDiffLayers)
|
return db.tree.cap(root, maxDiffLayers)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DiffHead returns the root hash of the topmost diff layer. If there are no
|
||||||
|
// diff layers (only the disk layer), it returns the disk layer root and false.
|
||||||
|
func (db *Database) DiffHead() (common.Hash, bool) {
|
||||||
|
db.lock.RLock()
|
||||||
|
defer db.lock.RUnlock()
|
||||||
|
|
||||||
|
return db.tree.diffHead()
|
||||||
|
}
|
||||||
|
|
||||||
|
// DisableStateHistory closes and disables the state history freezer. This is
|
||||||
|
// used by the archiver to bypass state history writes during diff layer flushing,
|
||||||
|
// since the archiver only needs trie nodes committed to disk and state history
|
||||||
|
// may have gaps from unclean shutdowns that prevent sequential appends.
|
||||||
|
func (db *Database) DisableStateHistory() {
|
||||||
|
db.lock.Lock()
|
||||||
|
defer db.lock.Unlock()
|
||||||
|
|
||||||
|
if db.stateFreezer != nil {
|
||||||
|
db.stateFreezer.Close()
|
||||||
|
db.stateFreezer = nil
|
||||||
|
log.Info("Disabled state history freezer")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Commit traverses downwards the layer tree from a specified layer with the
|
// Commit traverses downwards the layer tree from a specified layer with the
|
||||||
// provided state root and all the layers below are flattened downwards. It
|
// provided state root and all the layers below are flattened downwards. It
|
||||||
// can be used alone and mostly for test purposes.
|
// can be used alone and mostly for test purposes.
|
||||||
|
|
|
||||||
|
|
@ -278,9 +278,17 @@ func truncateFromHead(store ethdb.AncientStore, typ historyType, nhead uint64) (
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
// Ensure that the truncation target falls within the valid range.
|
// Ensure that the truncation target falls within the valid range.
|
||||||
if ohead < nhead || nhead < otail {
|
if nhead < otail {
|
||||||
return 0, fmt.Errorf("%w, %s, tail: %d, head: %d, target: %d", errHeadTruncationOutOfRange, typ, otail, ohead, nhead)
|
return 0, fmt.Errorf("%w, %s, tail: %d, head: %d, target: %d", errHeadTruncationOutOfRange, typ, otail, ohead, nhead)
|
||||||
}
|
}
|
||||||
|
// If the target is ahead of the current head, there's nothing to truncate.
|
||||||
|
// This can happen after unclean shutdowns where the state history was not
|
||||||
|
// fully written.
|
||||||
|
if ohead < nhead {
|
||||||
|
log.Warn("State history shorter than target, nothing to truncate",
|
||||||
|
"type", typ.String(), "head", ohead, "target", nhead)
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
// Short circuit if nothing to truncate.
|
// Short circuit if nothing to truncate.
|
||||||
if ohead == nhead {
|
if ohead == nhead {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ import (
|
||||||
// of the referenced layer by themselves.
|
// of the referenced layer by themselves.
|
||||||
type layerTree struct {
|
type layerTree struct {
|
||||||
base *diskLayer
|
base *diskLayer
|
||||||
|
head common.Hash // Root hash of the topmost layer (diff or disk)
|
||||||
layers map[common.Hash]layer
|
layers map[common.Hash]layer
|
||||||
|
|
||||||
// descendants is a two-dimensional map where the keys represent
|
// descendants is a two-dimensional map where the keys represent
|
||||||
|
|
@ -59,6 +60,7 @@ func (tree *layerTree) init(head layer) {
|
||||||
defer tree.lock.Unlock()
|
defer tree.lock.Unlock()
|
||||||
|
|
||||||
current := head
|
current := head
|
||||||
|
tree.head = head.rootHash()
|
||||||
tree.layers = make(map[common.Hash]layer)
|
tree.layers = make(map[common.Hash]layer)
|
||||||
tree.descendants = make(map[common.Hash]map[common.Hash]struct{})
|
tree.descendants = make(map[common.Hash]map[common.Hash]struct{})
|
||||||
|
|
||||||
|
|
@ -76,6 +78,18 @@ func (tree *layerTree) init(head layer) {
|
||||||
tree.lookup = newLookup(head, tree.isDescendant)
|
tree.lookup = newLookup(head, tree.isDescendant)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// diffHead returns the root hash of the topmost diff layer. If there are no
|
||||||
|
// diff layers, returns the disk layer root and false.
|
||||||
|
func (tree *layerTree) diffHead() (common.Hash, bool) {
|
||||||
|
tree.lock.RLock()
|
||||||
|
defer tree.lock.RUnlock()
|
||||||
|
|
||||||
|
if _, ok := tree.layers[tree.head].(*diffLayer); ok {
|
||||||
|
return tree.head, true
|
||||||
|
}
|
||||||
|
return tree.base.rootHash(), false
|
||||||
|
}
|
||||||
|
|
||||||
// get retrieves a layer belonging to the given state root.
|
// get retrieves a layer belonging to the given state root.
|
||||||
func (tree *layerTree) get(root common.Hash) layer {
|
func (tree *layerTree) get(root common.Hash) layer {
|
||||||
tree.lock.RLock()
|
tree.lock.RLock()
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue