// Copyright 2021 The go-ethereum Authors // This file is part of go-ethereum. // // go-ethereum is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // go-ethereum is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with go-ethereum. If not, see . package main import ( "bytes" "fmt" "os" "os/signal" "path/filepath" "slices" "strconv" "strings" "syscall" "time" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/console/prompt" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" "github.com/olekukonko/tablewriter" "github.com/urfave/cli/v2" ) var ( removeStateDataFlag = &cli.BoolFlag{ Name: "remove.state", Usage: "If set, selects the state data for removal", } removeChainDataFlag = &cli.BoolFlag{ Name: "remove.chain", Usage: "If set, selects the state data for removal", } // Flags for the freezer truncation command dryRunFlag = &cli.BoolFlag{ Name: "dry-run", Usage: "Perform a dry run without actually truncating the freezer", } yesFlag = &cli.BoolFlag{ Name: "yes", Usage: "Skip confirmation prompt", } keepHeadersFlag = &cli.BoolFlag{ Name: "keep-headers", Usage: "Keep headers when pruning history (default: true)", Value: true, } useHardcodedMergeFlag = &cli.BoolFlag{ Name: "use-hardcoded-merge", Usage: "Use hardcoded merge block for known networks instead of detecting it", } batchSizeFlag = &cli.IntFlag{ Name: "batch-size", Usage: "Number of headers to process in each batch (default: 10000)", Value: 10000, } ) // Known merge block numbers for different networks var knownMergeBlocks = map[string]uint64{ "mainnet": 15537394, "sepolia": 1735371, "goerli": 7382818, "holesky": 0, // Holesky was launched post-merge } var ( removedbCommand = &cli.Command{ Action: removeDB, Name: "removedb", Usage: "Remove blockchain and state databases", ArgsUsage: "", Flags: slices.Concat(utils.DatabaseFlags, []cli.Flag{removeStateDataFlag, removeChainDataFlag}), Description: ` Remove blockchain and state databases`, } dbCommand = &cli.Command{ Name: "db", Usage: "Low level database operations", ArgsUsage: "", Subcommands: []*cli.Command{ dbInspectCmd, dbStatCmd, dbCompactCmd, dbGetCmd, dbDeleteCmd, dbPutCmd, dbGetSlotsCmd, dbDumpFreezerIndex, dbImportCmd, dbExportCmd, dbMetadataCmd, dbCheckStateContentCmd, dbInspectHistoryCmd, dbPruneHistoryCmd, dbTruncateFreezerCmd, }, } dbInspectCmd = &cli.Command{ Action: inspect, Name: "inspect", ArgsUsage: " ", Flags: slices.Concat([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabaseFlags), Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } dbCheckStateContentCmd = &cli.Command{ Action: checkStateContent, Name: "check-state-content", ArgsUsage: "", Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Usage: "Verify that state data is cryptographically correct", Description: `This command iterates the entire database for 32-byte keys, looking for rlp-encoded trie nodes. For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates a data corruption.`, } dbStatCmd = &cli.Command{ Action: dbStats, Name: "stats", Usage: "Print leveldb statistics", Flags: slices.Concat([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabaseFlags), } dbCompactCmd = &cli.Command{ Action: dbCompact, Name: "compact", Usage: "Compact leveldb database. WARNING: May take a very long time", Flags: slices.Concat([]cli.Flag{ utils.SyncModeFlag, utils.CacheFlag, utils.CacheDatabaseFlag, }, utils.NetworkFlags, utils.DatabaseFlags), Description: `This command performs a database compaction. WARNING: This operation may take a very long time to finish, and may cause database corruption if it is aborted during execution'!`, } dbGetCmd = &cli.Command{ Action: dbGet, Name: "get", Usage: "Show the value of a database key", ArgsUsage: "", Flags: slices.Concat([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabaseFlags), Description: "This command looks up the specified database key from the database.", } dbDeleteCmd = &cli.Command{ Action: dbDelete, Name: "delete", Usage: "Delete a database key (WARNING: may corrupt your database)", ArgsUsage: "", Flags: slices.Concat([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabaseFlags), Description: `This command deletes the specified database key from the database. WARNING: This is a low-level operation which may cause database corruption!`, } dbPutCmd = &cli.Command{ Action: dbPut, Name: "put", Usage: "Set the value of a database key (WARNING: may corrupt your database)", ArgsUsage: " ", Flags: slices.Concat([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabaseFlags), Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, } dbGetSlotsCmd = &cli.Command{ Action: dbDumpTrie, Name: "dumptrie", Usage: "Show the storage key/values of a given storage trie", ArgsUsage: " ", Flags: slices.Concat([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabaseFlags), Description: "This command looks up the specified database key from the database.", } dbDumpFreezerIndex = &cli.Command{ Action: freezerInspect, Name: "freezer-index", Usage: "Dump out the index of a specific freezer table", ArgsUsage: " ", Flags: slices.Concat([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabaseFlags), Description: "This command displays information about the freezer index.", } dbImportCmd = &cli.Command{ Action: importLDBdata, Name: "import", Usage: "Imports leveldb-data from an exported RLP dump.", ArgsUsage: " has .gz suffix, gzip compression will be used.", ArgsUsage: " ", Flags: slices.Concat([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabaseFlags), Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.", } dbMetadataCmd = &cli.Command{ Action: showMetaData, Name: "metadata", Usage: "Shows metadata about the chain status.", Flags: slices.Concat([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabaseFlags), Description: "Shows metadata about the chain status.", } dbInspectHistoryCmd = &cli.Command{ Action: inspectHistory, Name: "inspect-history", Usage: "Inspect the state history within block range", ArgsUsage: "
[OPTIONAL ]", Flags: slices.Concat([]cli.Flag{ utils.SyncModeFlag, &cli.Uint64Flag{ Name: "start", Usage: "block number of the range start, zero means earliest history", }, &cli.Uint64Flag{ Name: "end", Usage: "block number of the range end(included), zero means latest history", }, &cli.BoolFlag{ Name: "raw", Usage: "display the decoded raw state value (otherwise shows rlp-encoded value)", }, }, utils.NetworkFlags, utils.DatabaseFlags), Description: "This command queries the history of the account or storage slot within the specified block range", } dbPruneHistoryCmd = &cli.Command{ Action: pruneHistory, Name: "prune-history", Usage: "Prune pre-merge history from the freezer", ArgsUsage: "", Flags: []cli.Flag{ dryRunFlag, yesFlag, keepHeadersFlag, }, Description: `This command prunes historical chain data before the merge block. The merge block is identified by the first block with zero difficulty. By default, headers are kept but bodies are pruned. This allows the chain to maintain its integrity while significantly reducing disk usage. If you want to prune everything including headers, use --keep-headers=false. `, } dbTruncateFreezerCmd = &cli.Command{ Action: truncateFreezer, Name: "truncate-freezer", Usage: "Truncate the freezer at the merge block, keeping headers but removing bodies", ArgsUsage: "", Flags: slices.Concat([]cli.Flag{ dryRunFlag, yesFlag, keepHeadersFlag, useHardcodedMergeFlag, batchSizeFlag, }, utils.NetworkFlags, utils.DatabaseFlags), Description: ` This command truncates the freezer at the merge block, keeping headers but removing bodies. This can significantly reduce disk space usage for nodes that don't need pre-merge block bodies. The command will: 1. Find the merge block (first block with zero difficulty) 2. Create a temporary copy of the headers and hashes up to the merge block 3. Truncate all freezer tables at the merge block 4. Re-insert the headers and hashes from the temporary copy WARNING: This operation cannot be undone. Make sure you have a backup if you might need the removed data in the future.`, } ) func removeDB(ctx *cli.Context) error { stack, config := makeConfigNode(ctx) // Resolve folder paths. var ( rootDir = stack.ResolvePath("chaindata") ancientDir = config.Eth.DatabaseFreezer ) switch { case ancientDir == "": ancientDir = filepath.Join(stack.ResolvePath("chaindata"), "ancient") case !filepath.IsAbs(ancientDir): ancientDir = config.Node.ResolvePath(ancientDir) } // Delete state data statePaths := []string{ rootDir, filepath.Join(ancientDir, rawdb.MerkleStateFreezerName), filepath.Join(ancientDir, rawdb.VerkleStateFreezerName), } confirmAndRemoveDB(statePaths, "state data", ctx, removeStateDataFlag.Name) // Delete ancient chain chainPaths := []string{filepath.Join( ancientDir, rawdb.ChainFreezerName, )} confirmAndRemoveDB(chainPaths, "ancient chain", ctx, removeChainDataFlag.Name) return nil } // removeFolder deletes all files (not folders) inside the directory 'dir' (but // not files in subfolders). func removeFolder(dir string) { filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { // If we're at the top level folder, recurse into if path == dir { return nil } // Delete all the files, but not subfolders if !info.IsDir() { os.Remove(path) return nil } return filepath.SkipDir }) } // confirmAndRemoveDB prompts the user for a last confirmation and removes the // list of folders if accepted. func confirmAndRemoveDB(paths []string, kind string, ctx *cli.Context, removeFlagName string) { var ( confirm bool err error ) msg := fmt.Sprintf("Location(s) of '%s': \n", kind) for _, path := range paths { msg += fmt.Sprintf("\t- %s\n", path) } fmt.Println(msg) if ctx.IsSet(removeFlagName) { confirm = ctx.Bool(removeFlagName) if confirm { fmt.Printf("Remove '%s'? [y/n] y\n", kind) } else { fmt.Printf("Remove '%s'? [y/n] n\n", kind) } } else { confirm, err = prompt.Stdin.PromptConfirm(fmt.Sprintf("Remove '%s'?", kind)) } switch { case err != nil: utils.Fatalf("%v", err) case !confirm: log.Info("Database deletion skipped", "kind", kind, "paths", paths) default: var ( deleted []string start = time.Now() ) for _, path := range paths { if common.FileExist(path) { removeFolder(path) deleted = append(deleted, path) } else { log.Info("Folder is not existent", "path", path) } } log.Info("Database successfully deleted", "kind", kind, "paths", deleted, "elapsed", common.PrettyDuration(time.Since(start))) } } func inspect(ctx *cli.Context) error { var ( prefix []byte start []byte ) if ctx.NArg() > 2 { return fmt.Errorf("max 2 arguments: %v", ctx.Command.ArgsUsage) } if ctx.NArg() >= 1 { if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil { return fmt.Errorf("failed to hex-decode 'prefix': %v", err) } else { prefix = d } } if ctx.NArg() >= 2 { if d, err := hexutil.Decode(ctx.Args().Get(1)); err != nil { return fmt.Errorf("failed to hex-decode 'start': %v", err) } else { start = d } } stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() return rawdb.InspectDatabase(db, prefix, start) } func checkStateContent(ctx *cli.Context) error { var ( prefix []byte start []byte ) if ctx.NArg() > 1 { return fmt.Errorf("max 1 argument: %v", ctx.Command.ArgsUsage) } if ctx.NArg() > 0 { if d, err := hexutil.Decode(ctx.Args().First()); err != nil { return fmt.Errorf("failed to hex-decode 'start': %v", err) } else { start = d } } stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() var ( it = rawdb.NewKeyLengthIterator(db.NewIterator(prefix, start), 32) hasher = crypto.NewKeccakState() got = make([]byte, 32) errs int count int startTime = time.Now() lastLog = time.Now() ) for it.Next() { count++ k := it.Key() v := it.Value() hasher.Reset() hasher.Write(v) hasher.Read(got) if !bytes.Equal(k, got) { errs++ fmt.Printf("Error at %#x\n", k) fmt.Printf(" Hash: %#x\n", got) fmt.Printf(" Data: %#x\n", v) } if time.Since(lastLog) > 8*time.Second { log.Info("Iterating the database", "at", fmt.Sprintf("%#x", k), "elapsed", common.PrettyDuration(time.Since(startTime))) lastLog = time.Now() } } if err := it.Error(); err != nil { return err } log.Info("Iterated the state content", "errors", errs, "items", count) return nil } func showDBStats(db ethdb.KeyValueStater) { stats, err := db.Stat() if err != nil { log.Warn("Failed to read database stats", "error", err) return } fmt.Println(stats) } func dbStats(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() showDBStats(db) return nil } func dbCompact(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, false) defer db.Close() log.Info("Stats before compaction") showDBStats(db) log.Info("Triggering compaction") if err := db.Compact(nil, nil); err != nil { log.Info("Compact err", "error", err) return err } log.Info("Stats after compaction") showDBStats(db) return nil } // dbGet shows the value of a given database key func dbGet(ctx *cli.Context) error { if ctx.NArg() != 1 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() key, err := common.ParseHexOrString(ctx.Args().Get(0)) if err != nil { log.Info("Could not decode the key", "error", err) return err } data, err := db.Get(key) if err != nil { log.Info("Get operation failed", "key", fmt.Sprintf("%#x", key), "error", err) return err } fmt.Printf("key %#x: %#x\n", key, data) return nil } // dbDelete deletes a key from the database func dbDelete(ctx *cli.Context) error { if ctx.NArg() != 1 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, false) defer db.Close() key, err := common.ParseHexOrString(ctx.Args().Get(0)) if err != nil { log.Info("Could not decode the key", "error", err) return err } data, err := db.Get(key) if err == nil { fmt.Printf("Previous value: %#x\n", data) } if err = db.Delete(key); err != nil { log.Info("Delete operation returned an error", "key", fmt.Sprintf("%#x", key), "error", err) return err } return nil } // dbPut overwrite a value in the database func dbPut(ctx *cli.Context) error { if ctx.NArg() != 2 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, false) defer db.Close() var ( key []byte value []byte data []byte err error ) key, err = common.ParseHexOrString(ctx.Args().Get(0)) if err != nil { log.Info("Could not decode the key", "error", err) return err } value, err = hexutil.Decode(ctx.Args().Get(1)) if err != nil { log.Info("Could not decode the value", "error", err) return err } data, err = db.Get(key) if err == nil { fmt.Printf("Previous value: %#x\n", data) } return db.Put(key, value) } // dbDumpTrie shows the key-value slots of a given storage trie func dbDumpTrie(ctx *cli.Context) error { if ctx.NArg() < 3 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() triedb := utils.MakeTrieDatabase(ctx, db, false, true, false) defer triedb.Close() var ( state []byte storage []byte account []byte start []byte max = int64(-1) err error ) if state, err = hexutil.Decode(ctx.Args().Get(0)); err != nil { log.Info("Could not decode the state root", "error", err) return err } if account, err = hexutil.Decode(ctx.Args().Get(1)); err != nil { log.Info("Could not decode the account hash", "error", err) return err } if storage, err = hexutil.Decode(ctx.Args().Get(2)); err != nil { log.Info("Could not decode the storage trie root", "error", err) return err } if ctx.NArg() > 3 { if start, err = hexutil.Decode(ctx.Args().Get(3)); err != nil { log.Info("Could not decode the seek position", "error", err) return err } } if ctx.NArg() > 4 { if max, err = strconv.ParseInt(ctx.Args().Get(4), 10, 64); err != nil { log.Info("Could not decode the max count", "error", err) return err } } id := trie.StorageTrieID(common.BytesToHash(state), common.BytesToHash(account), common.BytesToHash(storage)) theTrie, err := trie.New(id, triedb) if err != nil { return err } trieIt, err := theTrie.NodeIterator(start) if err != nil { return err } var count int64 it := trie.NewIterator(trieIt) for it.Next() { if max > 0 && count == max { fmt.Printf("Exiting after %d values\n", count) break } fmt.Printf(" %d. key %#x: %#x\n", count, it.Key, it.Value) count++ } return it.Err } func freezerInspect(ctx *cli.Context) error { if ctx.NArg() < 4 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } var ( freezer = ctx.Args().Get(0) table = ctx.Args().Get(1) ) start, err := strconv.ParseInt(ctx.Args().Get(2), 10, 64) if err != nil { log.Info("Could not read start-param", "err", err) return err } end, err := strconv.ParseInt(ctx.Args().Get(3), 10, 64) if err != nil { log.Info("Could not read count param", "err", err) return err } stack, _ := makeConfigNode(ctx) ancient := stack.ResolveAncient("chaindata", ctx.String(utils.AncientFlag.Name)) stack.Close() return rawdb.InspectFreezerTable(ancient, freezer, table, start, end) } func importLDBdata(ctx *cli.Context) error { start := 0 switch ctx.NArg() { case 1: break case 2: s, err := strconv.Atoi(ctx.Args().Get(1)) if err != nil { return fmt.Errorf("second arg must be an integer: %v", err) } start = s default: return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } var ( fName = ctx.Args().Get(0) stack, _ = makeConfigNode(ctx) interrupt = make(chan os.Signal, 1) stop = make(chan struct{}) ) defer stack.Close() signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(interrupt) defer close(interrupt) go func() { if _, ok := <-interrupt; ok { log.Info("Interrupted during ldb import, stopping at next batch") } close(stop) }() db := utils.MakeChainDatabase(ctx, stack, false) defer db.Close() return utils.ImportLDBData(db, fName, int64(start), stop) } type preimageIterator struct { iter ethdb.Iterator } func (iter *preimageIterator) Next() (byte, []byte, []byte, bool) { for iter.iter.Next() { key := iter.iter.Key() if bytes.HasPrefix(key, rawdb.PreimagePrefix) && len(key) == (len(rawdb.PreimagePrefix)+common.HashLength) { return utils.OpBatchAdd, key, iter.iter.Value(), true } } return 0, nil, nil, false } func (iter *preimageIterator) Release() { iter.iter.Release() } type snapshotIterator struct { init bool account ethdb.Iterator storage ethdb.Iterator } func (iter *snapshotIterator) Next() (byte, []byte, []byte, bool) { if !iter.init { iter.init = true return utils.OpBatchDel, rawdb.SnapshotRootKey, nil, true } for iter.account.Next() { key := iter.account.Key() if bytes.HasPrefix(key, rawdb.SnapshotAccountPrefix) && len(key) == (len(rawdb.SnapshotAccountPrefix)+common.HashLength) { return utils.OpBatchAdd, key, iter.account.Value(), true } } for iter.storage.Next() { key := iter.storage.Key() if bytes.HasPrefix(key, rawdb.SnapshotStoragePrefix) && len(key) == (len(rawdb.SnapshotStoragePrefix)+2*common.HashLength) { return utils.OpBatchAdd, key, iter.storage.Value(), true } } return 0, nil, nil, false } func (iter *snapshotIterator) Release() { iter.account.Release() iter.storage.Release() } // chainExporters defines the export scheme for all exportable chain data. var chainExporters = map[string]func(db ethdb.Database) utils.ChainDataIterator{ "preimage": func(db ethdb.Database) utils.ChainDataIterator { iter := db.NewIterator(rawdb.PreimagePrefix, nil) return &preimageIterator{iter: iter} }, "snapshot": func(db ethdb.Database) utils.ChainDataIterator { account := db.NewIterator(rawdb.SnapshotAccountPrefix, nil) storage := db.NewIterator(rawdb.SnapshotStoragePrefix, nil) return &snapshotIterator{account: account, storage: storage} }, } func exportChaindata(ctx *cli.Context) error { if ctx.NArg() < 2 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } // Parse the required chain data type, make sure it's supported. kind := ctx.Args().Get(0) kind = strings.ToLower(strings.Trim(kind, " ")) exporter, ok := chainExporters[kind] if !ok { var kinds []string for kind := range chainExporters { kinds = append(kinds, kind) } return fmt.Errorf("invalid data type %s, supported types: %s", kind, strings.Join(kinds, ", ")) } var ( stack, _ = makeConfigNode(ctx) interrupt = make(chan os.Signal, 1) stop = make(chan struct{}) ) defer stack.Close() signal.Notify(interrupt, syscall.SIGINT, syscall.SIGTERM) defer signal.Stop(interrupt) defer close(interrupt) go func() { if _, ok := <-interrupt; ok { log.Info("Interrupted during db export, stopping at next batch") } close(stop) }() db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop) } func showMetaData(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() ancients, err := db.Ancients() if err != nil { fmt.Fprintf(os.Stderr, "Error accessing ancients: %v", err) } data := rawdb.ReadChainMetadata(db) data = append(data, []string{"frozen", fmt.Sprintf("%d items", ancients)}) data = append(data, []string{"snapshotGenerator", snapshot.ParseGeneratorStatus(rawdb.ReadSnapshotGenerator(db))}) if b := rawdb.ReadHeadBlock(db); b != nil { data = append(data, []string{"headBlock.Hash", fmt.Sprintf("%v", b.Hash())}) data = append(data, []string{"headBlock.Root", fmt.Sprintf("%v", b.Root())}) data = append(data, []string{"headBlock.Number", fmt.Sprintf("%d (%#x)", b.Number(), b.Number())}) } if h := rawdb.ReadHeadHeader(db); h != nil { data = append(data, []string{"headHeader.Hash", fmt.Sprintf("%v", h.Hash())}) data = append(data, []string{"headHeader.Root", fmt.Sprintf("%v", h.Root)}) data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (%#x)", h.Number, h.Number)}) } table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Field", "Value"}) table.AppendBulk(data) table.Render() return nil } func inspectAccount(db *triedb.Database, start uint64, end uint64, address common.Address, raw bool) error { stats, err := db.AccountHistory(address, start, end) if err != nil { return err } fmt.Printf("Account history:\n\taddress: %s\n\tblockrange: [#%d-#%d]\n", address.Hex(), stats.Start, stats.End) from := stats.Start for i := 0; i < len(stats.Blocks); i++ { var content string if len(stats.Origins[i]) == 0 { content = "" } else { if !raw { content = fmt.Sprintf("%#x", stats.Origins[i]) } else { account := new(types.SlimAccount) if err := rlp.DecodeBytes(stats.Origins[i], account); err != nil { panic(err) } code := "" if len(account.CodeHash) > 0 { code = fmt.Sprintf("%#x", account.CodeHash) } root := "" if len(account.Root) > 0 { root = fmt.Sprintf("%#x", account.Root) } content = fmt.Sprintf("nonce: %d, balance: %d, codeHash: %s, root: %s", account.Nonce, account.Balance, code, root) } } fmt.Printf("#%d - #%d: %s\n", from, stats.Blocks[i], content) from = stats.Blocks[i] } return nil } func inspectStorage(db *triedb.Database, start uint64, end uint64, address common.Address, slot common.Hash, raw bool) error { // The hash of storage slot key is utilized in the history // rather than the raw slot key, make the conversion. stats, err := db.StorageHistory(address, slot, start, end) if err != nil { return err } fmt.Printf("Storage history:\n\taddress: %s\n\tslot: %s\n\tblockrange: [#%d-#%d]\n", address.Hex(), slot.Hex(), stats.Start, stats.End) from := stats.Start for i := 0; i < len(stats.Blocks); i++ { var content string if len(stats.Origins[i]) == 0 { content = "" } else { if !raw { content = fmt.Sprintf("%#x", stats.Origins[i]) } else { _, data, _, err := rlp.Split(stats.Origins[i]) if err != nil { fmt.Printf("Failed to decode storage slot, %v", err) return err } content = fmt.Sprintf("%#x", data) } } fmt.Printf("#%d - #%d: %s\n", from, stats.Blocks[i], content) from = stats.Blocks[i] } return nil } func inspectHistory(ctx *cli.Context) error { if ctx.NArg() == 0 || ctx.NArg() > 2 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } var ( address common.Address slot common.Hash ) if err := address.UnmarshalText([]byte(ctx.Args().Get(0))); err != nil { return err } if ctx.NArg() > 1 { if err := slot.UnmarshalText([]byte(ctx.Args().Get(1))); err != nil { return err } } // Load the databases. stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() triedb := utils.MakeTrieDatabase(ctx, db, false, false, false) defer triedb.Close() var ( err error start uint64 // the id of first history object to query end uint64 // the id (included) of last history object to query ) // State histories are identified by state ID rather than block number. // To address this, load the corresponding block header and perform the // conversion by this function. blockToID := func(blockNumber uint64) (uint64, error) { header := rawdb.ReadHeader(db, rawdb.ReadCanonicalHash(db, blockNumber), blockNumber) if header == nil { return 0, fmt.Errorf("block #%d is not existent", blockNumber) } id := rawdb.ReadStateID(db, header.Root) if id == nil { first, last, err := triedb.HistoryRange() if err == nil { return 0, fmt.Errorf("history of block #%d is not existent, available history range: [#%d-#%d]", blockNumber, first, last) } return 0, fmt.Errorf("history of block #%d is not existent", blockNumber) } return *id, nil } // Parse the starting block number for inspection. startNumber := ctx.Uint64("start") if startNumber != 0 { start, err = blockToID(startNumber) if err != nil { return err } } // Parse the ending block number for inspection. endBlock := ctx.Uint64("end") if endBlock != 0 { end, err = blockToID(endBlock) if err != nil { return err } } // Inspect the state history. if slot == (common.Hash{}) { return inspectAccount(triedb, start, end, address, ctx.Bool("raw")) } return inspectStorage(triedb, start, end, address, slot, ctx.Bool("raw")) } // pruneHistory implements the 'db prune-history' command, truncating the freezer // at the merge block. func pruneHistory(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, false) defer db.Close() // Find the merge block (first block with zero difficulty) log.Info("Looking for the merge block...") // Get the current head block headHash := rawdb.ReadHeadBlockHash(db) if headHash == (common.Hash{}) { return fmt.Errorf("chain head not found") } headNumber := rawdb.ReadHeaderNumber(db, headHash) if headNumber == nil { return fmt.Errorf("header number of head block not found") } var ( low = uint64(0) high = *headNumber mergeBlock = uint64(0) found = false ) log.Info("Searching for merge block using binary search", "head", *headNumber) for low <= high { mid := (low + high) / 2 header := rawdb.ReadHeader(db, rawdb.ReadCanonicalHash(db, mid), mid) if header == nil { return fmt.Errorf("header %d not found", mid) } if header.Difficulty.Sign() == 0 { // This is a post-merge block, look earlier high = mid - 1 mergeBlock = mid found = true } else { // This is a pre-merge block, look later low = mid + 1 } } if !found { return fmt.Errorf("merge block not found, chain may not have transitioned to PoS yet") } // The merge block is the first block with zero difficulty log.Info("Found merge block", "number", mergeBlock) // Check if we're in dry-run mode if ctx.Bool("dry-run") { log.Info("Dry run completed, no data was pruned", "mergeBlock", mergeBlock) return nil } // Ask for confirmation before proceeding fmt.Printf("You are about to prune all pre-merge history before block %d.\n", mergeBlock) fmt.Println("This operation cannot be undone and will permanently delete data.") fmt.Println("Make sure you have a backup if you might need this data in the future.") if !ctx.Bool("yes") { confirm, err := prompt.Stdin.PromptConfirm("Do you want to continue?") if err != nil { return err } if !confirm { return nil } } // Determine which tables to prune keepHeaders := ctx.Bool("keep-headers") // Get the freezer instance ancientDb, ok := db.(ethdb.AncientReader) if !ok { return fmt.Errorf("database doesn't support ancient storage") } // Get the first and last items in the freezer first, err := ancientDb.Ancients() if err != nil { return fmt.Errorf("failed to get ancients count: %v", err) } if first == 0 { log.Info("No ancient data to prune") return nil } // Ensure the merge block is in the freezer if mergeBlock < first { log.Info("Merge block is not in the freezer, nothing to prune", "mergeBlock", mergeBlock, "freezerStart", first) return nil } // Truncate the freezer tables if keepHeaders { log.Info("Pruning bodies but keeping headers", "mergeBlock", mergeBlock) // We can't truncate individual tables with the current API // Instead, we need to read the headers at the merge block and re-insert them // after truncating everything // First, read all headers up to the merge block log.Info("Reading headers to preserve them", "count", mergeBlock) headers := make([][]byte, mergeBlock) hashes := make([][]byte, mergeBlock) var err error for i := uint64(0); i < mergeBlock; i++ { headers[i], err = db.(ethdb.AncientReader).Ancient(rawdb.ChainFreezerHeaderTable, i) if err != nil { return fmt.Errorf("failed to read header %d: %v", i, err) } hashes[i], err = db.(ethdb.AncientReader).Ancient(rawdb.ChainFreezerHashTable, i) if err != nil { return fmt.Errorf("failed to read hash %d: %v", i, err) } } // Truncate all tables log.Info("Truncating all tables", "mergeBlock", mergeBlock) if err := truncateAncientStore(db, mergeBlock); err != nil { return fmt.Errorf("failed to truncate ancient store: %v", err) } // Re-insert the headers and hashes log.Info("Re-inserting headers", "count", len(headers)) freezerDb := db.(ethdb.AncientStore) _, err = freezerDb.ModifyAncients(func(op ethdb.AncientWriteOp) error { for i := uint64(0); i < mergeBlock; i++ { if err := op.AppendRaw(rawdb.ChainFreezerHeaderTable, i, headers[i]); err != nil { return fmt.Errorf("failed to re-insert header %d: %v", i, err) } if err := op.AppendRaw(rawdb.ChainFreezerHashTable, i, hashes[i]); err != nil { return fmt.Errorf("failed to re-insert hash %d: %v", i, err) } } return nil }) if err != nil { return fmt.Errorf("failed to re-insert headers: %v", err) } } else { log.Info("Pruning all pre-merge history", "mergeBlock", mergeBlock) // Truncate all tables if err := truncateAncientStore(db, mergeBlock); err != nil { return fmt.Errorf("failed to truncate ancient store: %v", err) } } log.Info("Successfully pruned pre-merge history", "mergeBlock", mergeBlock) return nil } // truncateAncientStore truncates all tables in the ancient store to the given block number func truncateAncientStore(db ethdb.Database, blockNum uint64) error { freezerDb, ok := db.(ethdb.AncientStore) if !ok { return fmt.Errorf("database doesn't support ancient writing") } // Get the current number of items in the store ancients, err := freezerDb.Ancients() if err != nil { return fmt.Errorf("failed to get ancients count: %v", err) } if blockNum >= ancients { log.Info("No need to truncate ancient store", "blockNum", blockNum, "ancients", ancients) return nil } log.Info("Truncating ancient store", "from", 0, "to", blockNum) // Truncate the head to the merge block _, err = freezerDb.TruncateHead(blockNum) if err != nil { return fmt.Errorf("failed to truncate head: %v", err) } return nil } // truncateFreezer implements the 'db truncate-freezer' command, truncating the freezer // at the merge block but keeping headers. func truncateFreezer(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() db := utils.MakeChainDatabase(ctx, stack, false) defer db.Close() // Find the merge block (first block with zero difficulty) log.Info("Looking for the merge block...") mergeBlock, err := findMergeBlock(ctx, db) if err != nil { return err } // Check if we're in dry-run mode if ctx.Bool(dryRunFlag.Name) { log.Info("Dry run completed, no data was pruned", "mergeBlock", mergeBlock) return nil } // Ask for confirmation before proceeding fmt.Printf("You are about to truncate the freezer at block %d, keeping headers but removing bodies.\n", mergeBlock) fmt.Println("This operation cannot be undone and will permanently delete data.") fmt.Println("Make sure you have a backup if you might need this data in the future.") if !ctx.Bool(yesFlag.Name) { confirm, err := prompt.Stdin.PromptConfirm("Do you want to continue?") if err != nil { return err } if !confirm { return nil } } // Get the freezer instance ancientDb, ok := db.(ethdb.AncientReader) if !ok { return fmt.Errorf("database doesn't support ancient storage") } // Get the first and last items in the freezer first, err := ancientDb.Ancients() if err != nil { return fmt.Errorf("failed to get ancients count: %v", err) } if first == 0 { log.Info("No ancient data to prune") return nil } // Ensure the merge block is in the freezer if mergeBlock < first { log.Info("Merge block is not in the freezer, nothing to prune", "mergeBlock", mergeBlock, "freezerStart", first) return nil } // Create a temporary directory for the headers freezer tmpDir, err := os.MkdirTemp("", "geth-headers-freezer-*") if err != nil { return fmt.Errorf("failed to create temporary directory: %v", err) } defer os.RemoveAll(tmpDir) log.Info("Created temporary directory for headers", "path", tmpDir) // Create a new freezer for headers and hashes headersFreezer, err := rawdb.NewFreezer(tmpDir, "headers", false, 2*1000*1000*1000, map[string]bool{ rawdb.ChainFreezerHeaderTable: false, rawdb.ChainFreezerHashTable: false, }) if err != nil { return fmt.Errorf("failed to create headers freezer: %v", err) } defer headersFreezer.Close() // Get batch size from flag batchSize := uint64(ctx.Int(batchSizeFlag.Name)) log.Info("Using batch size for processing", "batchSize", batchSize) // Copy headers and hashes to the temporary freezer in batches log.Info("Copying headers to temporary freezer", "count", mergeBlock) if err := extractHeaders(ancientDb, headersFreezer, mergeBlock, batchSize); err != nil { return err } // Truncate all tables in the original freezer log.Info("Truncating all tables in the original freezer", "mergeBlock", mergeBlock) if err := truncateAncientStore(db, 0); err != nil { return fmt.Errorf("failed to truncate ancient store: %v", err) } // Copy headers and hashes back from the temporary freezer in batches log.Info("Copying headers back to the main freezer", "count", mergeBlock) freezerDb := db.(ethdb.AncientStore) if err := reinsertHeaders(freezerDb, headersFreezer, mergeBlock, batchSize); err != nil { return err } log.Info("Successfully truncated freezer, keeping headers but removing bodies", "mergeBlock", mergeBlock) return nil } // extractHeaders copies headers from the main freezer to a temporary freezer in batches func extractHeaders(ancientDb ethdb.AncientReader, headersFreezer *rawdb.Freezer, mergeBlock, batchSize uint64) error { // Create a progress reporter log.Info("Starting to copy headers to temporary freezer", "count", mergeBlock) for i := uint64(0); i < mergeBlock; i += batchSize { end := i + batchSize if end > mergeBlock { end = mergeBlock } // Log progress as percentage progress := float64(end) / float64(mergeBlock) * 100 log.Info("Processing batch of headers", "from", i, "to", end, "progress", fmt.Sprintf("%.2f%%", progress)) // Process this batch _, err := headersFreezer.ModifyAncients(func(op ethdb.AncientWriteOp) error { for j := i; j < end; j++ { // Read header and hash headerBytes, err := ancientDb.Ancient(rawdb.ChainFreezerHeaderTable, j) if err != nil { return fmt.Errorf("failed to read header %d: %w", j, err) } hashBytes, err := ancientDb.Ancient(rawdb.ChainFreezerHashTable, j) if err != nil { return fmt.Errorf("failed to read hash %d: %w", j, err) } // Write to temporary freezer if err := op.AppendRaw(rawdb.ChainFreezerHeaderTable, j, headerBytes); err != nil { return fmt.Errorf("failed to write header %d: %w", j, err) } if err := op.AppendRaw(rawdb.ChainFreezerHashTable, j, hashBytes); err != nil { return fmt.Errorf("failed to write hash %d: %w", j, err) } } return nil }) if err != nil { return fmt.Errorf("failed to copy headers batch: %w", err) } // Update progress } return nil } // reinsertHeaders copies headers from the temporary freezer back to the main freezer in batches func reinsertHeaders(freezerDb ethdb.AncientStore, headersFreezer *rawdb.Freezer, mergeBlock, batchSize uint64) error { // Create a progress reporter for i := uint64(0); i < mergeBlock; i += batchSize { end := i + batchSize if end > mergeBlock { end = mergeBlock } log.Info("Processing batch of headers", "from", i, "to", end) _, err := freezerDb.ModifyAncients(func(op ethdb.AncientWriteOp) error { for j := i; j < end; j++ { log.Info("Completed copying headers back to main freezer", "count", mergeBlock) // Read from temporary freezer headerBytes, err := headersFreezer.Ancient(rawdb.ChainFreezerHeaderTable, j) if err != nil { return fmt.Errorf("failed to read header %d from temp freezer: %w", j, err) } hashBytes, err := headersFreezer.Ancient(rawdb.ChainFreezerHashTable, j) if err != nil { return fmt.Errorf("failed to read hash %d from temp freezer: %w", j, err) } // Write back to main freezer if err := op.AppendRaw(rawdb.ChainFreezerHeaderTable, j, headerBytes); err != nil { return fmt.Errorf("failed to write header %d to main freezer: %w", j, err) } if err := op.AppendRaw(rawdb.ChainFreezerHashTable, j, hashBytes); err != nil { return fmt.Errorf("failed to write hash %d to main freezer: %w", j, err) } } return nil }) if err != nil { return fmt.Errorf("failed to copy headers batch back to main freezer: %w", err) } // Update progress } return nil } // findMergeBlock detects the merge block (first block with zero difficulty) func findMergeBlock(ctx *cli.Context, db ethdb.Database) (uint64, error) { // Get the current head block headHash := rawdb.ReadHeadBlockHash(db) if headHash == (common.Hash{}) { return 0, fmt.Errorf("chain head not found") } headNumber := rawdb.ReadHeaderNumber(db, headHash) if headNumber == nil { return 0, fmt.Errorf("header number of head block not found") } var mergeBlock uint64 var found bool // Check if we should use hardcoded merge block values if ctx.Bool(useHardcodedMergeFlag.Name) { // Get the network ID to determine which hardcoded value to use networkName := ctx.String(utils.NetworkIdFlag.Name) if blockNum, ok := knownMergeBlocks[networkName]; ok { mergeBlock = blockNum found = true log.Info("Using hardcoded merge block", "network", networkName, "block", mergeBlock) } else { log.Warn("No hardcoded merge block for network, detecting dynamically", "network", networkName) } } // If no hardcoded value was used or found, detect the merge block if !found { // Binary search to find the merge block var low uint64 = 0 var high uint64 = *headNumber log.Info("Searching for merge block using binary search", "head", *headNumber) for low <= high { mid := (low + high) / 2 header := rawdb.ReadHeader(db, rawdb.ReadCanonicalHash(db, mid), mid) if header == nil { return 0, fmt.Errorf("header %d not found", mid) } if header.Difficulty.Sign() == 0 { // This is a post-merge block, look earlier high = mid - 1 mergeBlock = mid found = true } else { // This is a pre-merge block, look later low = mid + 1 } } if !found { return 0, fmt.Errorf("merge block not found, chain may not have transitioned to PoS yet") } // The merge block is the first block with zero difficulty log.Info("Found merge block", "number", mergeBlock) } return mergeBlock, nil }