forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-06-19 21:31:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

all: snapshot dump + unify with trie dump #22795 (#1205)

Browse source
This commit is contained in:
Daniel Liu 2025-08-22 16:39:28 +08:00 committed by GitHub
parent 793293f041
commit 516883d14c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 225 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

120
cmd/XDC/chaincmd.go
View file

@ -18,6 +18,7 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"os"
"runtime"
@ -28,11 +29,16 @@ import (
"github.com/XinFinOrg/XDPoSChain/cmd/utils"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/common/hexutil"
"github.com/XinFinOrg/XDPoSChain/core"
"github.com/XinFinOrg/XDPoSChain/core/rawdb"
"github.com/XinFinOrg/XDPoSChain/core/state"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/crypto"
"github.com/XinFinOrg/XDPoSChain/ethdb"
xdc_genesis "github.com/XinFinOrg/XDPoSChain/genesis"
"github.com/XinFinOrg/XDPoSChain/log"
"github.com/XinFinOrg/XDPoSChain/node"
"github.com/urfave/cli/v2"
)
@ -125,19 +131,20 @@ The export-preimages command export hash preimages to an RLP encoded stream`,
Action: dump,
Name: "dump",
Usage: "Dump a specific block from storage",
ArgsUsage: "[<blockHash> | <blockNum>]...",
ArgsUsage: "[? <blockHash> | <blockNum>]",
Flags: slices.Concat([]cli.Flag{
utils.CacheFlag,
utils.SyncModeFlag,
utils.IterativeOutputFlag,
utils.ExcludeCodeFlag,
utils.ExcludeStorageFlag,
utils.IncludeIncompletesFlag,
utils.StartKeyFlag,
utils.DumpLimitFlag,
}, utils.DatabaseFlags),
Category: "BLOCKCHAIN COMMANDS",
Description: `
The arguments are interpreted as block numbers or hashes.
Use "ethereum dump 0" to dump the genesis block.`,
This command dumps out the state for a given block (or latest, if none provided).
`,
}
)
@ -349,42 +356,85 @@ func exportPreimages(ctx *cli.Context) error {
return nil
}
func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) {
db := utils.MakeChainDatabase(ctx, stack, true)
var header *types.Header
if ctx.NArg() > 1 {
return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg())
}
if ctx.NArg() == 1 {
arg := ctx.Args().First()
if hashish(arg) {
hash := common.HexToHash(arg)
if number := rawdb.ReadHeaderNumber(db, hash); number != nil {
header = rawdb.ReadHeader(db, hash, *number)
} else {
return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash)
}
} else {
number, err := strconv.Atoi(arg)
if err != nil {
return nil, nil, common.Hash{}, err
}
if hash := rawdb.ReadCanonicalHash(db, uint64(number)); hash != (common.Hash{}) {
header = rawdb.ReadHeader(db, hash, uint64(number))
} else {
return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number)
}
}
} else {
// Use latest
header = rawdb.ReadHeadHeader(db)
}
if header == nil {
return nil, nil, common.Hash{}, errors.New("no head block found")
}
startArg := common.FromHex(ctx.String(utils.StartKeyFlag.Name))
var start common.Hash
switch len(startArg) {
case 0: // common.Hash
case 32:
start = common.BytesToHash(startArg)
case 20:
start = crypto.Keccak256Hash(startArg)
log.Info("Converting start-address to hash", "address", common.BytesToAddress(startArg), "hash", start.Hex())
default:
return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg)
}
var conf = &state.DumpConfig{
SkipCode: ctx.Bool(utils.ExcludeCodeFlag.Name),
SkipStorage: ctx.Bool(utils.ExcludeStorageFlag.Name),
OnlyWithAddresses: !ctx.Bool(utils.IncludeIncompletesFlag.Name),
Start: start.Bytes(),
Max: ctx.Uint64(utils.DumpLimitFlag.Name),
}
log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(),
"skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage,
"start", hexutil.Encode(conf.Start), "limit", conf.Max)
return conf, db, header.Root, nil
}
func dump(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chain, chainDb := utils.MakeChain(ctx, stack, true)
defer chainDb.Close()
for _, arg := range ctx.Args().Slice() {
var block *types.Block
if hashish(arg) {
block = chain.GetBlockByHash(common.HexToHash(arg))
} else {
num, _ := strconv.Atoi(arg)
block = chain.GetBlockByNumber(uint64(num))
}
if block == nil {
fmt.Println("{}")
utils.Fatalf("block not found")
} else {
state, err := state.New(block.Root(), state.NewDatabase(chainDb))
if err != nil {
utils.Fatalf("could not create new state: %v", err)
}
excludeCode := ctx.Bool(utils.ExcludeCodeFlag.Name)
excludeStorage := ctx.Bool(utils.ExcludeStorageFlag.Name)
includeMissing := ctx.Bool(utils.IncludeIncompletesFlag.Name)
if ctx.Bool(utils.IterativeOutputFlag.Name) {
state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout))
} else {
if includeMissing {
fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" +
" otherwise the accounts will overwrite each other in the resulting mapping.")
}
fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false))
}
conf, db, root, err := parseDumpConfig(ctx, stack)
if err != nil {
return err
}
state, err := state.New(root, state.NewDatabase(db))
if err != nil {
return err
}
if ctx.Bool(utils.IterativeOutputFlag.Name) {
state.IterativeDump(conf, json.NewEncoder(os.Stdout))
} else {
if conf.OnlyWithAddresses {
fmt.Fprintf(os.Stderr, "If you want to include accounts with missing preimages, you need iterative output, since"+
" otherwise the accounts will overwrite each other in the resulting mapping.")
return fmt.Errorf("incompatible options")
}
fmt.Println(string(state.Dump(conf)))
}
return nil
}

2
cmd/evm/runner.go
View file

@ -197,7 +197,7 @@ func runCmd(ctx *cli.Context) error {
if ctx.Bool(DumpFlag.Name) {
statedb.IntermediateRoot(true)
fmt.Println(string(statedb.Dump(false, false, true)))
fmt.Println(string(statedb.Dump(nil)))
}
if memProfilePath := ctx.String(MemProfileFlag.Name); memProfilePath != "" {

10
cmd/evm/staterunner.go
View file

@ -89,18 +89,18 @@ func stateTestCmd(ctx *cli.Context) error {
for _, st := range test.Subtests() {
// Run the test and aggregate the result
result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
state, err := test.Run(st, cfg)
s, err := test.Run(st, cfg)
if err != nil {
// Test failed, mark as so and dump any state to aid debugging
result.Pass, result.Error = false, err.Error()
if ctx.Bool(DumpFlag.Name) && state != nil {
dump := state.RawDump(false, false, true)
if ctx.Bool(DumpFlag.Name) && s != nil {
dump := s.RawDump(nil)
result.State = &dump
}
}
// print state root for evmlab tracing (already committed above, so no need to delete objects again
if ctx.Bool(MachineFlag.Name) && state != nil {
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false))
if ctx.Bool(MachineFlag.Name) && s != nil {
fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", s.IntermediateRoot(false))
}
results = append(results, *result)

10
cmd/utils/flags.go
View file

@ -162,6 +162,16 @@ var (
Name: "nocode",
Usage: "Exclude contract code (save db lookups)",
}
StartKeyFlag = &cli.StringFlag{
Name: "start",
Usage: "Start position. Either a hash or address",
Value: "0x0000000000000000000000000000000000000000000000000000000000000000",
}
DumpLimitFlag = &cli.Uint64Flag{
Name: "limit",
Usage: "Max number of elements (0 = no limit)",
Value: 0,
}
SyncModeFlag = &cli.StringFlag{
Name: "syncmode",

26
core/rawdb/accessors_chain.go
View file

@ -599,3 +599,29 @@ func deleteBlockWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number
DeleteBody(db, hash, number)
DeleteTd(db, hash, number)
}
// ReadHeadHeader returns the current canonical head header.
func ReadHeadHeader(db ethdb.Reader) *types.Header {
headHeaderHash := ReadHeadHeaderHash(db)
if headHeaderHash == (common.Hash{}) {
return nil
}
headHeaderNumber := ReadHeaderNumber(db, headHeaderHash)
if headHeaderNumber == nil {
return nil
}
return ReadHeader(db, headHeaderHash, *headHeaderNumber)
}
// ReadHeadHeader returns the current canonical head block.
func ReadHeadBlock(db ethdb.Reader) *types.Block {
headBlockHash := ReadHeadBlockHash(db)
if headBlockHash == (common.Hash{}) {
return nil
}
headBlockNumber := ReadHeaderNumber(db, headBlockHash)
if headBlockNumber == nil {
return nil
}
return ReadBlock(db, headBlockHash, *headBlockNumber)
}

80
core/state/dump.go
View file

@ -19,6 +19,7 @@ package state
import (
"encoding/json"
"fmt"
"time"
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/common/hexutil"
@ -28,6 +29,16 @@ import (
"github.com/XinFinOrg/XDPoSChain/trie"
)
// DumpConfig is a set of options to control what portions of the statewill be
// iterated and collected.
type DumpConfig struct {
SkipCode bool
SkipStorage bool
OnlyWithAddresses bool
Start []byte
Max uint64
}
// DumpCollector interface which the state trie calls during iteration
type DumpCollector interface {
// OnRoot is called with the state root
@ -40,9 +51,9 @@ type DumpCollector interface {
type DumpAccount struct {
Balance string `json:"balance"`
Nonce uint64 `json:"nonce"`
Root string `json:"root"`
CodeHash string `json:"codeHash"`
Code string `json:"code,omitempty"`
Root hexutil.Bytes `json:"root"`
CodeHash hexutil.Bytes `json:"codeHash"`
Code hexutil.Bytes `json:"code,omitempty"`
Storage map[common.Hash]string `json:"storage,omitempty"`
Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode
SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key
@ -111,38 +122,50 @@ func (d iterativeDump) OnRoot(root common.Hash) {
}{root})
}
func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) {
missingPreimages := 0
// DumpToCollector iterates the state according to the given options and inserts
// the items into a collector for aggregation or serialization.
func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) {
// Sanitize the input to allow nil configs
if conf == nil {
conf = new(DumpConfig)
}
var (
missingPreimages int
accounts uint64
start = time.Now()
logged = time.Now()
)
log.Info("Trie dumping started", "root", s.trie.Hash())
c.OnRoot(s.trie.Hash())
var count int
it := trie.NewIterator(s.trie.NodeIterator(start))
it := trie.NewIterator(s.trie.NodeIterator(conf.Start))
for it.Next() {
var data types.StateAccount
if err := rlp.DecodeBytes(it.Value, &data); err != nil {
panic(err)
}
account := DumpAccount{
Balance: data.Balance.String(),
Nonce: data.Nonce,
Root: common.Bytes2Hex(data.Root[:]),
CodeHash: common.Bytes2Hex(data.CodeHash),
Balance: data.Balance.String(),
Nonce: data.Nonce,
Root: data.Root[:],
CodeHash: data.CodeHash,
SecureKey: it.Key,
}
addrBytes := s.trie.GetKey(it.Key)
if addrBytes == nil {
// Preimage missing
missingPreimages++
if excludeMissingPreimages {
if conf.OnlyWithAddresses {
continue
}
account.SecureKey = it.Key
}
addr := common.BytesToAddress(addrBytes)
obj := newObject(s, addr, data)
if !excludeCode {
account.Code = common.Bytes2Hex(obj.Code(s.db))
if !conf.SkipCode {
account.Code = obj.Code(s.db)
}
if !excludeStorage {
if !conf.SkipStorage {
account.Storage = make(map[common.Hash]string)
storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil))
for storageIt.Next() {
@ -155,8 +178,13 @@ func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage,
}
}
c.OnAccount(addr, account)
count++
if maxResults > 0 && count >= maxResults {
accounts++
if time.Since(logged) > 8*time.Second {
log.Info("Trie dumping in progress", "at", it.Key, "accounts", accounts,
"elapsed", common.PrettyDuration(time.Since(start)))
logged = time.Now()
}
if conf.Max > 0 && accounts >= conf.Max {
if it.Next() {
nextKey = it.Key
}
@ -166,22 +194,24 @@ func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage,
if missingPreimages > 0 {
log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
}
log.Info("Trie dumping complete", "accounts", accounts,
"elapsed", common.PrettyDuration(time.Since(start)))
return nextKey
}
// RawDump returns the entire state an a single large object
func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump {
func (s *StateDB) RawDump(opts *DumpConfig) Dump {
dump := &Dump{
Accounts: make(map[common.Address]DumpAccount),
}
s.DumpToCollector(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
s.DumpToCollector(dump, opts)
return *dump
}
// Dump returns a JSON string representing the entire state as a single json-object
func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte {
dump := s.RawDump(excludeCode, excludeStorage, excludeMissingPreimages)
func (s *StateDB) Dump(opts *DumpConfig) []byte {
dump := s.RawDump(opts)
json, err := json.MarshalIndent(dump, "", " ")
if err != nil {
fmt.Println("Dump err", err)
@ -190,15 +220,15 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool
}
// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
s.DumpToCollector(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
func (s *StateDB) IterativeDump(opts *DumpConfig, output *json.Encoder) {
s.DumpToCollector(iterativeDump{output}, opts)
}
// IteratorDump dumps out a batch of accounts starts with the given start key
func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump {
func (s *StateDB) IteratorDump(opts *DumpConfig) IteratorDump {
iterator := &IteratorDump{
Accounts: make(map[common.Address]DumpAccount),
}
iterator.Next = s.DumpToCollector(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults)
iterator.Next = s.DumpToCollector(iterator, opts)
return *iterator
}

19
core/state/state_test.go
View file

@ -56,28 +56,31 @@ func TestDump(t *testing.T) {
s.state.Commit(false)
// check that DumpToCollector contains the state objects that are in trie
got := string(s.state.Dump(false, false, true))
got := string(s.state.Dump(nil))
want := `{
"root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2",
"accounts": {
"0x0000000000000000000000000000000000000001": {
"balance": "22",
"nonce": 0,
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
"root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
"key": "0x1468288056310c82aa4c01a7e12a10f8111a0560e72b700555479031b86c357d"
},
"0x0000000000000000000000000000000000000002": {
"balance": "44",
"nonce": 0,
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
"root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
"key": "0xd52688a8f926c816ca1e079067caba944f158e764817b83fc43594370ca9cf62"
},
"0x0000000000000000000000000000000000000102": {
"balance": "0",
"nonce": 0,
"root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3",
"code": "03030303030303"
"root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
"codeHash": "0x87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3",
"code": "0x03030303030303",
"key": "0xa17eacbc25cda025e81db9c5c62868822c73ce097cee2a63e33a2e41268358a1"
}
}
}`

19
eth/api.go
View file

@ -281,6 +281,10 @@ func (api *AdminAPI) ImportChain(file string) (bool, error) {
// DumpBlock retrieves the entire state of the database at a given block.
func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) {
opts := &state.DumpConfig{
OnlyWithAddresses: true,
Max: AccountRangeMaxResults, // Sanity limit over RPC
}
if blockNr == rpc.PendingBlockNumber {
blockNr = rpc.LatestBlockNumber
}
@ -297,7 +301,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) {
if err != nil {
return state.Dump{}, err
}
return stateDb.RawDump(false, false, true), nil
return stateDb.RawDump(opts), nil
}
// DebugAPI is the collection of Ethereum full node APIs exposed over
@ -367,10 +371,17 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start []b
}
}
if maxResults > AccountRangeMaxResults || maxResults <= 0 {
maxResults = AccountRangeMaxResults
opts := &state.DumpConfig{
SkipCode: nocode,
SkipStorage: nostorage,
OnlyWithAddresses: !incompletes,
Start: start,
Max: uint64(maxResults),
}
return stateDb.IteratorDump(nocode, nostorage, incompletes, start, maxResults), nil
if maxResults > AccountRangeMaxResults || maxResults <= 0 {
opts.Max = AccountRangeMaxResults
}
return stateDb.IteratorDump(opts), nil
}
// StorageRangeResult is the result of a debug_storageRangeAt API call.

23
eth/api_test.go
View file

@ -36,7 +36,13 @@ import (
var dumper = spew.ConfigState{Indent: " "}
func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump {
result := statedb.IteratorDump(true, true, false, start.Bytes(), requestedNum)
result := statedb.IteratorDump(&state.DumpConfig{
SkipCode: true,
SkipStorage: true,
OnlyWithAddresses: false,
Start: start.Bytes(),
Max: uint64(requestedNum),
})
if len(result.Accounts) != expectedNum {
t.Fatalf("expected %d results, got %d", expectedNum, len(result.Accounts))
@ -129,12 +135,17 @@ func TestAccountRange(t *testing.T) {
func TestEmptyAccountRange(t *testing.T) {
var (
statedb = state.NewDatabase(rawdb.NewMemoryDatabase())
state, _ = state.New(common.Hash{}, statedb)
statedb = state.NewDatabase(rawdb.NewMemoryDatabase())
st, _ = state.New(common.Hash{}, statedb)
)
state.Commit(true)
state.IntermediateRoot(true)
results := state.IteratorDump(true, true, true, (common.Hash{}).Bytes(), AccountRangeMaxResults)
st.Commit(true)
st.IntermediateRoot(true)
results := st.IteratorDump(&state.DumpConfig{
SkipCode: true,
SkipStorage: true,
OnlyWithAddresses: true,
Max: uint64(AccountRangeMaxResults),
})
if bytes.Equal(results.Next, (common.Hash{}).Bytes()) {
t.Fatalf("Empty results should not return a second page")
}