diff --git a/cmd/XDC/chaincmd.go b/cmd/XDC/chaincmd.go index 5025e7a0c4..ca995d3af7 100644 --- a/cmd/XDC/chaincmd.go +++ b/cmd/XDC/chaincmd.go @@ -18,6 +18,7 @@ package main import ( "encoding/json" + "errors" "fmt" "os" "runtime" @@ -28,11 +29,16 @@ import ( "github.com/XinFinOrg/XDPoSChain/cmd/utils" "github.com/XinFinOrg/XDPoSChain/common" + "github.com/XinFinOrg/XDPoSChain/common/hexutil" "github.com/XinFinOrg/XDPoSChain/core" + "github.com/XinFinOrg/XDPoSChain/core/rawdb" "github.com/XinFinOrg/XDPoSChain/core/state" "github.com/XinFinOrg/XDPoSChain/core/types" + "github.com/XinFinOrg/XDPoSChain/crypto" + "github.com/XinFinOrg/XDPoSChain/ethdb" xdc_genesis "github.com/XinFinOrg/XDPoSChain/genesis" "github.com/XinFinOrg/XDPoSChain/log" + "github.com/XinFinOrg/XDPoSChain/node" "github.com/urfave/cli/v2" ) @@ -125,19 +131,20 @@ The export-preimages command export hash preimages to an RLP encoded stream`, Action: dump, Name: "dump", Usage: "Dump a specific block from storage", - ArgsUsage: "[ | ]...", + ArgsUsage: "[? | ]", Flags: slices.Concat([]cli.Flag{ utils.CacheFlag, - utils.SyncModeFlag, utils.IterativeOutputFlag, utils.ExcludeCodeFlag, utils.ExcludeStorageFlag, utils.IncludeIncompletesFlag, + utils.StartKeyFlag, + utils.DumpLimitFlag, }, utils.DatabaseFlags), Category: "BLOCKCHAIN COMMANDS", Description: ` -The arguments are interpreted as block numbers or hashes. -Use "ethereum dump 0" to dump the genesis block.`, +This command dumps out the state for a given block (or latest, if none provided). +`, } ) @@ -349,42 +356,85 @@ func exportPreimages(ctx *cli.Context) error { return nil } +func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) { + db := utils.MakeChainDatabase(ctx, stack, true) + var header *types.Header + if ctx.NArg() > 1 { + return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg()) + } + if ctx.NArg() == 1 { + arg := ctx.Args().First() + if hashish(arg) { + hash := common.HexToHash(arg) + if number := rawdb.ReadHeaderNumber(db, hash); number != nil { + header = rawdb.ReadHeader(db, hash, *number) + } else { + return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash) + } + } else { + number, err := strconv.Atoi(arg) + if err != nil { + return nil, nil, common.Hash{}, err + } + if hash := rawdb.ReadCanonicalHash(db, uint64(number)); hash != (common.Hash{}) { + header = rawdb.ReadHeader(db, hash, uint64(number)) + } else { + return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) + } + } + } else { + // Use latest + header = rawdb.ReadHeadHeader(db) + } + if header == nil { + return nil, nil, common.Hash{}, errors.New("no head block found") + } + startArg := common.FromHex(ctx.String(utils.StartKeyFlag.Name)) + var start common.Hash + switch len(startArg) { + case 0: // common.Hash + case 32: + start = common.BytesToHash(startArg) + case 20: + start = crypto.Keccak256Hash(startArg) + log.Info("Converting start-address to hash", "address", common.BytesToAddress(startArg), "hash", start.Hex()) + default: + return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg) + } + var conf = &state.DumpConfig{ + SkipCode: ctx.Bool(utils.ExcludeCodeFlag.Name), + SkipStorage: ctx.Bool(utils.ExcludeStorageFlag.Name), + OnlyWithAddresses: !ctx.Bool(utils.IncludeIncompletesFlag.Name), + Start: start.Bytes(), + Max: ctx.Uint64(utils.DumpLimitFlag.Name), + } + log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(), + "skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage, + "start", hexutil.Encode(conf.Start), "limit", conf.Max) + return conf, db, header.Root, nil +} + func dump(ctx *cli.Context) error { stack, _ := makeConfigNode(ctx) defer stack.Close() - chain, chainDb := utils.MakeChain(ctx, stack, true) - defer chainDb.Close() - - for _, arg := range ctx.Args().Slice() { - var block *types.Block - if hashish(arg) { - block = chain.GetBlockByHash(common.HexToHash(arg)) - } else { - num, _ := strconv.Atoi(arg) - block = chain.GetBlockByNumber(uint64(num)) - } - if block == nil { - fmt.Println("{}") - utils.Fatalf("block not found") - } else { - state, err := state.New(block.Root(), state.NewDatabase(chainDb)) - if err != nil { - utils.Fatalf("could not create new state: %v", err) - } - excludeCode := ctx.Bool(utils.ExcludeCodeFlag.Name) - excludeStorage := ctx.Bool(utils.ExcludeStorageFlag.Name) - includeMissing := ctx.Bool(utils.IncludeIncompletesFlag.Name) - if ctx.Bool(utils.IterativeOutputFlag.Name) { - state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout)) - } else { - if includeMissing { - fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" + - " otherwise the accounts will overwrite each other in the resulting mapping.") - } - fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false)) - } + conf, db, root, err := parseDumpConfig(ctx, stack) + if err != nil { + return err + } + state, err := state.New(root, state.NewDatabase(db)) + if err != nil { + return err + } + if ctx.Bool(utils.IterativeOutputFlag.Name) { + state.IterativeDump(conf, json.NewEncoder(os.Stdout)) + } else { + if conf.OnlyWithAddresses { + fmt.Fprintf(os.Stderr, "If you want to include accounts with missing preimages, you need iterative output, since"+ + " otherwise the accounts will overwrite each other in the resulting mapping.") + return fmt.Errorf("incompatible options") } + fmt.Println(string(state.Dump(conf))) } return nil } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index bec7b4434d..25e90496f8 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -197,7 +197,7 @@ func runCmd(ctx *cli.Context) error { if ctx.Bool(DumpFlag.Name) { statedb.IntermediateRoot(true) - fmt.Println(string(statedb.Dump(false, false, true))) + fmt.Println(string(statedb.Dump(nil))) } if memProfilePath := ctx.String(MemProfileFlag.Name); memProfilePath != "" { diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index d90cdf215f..c8115a5336 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -89,18 +89,18 @@ func stateTestCmd(ctx *cli.Context) error { for _, st := range test.Subtests() { // Run the test and aggregate the result result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true} - state, err := test.Run(st, cfg) + s, err := test.Run(st, cfg) if err != nil { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() - if ctx.Bool(DumpFlag.Name) && state != nil { - dump := state.RawDump(false, false, true) + if ctx.Bool(DumpFlag.Name) && s != nil { + dump := s.RawDump(nil) result.State = &dump } } // print state root for evmlab tracing (already committed above, so no need to delete objects again - if ctx.Bool(MachineFlag.Name) && state != nil { - fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", state.IntermediateRoot(false)) + if ctx.Bool(MachineFlag.Name) && s != nil { + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%x\"}\n", s.IntermediateRoot(false)) } results = append(results, *result) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 419ca477b8..693fcfc8d1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -162,6 +162,16 @@ var ( Name: "nocode", Usage: "Exclude contract code (save db lookups)", } + StartKeyFlag = &cli.StringFlag{ + Name: "start", + Usage: "Start position. Either a hash or address", + Value: "0x0000000000000000000000000000000000000000000000000000000000000000", + } + DumpLimitFlag = &cli.Uint64Flag{ + Name: "limit", + Usage: "Max number of elements (0 = no limit)", + Value: 0, + } SyncModeFlag = &cli.StringFlag{ Name: "syncmode", diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 071d69445b..81579cbfc2 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -599,3 +599,29 @@ func deleteBlockWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number DeleteBody(db, hash, number) DeleteTd(db, hash, number) } + +// ReadHeadHeader returns the current canonical head header. +func ReadHeadHeader(db ethdb.Reader) *types.Header { + headHeaderHash := ReadHeadHeaderHash(db) + if headHeaderHash == (common.Hash{}) { + return nil + } + headHeaderNumber := ReadHeaderNumber(db, headHeaderHash) + if headHeaderNumber == nil { + return nil + } + return ReadHeader(db, headHeaderHash, *headHeaderNumber) +} + +// ReadHeadHeader returns the current canonical head block. +func ReadHeadBlock(db ethdb.Reader) *types.Block { + headBlockHash := ReadHeadBlockHash(db) + if headBlockHash == (common.Hash{}) { + return nil + } + headBlockNumber := ReadHeaderNumber(db, headBlockHash) + if headBlockNumber == nil { + return nil + } + return ReadBlock(db, headBlockHash, *headBlockNumber) +} diff --git a/core/state/dump.go b/core/state/dump.go index d6f94425ed..a3cbbbb1dc 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -19,6 +19,7 @@ package state import ( "encoding/json" "fmt" + "time" "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/common/hexutil" @@ -28,6 +29,16 @@ import ( "github.com/XinFinOrg/XDPoSChain/trie" ) +// DumpConfig is a set of options to control what portions of the statewill be +// iterated and collected. +type DumpConfig struct { + SkipCode bool + SkipStorage bool + OnlyWithAddresses bool + Start []byte + Max uint64 +} + // DumpCollector interface which the state trie calls during iteration type DumpCollector interface { // OnRoot is called with the state root @@ -40,9 +51,9 @@ type DumpCollector interface { type DumpAccount struct { Balance string `json:"balance"` Nonce uint64 `json:"nonce"` - Root string `json:"root"` - CodeHash string `json:"codeHash"` - Code string `json:"code,omitempty"` + Root hexutil.Bytes `json:"root"` + CodeHash hexutil.Bytes `json:"codeHash"` + Code hexutil.Bytes `json:"code,omitempty"` Storage map[common.Hash]string `json:"storage,omitempty"` Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key @@ -111,38 +122,50 @@ func (d iterativeDump) OnRoot(root common.Hash) { }{root}) } -func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) { - missingPreimages := 0 +// DumpToCollector iterates the state according to the given options and inserts +// the items into a collector for aggregation or serialization. +func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) { + // Sanitize the input to allow nil configs + if conf == nil { + conf = new(DumpConfig) + } + var ( + missingPreimages int + accounts uint64 + start = time.Now() + logged = time.Now() + ) + log.Info("Trie dumping started", "root", s.trie.Hash()) c.OnRoot(s.trie.Hash()) - var count int - it := trie.NewIterator(s.trie.NodeIterator(start)) + it := trie.NewIterator(s.trie.NodeIterator(conf.Start)) for it.Next() { var data types.StateAccount if err := rlp.DecodeBytes(it.Value, &data); err != nil { panic(err) } account := DumpAccount{ - Balance: data.Balance.String(), - Nonce: data.Nonce, - Root: common.Bytes2Hex(data.Root[:]), - CodeHash: common.Bytes2Hex(data.CodeHash), + Balance: data.Balance.String(), + Nonce: data.Nonce, + Root: data.Root[:], + CodeHash: data.CodeHash, + SecureKey: it.Key, } addrBytes := s.trie.GetKey(it.Key) if addrBytes == nil { // Preimage missing missingPreimages++ - if excludeMissingPreimages { + if conf.OnlyWithAddresses { continue } account.SecureKey = it.Key } addr := common.BytesToAddress(addrBytes) obj := newObject(s, addr, data) - if !excludeCode { - account.Code = common.Bytes2Hex(obj.Code(s.db)) + if !conf.SkipCode { + account.Code = obj.Code(s.db) } - if !excludeStorage { + if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil)) for storageIt.Next() { @@ -155,8 +178,13 @@ func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, } } c.OnAccount(addr, account) - count++ - if maxResults > 0 && count >= maxResults { + accounts++ + if time.Since(logged) > 8*time.Second { + log.Info("Trie dumping in progress", "at", it.Key, "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + if conf.Max > 0 && accounts >= conf.Max { if it.Next() { nextKey = it.Key } @@ -166,22 +194,24 @@ func (s *StateDB) DumpToCollector(c DumpCollector, excludeCode, excludeStorage, if missingPreimages > 0 { log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) } + log.Info("Trie dumping complete", "accounts", accounts, + "elapsed", common.PrettyDuration(time.Since(start))) return nextKey } // RawDump returns the entire state an a single large object -func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump { +func (s *StateDB) RawDump(opts *DumpConfig) Dump { dump := &Dump{ Accounts: make(map[common.Address]DumpAccount), } - s.DumpToCollector(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0) + s.DumpToCollector(dump, opts) return *dump } // Dump returns a JSON string representing the entire state as a single json-object -func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte { - dump := s.RawDump(excludeCode, excludeStorage, excludeMissingPreimages) +func (s *StateDB) Dump(opts *DumpConfig) []byte { + dump := s.RawDump(opts) json, err := json.MarshalIndent(dump, "", " ") if err != nil { fmt.Println("Dump err", err) @@ -190,15 +220,15 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool } // IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout -func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) { - s.DumpToCollector(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0) +func (s *StateDB) IterativeDump(opts *DumpConfig, output *json.Encoder) { + s.DumpToCollector(iterativeDump{output}, opts) } // IteratorDump dumps out a batch of accounts starts with the given start key -func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump { +func (s *StateDB) IteratorDump(opts *DumpConfig) IteratorDump { iterator := &IteratorDump{ Accounts: make(map[common.Address]DumpAccount), } - iterator.Next = s.DumpToCollector(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults) + iterator.Next = s.DumpToCollector(iterator, opts) return *iterator } diff --git a/core/state/state_test.go b/core/state/state_test.go index 9f3a904852..6ac156939c 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -56,28 +56,31 @@ func TestDump(t *testing.T) { s.state.Commit(false) // check that DumpToCollector contains the state objects that are in trie - got := string(s.state.Dump(false, false, true)) + got := string(s.state.Dump(nil)) want := `{ "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2", "accounts": { "0x0000000000000000000000000000000000000001": { "balance": "22", "nonce": 0, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "codeHash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "key": "0x1468288056310c82aa4c01a7e12a10f8111a0560e72b700555479031b86c357d" }, "0x0000000000000000000000000000000000000002": { "balance": "44", "nonce": 0, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" + "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "codeHash": "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + "key": "0xd52688a8f926c816ca1e079067caba944f158e764817b83fc43594370ca9cf62" }, "0x0000000000000000000000000000000000000102": { "balance": "0", "nonce": 0, - "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3", - "code": "03030303030303" + "root": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "codeHash": "0x87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3", + "code": "0x03030303030303", + "key": "0xa17eacbc25cda025e81db9c5c62868822c73ce097cee2a63e33a2e41268358a1" } } }` diff --git a/eth/api.go b/eth/api.go index 9872aebd8c..91fa756f71 100644 --- a/eth/api.go +++ b/eth/api.go @@ -281,6 +281,10 @@ func (api *AdminAPI) ImportChain(file string) (bool, error) { // DumpBlock retrieves the entire state of the database at a given block. func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { + opts := &state.DumpConfig{ + OnlyWithAddresses: true, + Max: AccountRangeMaxResults, // Sanity limit over RPC + } if blockNr == rpc.PendingBlockNumber { blockNr = rpc.LatestBlockNumber } @@ -297,7 +301,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { if err != nil { return state.Dump{}, err } - return stateDb.RawDump(false, false, true), nil + return stateDb.RawDump(opts), nil } // DebugAPI is the collection of Ethereum full node APIs exposed over @@ -367,10 +371,17 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start []b } } - if maxResults > AccountRangeMaxResults || maxResults <= 0 { - maxResults = AccountRangeMaxResults + opts := &state.DumpConfig{ + SkipCode: nocode, + SkipStorage: nostorage, + OnlyWithAddresses: !incompletes, + Start: start, + Max: uint64(maxResults), } - return stateDb.IteratorDump(nocode, nostorage, incompletes, start, maxResults), nil + if maxResults > AccountRangeMaxResults || maxResults <= 0 { + opts.Max = AccountRangeMaxResults + } + return stateDb.IteratorDump(opts), nil } // StorageRangeResult is the result of a debug_storageRangeAt API call. diff --git a/eth/api_test.go b/eth/api_test.go index 5ecb58b500..37e7997cec 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -36,7 +36,13 @@ import ( var dumper = spew.ConfigState{Indent: " "} func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump { - result := statedb.IteratorDump(true, true, false, start.Bytes(), requestedNum) + result := statedb.IteratorDump(&state.DumpConfig{ + SkipCode: true, + SkipStorage: true, + OnlyWithAddresses: false, + Start: start.Bytes(), + Max: uint64(requestedNum), + }) if len(result.Accounts) != expectedNum { t.Fatalf("expected %d results, got %d", expectedNum, len(result.Accounts)) @@ -129,12 +135,17 @@ func TestAccountRange(t *testing.T) { func TestEmptyAccountRange(t *testing.T) { var ( - statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) - state, _ = state.New(common.Hash{}, statedb) + statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) + st, _ = state.New(common.Hash{}, statedb) ) - state.Commit(true) - state.IntermediateRoot(true) - results := state.IteratorDump(true, true, true, (common.Hash{}).Bytes(), AccountRangeMaxResults) + st.Commit(true) + st.IntermediateRoot(true) + results := st.IteratorDump(&state.DumpConfig{ + SkipCode: true, + SkipStorage: true, + OnlyWithAddresses: true, + Max: uint64(AccountRangeMaxResults), + }) if bytes.Equal(results.Next, (common.Hash{}).Bytes()) { t.Fatalf("Empty results should not return a second page") }