diff --git a/cmd/XDC/chaincmd.go b/cmd/XDC/chaincmd.go index 0566f0faed..a9374d4ab4 100644 --- a/cmd/XDC/chaincmd.go +++ b/cmd/XDC/chaincmd.go @@ -129,6 +129,10 @@ The export-preimages command export hash preimages to an RLP encoded stream`, Flags: slices.Concat([]cli.Flag{ utils.CacheFlag, utils.SyncModeFlag, + utils.IterativeOutputFlag, + utils.ExcludeCodeFlag, + utils.ExcludeStorageFlag, + utils.IncludeIncompletesFlag, }, utils.DatabaseFlags), Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -366,7 +370,18 @@ func dump(ctx *cli.Context) error { if err != nil { utils.Fatalf("could not create new state: %v", err) } - fmt.Printf("%s\n", state.Dump()) + excludeCode := ctx.Bool(utils.ExcludeCodeFlag.Name) + excludeStorage := ctx.Bool(utils.ExcludeStorageFlag.Name) + includeMissing := ctx.Bool(utils.IncludeIncompletesFlag.Name) + if ctx.Bool(utils.IterativeOutputFlag.Name) { + state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout)) + } else { + if includeMissing { + fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" + + " otherwise the accounts will overwrite each other in the resulting mapping.") + } + fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false)) + } } } return nil diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index b6d48d5b10..bec7b4434d 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -197,7 +197,7 @@ func runCmd(ctx *cli.Context) error { if ctx.Bool(DumpFlag.Name) { statedb.IntermediateRoot(true) - fmt.Println(string(statedb.Dump())) + fmt.Println(string(statedb.Dump(false, false, true))) } if memProfilePath := ctx.String(MemProfileFlag.Name); memProfilePath != "" { diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index d093c73307..d90cdf215f 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -94,7 +94,7 @@ func stateTestCmd(ctx *cli.Context) error { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() if ctx.Bool(DumpFlag.Name) && state != nil { - dump := state.RawDump() + dump := state.RawDump(false, false, true) result.State = &dump } } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 4e9336fb30..a112ea7d9b 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -143,6 +143,25 @@ var ( Category: flags.NetworkingCategory, } + // Dump command options. + IterativeOutputFlag = &cli.BoolFlag{ + Name: "iterative", + Usage: "Print streaming JSON iteratively, delimited by newlines", + Value: true, + } + ExcludeStorageFlag = &cli.BoolFlag{ + Name: "nostorage", + Usage: "Exclude storage entries (save db lookups)", + } + IncludeIncompletesFlag = &cli.BoolFlag{ + Name: "incompletes", + Usage: "Include accounts for which we don't have the address (missing preimage)", + } + ExcludeCodeFlag = &cli.BoolFlag{ + Name: "nocode", + Usage: "Exclude contract code (save db lookups)", + } + SyncModeFlag = &cli.StringFlag{ Name: "syncmode", Usage: `Blockchain sync mode ("fast" or "full")`, diff --git a/core/state/dump.go b/core/state/dump.go index f7f4c4506d..500c8aee39 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -21,61 +21,131 @@ import ( "fmt" "github.com/XinFinOrg/XDPoSChain/common" + "github.com/XinFinOrg/XDPoSChain/common/hexutil" + "github.com/XinFinOrg/XDPoSChain/log" "github.com/XinFinOrg/XDPoSChain/rlp" "github.com/XinFinOrg/XDPoSChain/trie" ) type DumpAccount struct { - Balance string `json:"balance"` - Nonce uint64 `json:"nonce"` - Root string `json:"root"` - CodeHash string `json:"codeHash"` - Code string `json:"code"` - Storage map[string]string `json:"storage"` + Balance string `json:"balance"` + Nonce uint64 `json:"nonce"` + Root string `json:"root"` + CodeHash string `json:"codeHash"` + Code string `json:"code,omitempty"` + Storage map[common.Hash]string `json:"storage,omitempty"` + Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode + SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key } type Dump struct { - Root string `json:"root"` - Accounts map[string]DumpAccount `json:"accounts"` + Root string `json:"root"` + Accounts map[common.Address]DumpAccount `json:"accounts"` } -func (s *StateDB) RawDump() Dump { - dump := Dump{ - Root: fmt.Sprintf("%x", s.trie.Hash()), - Accounts: make(map[string]DumpAccount), - } +// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively +type iterativeDump json.Encoder +// Collector interface which the state trie calls during iteration +type collector interface { + onRoot(common.Hash) + onAccount(common.Address, DumpAccount) +} + +func (d *Dump) onRoot(root common.Hash) { + d.Root = fmt.Sprintf("%x", root) +} + +func (d *Dump) onAccount(addr common.Address, account DumpAccount) { + d.Accounts[addr] = account +} + +func (d iterativeDump) onAccount(addr common.Address, account DumpAccount) { + dumpAccount := &DumpAccount{ + Balance: account.Balance, + Nonce: account.Nonce, + Root: account.Root, + CodeHash: account.CodeHash, + Code: account.Code, + Storage: account.Storage, + SecureKey: account.SecureKey, + Address: nil, + } + if addr != (common.Address{}) { + dumpAccount.Address = &addr + } + (*json.Encoder)(&d).Encode(dumpAccount) +} + +func (d iterativeDump) onRoot(root common.Hash) { + (*json.Encoder)(&d).Encode(struct { + Root common.Hash `json:"root"` + }{root}) +} + +func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) { + emptyAddress := (common.Address{}) + missingPreimages := 0 + c.onRoot(s.trie.Hash()) it := trie.NewIterator(s.trie.NodeIterator(nil)) for it.Next() { - addr := s.trie.GetKey(it.Key) var data Account if err := rlp.DecodeBytes(it.Value, &data); err != nil { panic(err) } - - obj := newObject(nil, common.BytesToAddress(addr), data) + addr := common.BytesToAddress(s.trie.GetKey(it.Key)) + obj := newObject(nil, addr, data) account := DumpAccount{ Balance: data.Balance.String(), Nonce: data.Nonce, Root: common.Bytes2Hex(data.Root[:]), CodeHash: common.Bytes2Hex(data.CodeHash), - Code: common.Bytes2Hex(obj.Code(s.db)), - Storage: make(map[string]string), } - storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil)) - for storageIt.Next() { - account.Storage[common.Bytes2Hex(s.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value) + if emptyAddress == addr { + // Preimage missing + missingPreimages++ + if excludeMissingPreimages { + continue + } + account.SecureKey = it.Key } - dump.Accounts[common.Bytes2Hex(addr)] = account + if !excludeCode { + account.Code = common.Bytes2Hex(obj.Code(s.db)) + } + if !excludeStorage { + account.Storage = make(map[common.Hash]string) + storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil)) + for storageIt.Next() { + account.Storage[common.BytesToHash(s.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value) + } + } + c.onAccount(addr, account) + } + if missingPreimages > 0 { + log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) } - return dump } -func (s *StateDB) Dump() []byte { - json, err := json.MarshalIndent(s.RawDump(), "", " ") +// RawDump returns the entire state an a single large object +func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump { + dump := &Dump{ + Accounts: make(map[common.Address]DumpAccount), + } + s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages) + return *dump +} + +// Dump returns a JSON string representing the entire state as a single json-object +func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte { + dump := s.RawDump(excludeCode, excludeStorage, excludeMissingPreimages) + json, err := json.MarshalIndent(dump, "", " ") if err != nil { fmt.Println("dump err", err) } - return json } + +// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout +func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) { + s.dump(iterativeDump(*output), excludeCode, excludeStorage, excludeMissingPreimages) +} diff --git a/core/state/state_test.go b/core/state/state_test.go index 6f3c3598d4..eedd5931bf 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -51,33 +51,28 @@ func (s *StateSuite) TestDump(c *checker.C) { s.state.Commit(false) // check that dump contains the state objects that are in trie - got := string(s.state.Dump()) + got := string(s.state.Dump(false, false, true)) want := `{ "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2", "accounts": { - "0000000000000000000000000000000000000001": { + "0x0000000000000000000000000000000000000001": { "balance": "22", "nonce": 0, "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - "code": "", - "storage": {} + "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" }, - "0000000000000000000000000000000000000002": { + "0x0000000000000000000000000000000000000002": { "balance": "44", "nonce": 0, "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", - "code": "", - "storage": {} + "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470" }, - "0000000000000000000000000000000000000102": { + "0x0000000000000000000000000000000000000102": { "balance": "0", "nonce": 0, "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", "codeHash": "87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3", - "code": "03030303030303", - "storage": {} + "code": "03030303030303" } } }` diff --git a/core/state/statedb.go b/core/state/statedb.go index 3c7c60d65a..8bbde0a51e 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -596,15 +596,15 @@ func (s *StateDB) CreateAccount(addr common.Address) { } } -func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) error { - so := db.getStateObject(addr) +func (s *StateDB) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) error { + so := s.getStateObject(addr) if so == nil { return nil } - it := trie.NewIterator(so.getTrie(db.db).NodeIterator(nil)) + it := trie.NewIterator(so.getTrie(s.db).NodeIterator(nil)) for it.Next() { - key := common.BytesToHash(db.trie.GetKey(it.Key)) + key := common.BytesToHash(s.trie.GetKey(it.Key)) if value, dirty := so.dirtyStorage[key]; dirty { if !cb(key, value) { return nil diff --git a/eth/api.go b/eth/api.go index c6b92c99a9..d1c74638bf 100644 --- a/eth/api.go +++ b/eth/api.go @@ -297,7 +297,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) { if err != nil { return state.Dump{}, err } - return stateDb.RawDump(), nil + return stateDb.RawDump(false, false, true), nil } // DebugAPI is the collection of Ethereum full node APIs exposed over