core, cmd: streaming json output for command #15475 (#1184)

cmd/XDC/chaincmd.go

@@ -129,6 +129,10 @@ The export-preimages command export hash preimages to an RLP encoded stream`,
 		Flags: slices.Concat([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
+			utils.IterativeOutputFlag,
+			utils.ExcludeCodeFlag,
+			utils.ExcludeStorageFlag,
+			utils.IncludeIncompletesFlag,
 		}, utils.DatabaseFlags),
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -366,7 +370,18 @@ func dump(ctx *cli.Context) error {
 			if err != nil {
 				utils.Fatalf("could not create new state: %v", err)
 			}
-			fmt.Printf("%s\n", state.Dump())
+			excludeCode := ctx.Bool(utils.ExcludeCodeFlag.Name)
+			excludeStorage := ctx.Bool(utils.ExcludeStorageFlag.Name)
+			includeMissing := ctx.Bool(utils.IncludeIncompletesFlag.Name)
+			if ctx.Bool(utils.IterativeOutputFlag.Name) {
+				state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout))
+			} else {
+				if includeMissing {
+					fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" +
+						" otherwise the accounts will overwrite each other in the resulting mapping.")
+				}
+				fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false))
+			}
 		}
 	}
 	return nil

cmd/evm/runner.go

@@ -197,7 +197,7 @@ func runCmd(ctx *cli.Context) error {
 
 	if ctx.Bool(DumpFlag.Name) {
 		statedb.IntermediateRoot(true)
-		fmt.Println(string(statedb.Dump()))
+		fmt.Println(string(statedb.Dump(false, false, true)))
 	}
 
 	if memProfilePath := ctx.String(MemProfileFlag.Name); memProfilePath != "" {

cmd/evm/staterunner.go

@@ -94,7 +94,7 @@ func stateTestCmd(ctx *cli.Context) error {
 				// Test failed, mark as so and dump any state to aid debugging
 				result.Pass, result.Error = false, err.Error()
 				if ctx.Bool(DumpFlag.Name) && state != nil {
-					dump := state.RawDump()
+					dump := state.RawDump(false, false, true)
 					result.State = &dump
 				}
 			}

cmd/utils/flags.go

@@ -143,6 +143,25 @@ var (
 		Category: flags.NetworkingCategory,
 	}
 
+	// Dump command options.
+	IterativeOutputFlag = &cli.BoolFlag{
+		Name:  "iterative",
+		Usage: "Print streaming JSON iteratively, delimited by newlines",
+		Value: true,
+	}
+	ExcludeStorageFlag = &cli.BoolFlag{
+		Name:  "nostorage",
+		Usage: "Exclude storage entries (save db lookups)",
+	}
+	IncludeIncompletesFlag = &cli.BoolFlag{
+		Name:  "incompletes",
+		Usage: "Include accounts for which we don't have the address (missing preimage)",
+	}
+	ExcludeCodeFlag = &cli.BoolFlag{
+		Name:  "nocode",
+		Usage: "Exclude contract code (save db lookups)",
+	}
+
 	SyncModeFlag = &cli.StringFlag{
 		Name:     "syncmode",
 		Usage:    `Blockchain sync mode ("fast" or "full")`,

core/state/dump.go

@@ -21,61 +21,131 @@ import (
 	"fmt"
 
 	"github.com/XinFinOrg/XDPoSChain/common"
+	"github.com/XinFinOrg/XDPoSChain/common/hexutil"
+	"github.com/XinFinOrg/XDPoSChain/log"
 	"github.com/XinFinOrg/XDPoSChain/rlp"
 	"github.com/XinFinOrg/XDPoSChain/trie"
 )
 
 type DumpAccount struct {
-	Balance  string            `json:"balance"`
+	Balance   string                 `json:"balance"`
-	Nonce    uint64            `json:"nonce"`
+	Nonce     uint64                 `json:"nonce"`
-	Root     string            `json:"root"`
+	Root      string                 `json:"root"`
-	CodeHash string            `json:"codeHash"`
+	CodeHash  string                 `json:"codeHash"`
-	Code     string            `json:"code"`
+	Code      string                 `json:"code,omitempty"`
-	Storage  map[string]string `json:"storage"`
+	Storage   map[common.Hash]string `json:"storage,omitempty"`
+	Address   *common.Address        `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode
+	SecureKey hexutil.Bytes          `json:"key,omitempty"`     // If we don't have address, we can output the key
 }
 
 type Dump struct {
-	Root     string                 `json:"root"`
+	Root     string                         `json:"root"`
-	Accounts map[string]DumpAccount `json:"accounts"`
+	Accounts map[common.Address]DumpAccount `json:"accounts"`
 }
 
-func (s *StateDB) RawDump() Dump {
+// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
-	dump := Dump{
+type iterativeDump json.Encoder
-		Root:     fmt.Sprintf("%x", s.trie.Hash()),
-		Accounts: make(map[string]DumpAccount),
-	}
 
+// Collector interface which the state trie calls during iteration
+type collector interface {
+	onRoot(common.Hash)
+	onAccount(common.Address, DumpAccount)
+}
+
+func (d *Dump) onRoot(root common.Hash) {
+	d.Root = fmt.Sprintf("%x", root)
+}
+
+func (d *Dump) onAccount(addr common.Address, account DumpAccount) {
+	d.Accounts[addr] = account
+}
+
+func (d iterativeDump) onAccount(addr common.Address, account DumpAccount) {
+	dumpAccount := &DumpAccount{
+		Balance:   account.Balance,
+		Nonce:     account.Nonce,
+		Root:      account.Root,
+		CodeHash:  account.CodeHash,
+		Code:      account.Code,
+		Storage:   account.Storage,
+		SecureKey: account.SecureKey,
+		Address:   nil,
+	}
+	if addr != (common.Address{}) {
+		dumpAccount.Address = &addr
+	}
+	(*json.Encoder)(&d).Encode(dumpAccount)
+}
+
+func (d iterativeDump) onRoot(root common.Hash) {
+	(*json.Encoder)(&d).Encode(struct {
+		Root common.Hash `json:"root"`
+	}{root})
+}
+
+func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
+	emptyAddress := (common.Address{})
+	missingPreimages := 0
+	c.onRoot(s.trie.Hash())
 	it := trie.NewIterator(s.trie.NodeIterator(nil))
 	for it.Next() {
-		addr := s.trie.GetKey(it.Key)
 		var data Account
 		if err := rlp.DecodeBytes(it.Value, &data); err != nil {
 			panic(err)
 		}
-
+		addr := common.BytesToAddress(s.trie.GetKey(it.Key))
-		obj := newObject(nil, common.BytesToAddress(addr), data)
+		obj := newObject(nil, addr, data)
 		account := DumpAccount{
 			Balance:  data.Balance.String(),
 			Nonce:    data.Nonce,
 			Root:     common.Bytes2Hex(data.Root[:]),
 			CodeHash: common.Bytes2Hex(data.CodeHash),
-			Code:     common.Bytes2Hex(obj.Code(s.db)),
-			Storage:  make(map[string]string),
 		}
-		storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil))
+		if emptyAddress == addr {
-		for storageIt.Next() {
+			// Preimage missing
-			account.Storage[common.Bytes2Hex(s.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
+			missingPreimages++
+			if excludeMissingPreimages {
+				continue
+			}
+			account.SecureKey = it.Key
 		}
-		dump.Accounts[common.Bytes2Hex(addr)] = account
+		if !excludeCode {
+			account.Code = common.Bytes2Hex(obj.Code(s.db))
+		}
+		if !excludeStorage {
+			account.Storage = make(map[common.Hash]string)
+			storageIt := trie.NewIterator(obj.getTrie(s.db).NodeIterator(nil))
+			for storageIt.Next() {
+				account.Storage[common.BytesToHash(s.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
+			}
+		}
+		c.onAccount(addr, account)
+	}
+	if missingPreimages > 0 {
+		log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
 	}
-	return dump
 }
 
-func (s *StateDB) Dump() []byte {
+// RawDump returns the entire state an a single large object
-	json, err := json.MarshalIndent(s.RawDump(), "", "    ")
+func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump {
+	dump := &Dump{
+		Accounts: make(map[common.Address]DumpAccount),
+	}
+	s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
+	return *dump
+}
+
+// Dump returns a JSON string representing the entire state as a single json-object
+func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte {
+	dump := s.RawDump(excludeCode, excludeStorage, excludeMissingPreimages)
+	json, err := json.MarshalIndent(dump, "", "    ")
 	if err != nil {
 		fmt.Println("dump err", err)
 	}
-
 	return json
 }
+
+// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
+func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
+	s.dump(iterativeDump(*output), excludeCode, excludeStorage, excludeMissingPreimages)
+}

core/state/state_test.go

@@ -51,33 +51,28 @@ func (s *StateSuite) TestDump(c *checker.C) {
 	s.state.Commit(false)
 
 	// check that dump contains the state objects that are in trie
-	got := string(s.state.Dump())
+	got := string(s.state.Dump(false, false, true))
 	want := `{
     "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2",
     "accounts": {
-        "0000000000000000000000000000000000000001": {
+        "0x0000000000000000000000000000000000000001": {
             "balance": "22",
             "nonce": 0,
             "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
-            "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+            "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
-            "code": "",
-            "storage": {}
         },
-        "0000000000000000000000000000000000000002": {
+        "0x0000000000000000000000000000000000000002": {
             "balance": "44",
             "nonce": 0,
             "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
-            "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+            "codeHash": "c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470"
-            "code": "",
-            "storage": {}
         },
-        "0000000000000000000000000000000000000102": {
+        "0x0000000000000000000000000000000000000102": {
             "balance": "0",
             "nonce": 0,
             "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
             "codeHash": "87874902497a5bb968da31a2998d8f22e949d1ef6214bcdedd8bae24cca4b9e3",
-            "code": "03030303030303",
+            "code": "03030303030303"
-            "storage": {}
         }
     }
 }`

core/state/statedb.go

@@ -596,15 +596,15 @@ func (s *StateDB) CreateAccount(addr common.Address) {
 	}
 }
 
-func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) error {
+func (s *StateDB) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) error {
-	so := db.getStateObject(addr)
+	so := s.getStateObject(addr)
 	if so == nil {
 		return nil
 	}
-	it := trie.NewIterator(so.getTrie(db.db).NodeIterator(nil))
+	it := trie.NewIterator(so.getTrie(s.db).NodeIterator(nil))
 
 	for it.Next() {
-		key := common.BytesToHash(db.trie.GetKey(it.Key))
+		key := common.BytesToHash(s.trie.GetKey(it.Key))
 		if value, dirty := so.dirtyStorage[key]; dirty {
 			if !cb(key, value) {
 				return nil

eth/api.go

@@ -297,7 +297,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) {
 	if err != nil {
 		return state.Dump{}, err
 	}
-	return stateDb.RawDump(), nil
+	return stateDb.RawDump(false, false, true), nil
 }
 
 // DebugAPI is the collection of Ethereum full node APIs exposed over