dump: add option to continue even if preimages are missing

cmd/evm/staterunner.go

@@ -105,7 +105,7 @@ func stateTestCmd(ctx *cli.Context) error {
 				// Test failed, mark as so and dump any state to aid debugging
 				result.Pass, result.Error = false, err.Error()
 				if ctx.GlobalBool(DumpFlag.Name) && state != nil {
-					dump := state.RawDump(false, false)
+					dump := state.RawDump(false, false, true)
 					result.State = &dump
 				}
 			}

cmd/geth/chaincmd.go

@@ -165,6 +165,7 @@ Remove blockchain and state databases`,
 			utils.IterativeOutputFlag,
 			utils.ExcludeCodeFlag,
 			utils.ExcludeStorageFlag,
+			utils.IncludeMissingPreimagesFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -507,6 +508,7 @@ func dump(ctx *cli.Context) error {
 	defer stack.Close()
 
 	chain, chainDb := utils.MakeChain(ctx, stack)
+	defer chainDb.Close()
 	for _, arg := range ctx.Args() {
 		var block *types.Block
 		if hashish(arg) {
@@ -525,14 +527,18 @@ func dump(ctx *cli.Context) error {
 			}
 			excludeCode := ctx.GlobalBool(utils.ExcludeCodeFlag.Name)
 			excludeStorage := ctx.GlobalBool(utils.ExcludeStorageFlag.Name)
+			includeMissing := ctx.GlobalBool(utils.IncludeMissingPreimagesFlag.Name)
 			if ctx.GlobalBool(utils.IterativeOutputFlag.Name) {
-				state.IterativeDump(excludeCode, excludeStorage, json.NewEncoder(os.Stdout))
+				state.IterativeDump(excludeCode, excludeStorage, !includeMissing, json.NewEncoder(os.Stdout))
 			} else {
-				fmt.Printf("%s\n", state.Dump(excludeCode, excludeStorage))
+				if includeMissing {
+					fmt.Printf("If you want to include accounts with missing preimages, you need iterative output, since" +
+						" otherwise the accounts will overwrite each other in the resulting mapping.")
+				}
+				fmt.Printf("%v %s\n", includeMissing, state.Dump(excludeCode, excludeStorage, false))
 			}
 		}
 	}
-	chainDb.Close()
 	return nil
 }
 

cmd/geth/main.go

@@ -193,6 +193,7 @@ var (
 		utils.IterativeOutputFlag,
 		utils.ExcludeCodeFlag,
 		utils.ExcludeStorageFlag,
+		utils.IncludeMissingPreimagesFlag,
 	}
 )
 

cmd/utils/flags.go

@@ -204,6 +204,10 @@ var (
 		Name:  "dump.nostorage",
 		Usage: "When set, exclude storage entries (saves db lookups)",
 	}
+	IncludeMissingPreimagesFlag = cli.BoolFlag{
+		Name:  "dump.includeincomplete",
+		Usage: "When set, include also those we do not have address of (missing preimage)",
+	}
 	ExcludeCodeFlag = cli.BoolFlag{
 		Name:  "dump.nocode",
 		Usage: "When set, exclude contract code (saves db lookups)",

core/state/dump.go

@@ -19,6 +19,7 @@ package state
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
@@ -28,13 +29,15 @@ import (
 
 // DumpAccount represents an account in the state
 type DumpAccount struct {
-	Balance  string                 `json:"balance"`
-	Nonce    uint64                 `json:"nonce"`
-	Root     string                 `json:"root"`
-	CodeHash string                 `json:"codeHash"`
-	Code     string                 `json:"code"`
-	Storage  map[common.Hash]string `json:"storage"`
-	Address  *common.Address        `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode
+	Balance   string                 `json:"balance"`
+	Nonce     uint64                 `json:"nonce"`
+	Root      string                 `json:"root"`
+	CodeHash  string                 `json:"codeHash"`
+	Code      string                 `json:"code"`
+	Storage   map[common.Hash]string `json:"storage"`
+	Address   *common.Address        `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode
+	SecureKey hexutil.Bytes          `json:"key,omitempty"`     // If we don't have address, we can output the key
+
 }
 
 // Dump represents the full dump in a collected format, as one large map
@@ -62,13 +65,14 @@ func (self *Dump) onAccount(addr common.Address, account DumpAccount) {
 
 func (self iterativeDump) onAccount(addr common.Address, account DumpAccount) {
 	dumpAccount := &DumpAccount{
-		Balance:  account.Balance,
-		Nonce:    account.Nonce,
-		Root:     account.Root,
-		CodeHash: account.CodeHash,
-		Code:     account.Code,
-		Storage:  account.Storage,
-		Address:  nil,
+		Balance:   account.Balance,
+		Nonce:     account.Nonce,
+		Root:      account.Root,
+		CodeHash:  account.CodeHash,
+		Code:      account.Code,
+		Storage:   account.Storage,
+		SecureKey: account.SecureKey,
+		Address:   nil,
 	}
 	if addr != (common.Address{}) {
 		dumpAccount.Address = &addr
@@ -81,24 +85,17 @@ func (self iterativeDump) onRoot(root common.Hash) {
 	}{root})
 }
 
-func (self *StateDB) dump(c collector, excludeCode, excludeStorage bool) {
+func (self *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
 	emptyAddress := (common.Address{})
 	missingPreimages := 0
 	c.onRoot(self.trie.Hash())
 	it := trie.NewIterator(self.trie.NodeIterator(nil))
 	for it.Next() {
-		addr := common.BytesToAddress(self.trie.GetKey(it.Key))
-		if emptyAddress == addr {
-			// We don't have the preimage. All accounts missing preimages
-			// will be 'mapped' and overwrite the same entry, which is quite useless.
-			// Make note and continue
-			missingPreimages++
-			continue
-		}
 		var data Account
 		if err := rlp.DecodeBytes(it.Value, &data); err != nil {
 			panic(err)
 		}
+		addr := common.BytesToAddress(self.trie.GetKey(it.Key))
 		obj := newObject(nil, addr, data)
 		account := DumpAccount{
 			Balance:  data.Balance.String(),
@@ -106,6 +103,14 @@ func (self *StateDB) dump(c collector, excludeCode, excludeStorage bool) {
 			Root:     common.Bytes2Hex(data.Root[:]),
 			CodeHash: common.Bytes2Hex(data.CodeHash),
 		}
+		if emptyAddress == addr {
+			// Preimage missing
+			missingPreimages++
+			if excludeMissingPreimages {
+				continue
+			}
+			account.SecureKey = it.Key
+		}
 		if !excludeCode {
 			account.Code = common.Bytes2Hex(obj.Code(self.db))
 		}
@@ -124,18 +129,18 @@ func (self *StateDB) dump(c collector, excludeCode, excludeStorage bool) {
 }
 
 // RawDump returns the entire state an a single large object
-func (self *StateDB) RawDump(excludeCode, excludeStorage bool) Dump {
+func (self *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages bool) Dump {
 	dump := &Dump{
 		Accounts: make(map[common.Address]DumpAccount),
 	}
-	self.dump(dump, excludeCode, excludeStorage)
+	self.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
 	return *dump
 }
 
 // Dump returns a JSON string representing the entire state as a single json-object
-func (self *StateDB) Dump(excludeCode, excludeStorage bool) []byte {
-	dump := self.RawDump(excludeCode, excludeStorage)
-	json, err := json.MarshalIndent(dump, "", "  ")
+func (self *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool) []byte {
+	dump := self.RawDump(excludeCode, excludeStorage, excludeMissingPreimages)
+	json, err := json.MarshalIndent(dump, "", "    ")
 	if err != nil {
 		fmt.Println("dump err", err)
 	}
@@ -143,6 +148,6 @@ func (self *StateDB) Dump(excludeCode, excludeStorage bool) []byte {
 }
 
 // IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
-func (self *StateDB) IterativeDump(excludeCode, excludeStorage bool, output *json.Encoder) {
-	self.dump(iterativeDump(*output), excludeCode, excludeStorage)
+func (self *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
+	self.dump(iterativeDump(*output), excludeCode, excludeStorage, excludeMissingPreimages)
 }

core/state/state_test.go

@@ -52,11 +52,11 @@ func (s *StateSuite) TestDump(c *checker.C) {
 	s.state.Commit(false)
 
 	// check that dump contains the state objects that are in trie
-	got := string(s.state.Dump(false, false))
+	got := string(s.state.Dump(false, false, true))
 	want := `{
     "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2",
     "accounts": {
-        "0000000000000000000000000000000000000001": {
+        "0x0000000000000000000000000000000000000001": {
             "balance": "22",
             "nonce": 0,
             "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
@@ -64,7 +64,7 @@ func (s *StateSuite) TestDump(c *checker.C) {
             "code": "",
             "storage": {}
         },
-        "0000000000000000000000000000000000000002": {
+        "0x0000000000000000000000000000000000000002": {
             "balance": "44",
             "nonce": 0,
             "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
@@ -72,7 +72,7 @@ func (s *StateSuite) TestDump(c *checker.C) {
             "code": "",
             "storage": {}
         },
-        "0000000000000000000000000000000000000102": {
+        "0x0000000000000000000000000000000000000102": {
             "balance": "0",
             "nonce": 0,
             "root": "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",

eth/api.go

@@ -266,7 +266,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error
 		// both the pending block as well as the pending state from
 		// the miner and operate on those
 		_, stateDb := api.eth.miner.Pending()
-		return stateDb.RawDump(false, false), nil
+		return stateDb.RawDump(false, false, true), nil
 	}
 	var block *types.Block
 	if blockNr == rpc.LatestBlockNumber {
@@ -281,7 +281,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error
 	if err != nil {
 		return state.Dump{}, err
 	}
-	return stateDb.RawDump(false, false), nil
+	return stateDb.RawDump(false, false, true), nil
 }
 
 // PrivateDebugAPI is the collection of Ethereum full node APIs exposed over