core/state, cmd/geth: streaming json output dump cmd + optional code+storage

cmd/evm/runner.go

@@ -209,7 +209,7 @@ func runCmd(ctx *cli.Context) error {
 	if ctx.GlobalBool(DumpFlag.Name) {
 		statedb.Commit(true)
 		statedb.IntermediateRoot(true)
-		fmt.Println(string(statedb.Dump()))
+		fmt.Println(string(statedb.Dump(false, false)))
 	}
 
 	if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" {

cmd/evm/staterunner.go

@@ -105,7 +105,7 @@ func stateTestCmd(ctx *cli.Context) error {
 				// Test failed, mark as so and dump any state to aid debugging
 				result.Pass, result.Error = false, err.Error()
 				if ctx.GlobalBool(DumpFlag.Name) && state != nil {
-					dump := state.RawDump()
+					dump := state.RawDump(false, false)
 					result.State = &dump
 				}
 			}

cmd/geth/chaincmd.go

@@ -162,6 +162,9 @@ Remove blockchain and state databases`,
 			utils.DataDirFlag,
 			utils.CacheFlag,
 			utils.SyncModeFlag,
+			utils.IterativeOutputFlag,
+			utils.ExcludeCodeFlag,
+			utils.ExcludeStorageFlag,
 		},
 		Category: "BLOCKCHAIN COMMANDS",
 		Description: `
@@ -287,7 +290,7 @@ func importChain(ctx *cli.Context) error {
 	fmt.Printf("Allocations:   %.3f million\n", float64(mem.Mallocs)/1000000)
 	fmt.Printf("GC pause:      %v\n\n", time.Duration(mem.PauseTotalNs))
 
-	if ctx.GlobalIsSet(utils.NoCompactionFlag.Name) {
+	if ctx.GlobalBool(utils.NoCompactionFlag.Name) {
 		return nil
 	}
 
@@ -520,7 +523,13 @@ func dump(ctx *cli.Context) error {
 			if err != nil {
 				utils.Fatalf("could not create new state: %v", err)
 			}
-			fmt.Printf("%s\n", state.Dump())
+			excludeCode := ctx.GlobalBool(utils.ExcludeCodeFlag.Name)
+			excludeStorage := ctx.GlobalBool(utils.ExcludeStorageFlag.Name)
+			if ctx.GlobalBool(utils.IterativeOutputFlag.Name) {
+				state.IterativeDump(excludeCode, excludeStorage, json.NewEncoder(os.Stdout))
+			} else {
+				fmt.Printf("%s\n", state.Dump(excludeCode, excludeStorage))
+			}
 		}
 	}
 	chainDb.Close()

cmd/geth/main.go

@@ -188,6 +188,12 @@ var (
 		utils.MetricsInfluxDBPasswordFlag,
 		utils.MetricsInfluxDBTagsFlag,
 	}
+
+	dumpFlags = []cli.Flag{
+		utils.IterativeOutputFlag,
+		utils.ExcludeCodeFlag,
+		utils.ExcludeStorageFlag,
+	}
 )
 
 func init() {
@@ -231,6 +237,7 @@ func init() {
 	app.Flags = append(app.Flags, debug.Flags...)
 	app.Flags = append(app.Flags, whisperFlags...)
 	app.Flags = append(app.Flags, metricsFlags...)
+	app.Flags = append(app.Flags, dumpFlags...)
 
 	app.Before = func(ctx *cli.Context) error {
 		logdir := ""

cmd/utils/flags.go

@@ -196,6 +196,18 @@ var (
 		Name:  "ulc.trusted",
 		Usage: "List of trusted ULC servers",
 	}
+	IterativeOutputFlag = cli.BoolFlag{
+		Name:  "dump.iterative",
+		Usage: "Print streaming JSON iteratively as json objects, delimited by newlines",
+	}
+	ExcludeStorageFlag = cli.BoolFlag{
+		Name:  "dump.nostorage",
+		Usage: "When set, exclude storage entries (saves db lookups)",
+	}
+	ExcludeCodeFlag = cli.BoolFlag{
+		Name:  "dump.nocode",
+		Usage: "When set, exclude contract code (saves db lookups)",
+	}
 	defaultSyncMode = eth.DefaultConfig.SyncMode
 	SyncModeFlag    = TextMarshalerFlag{
 		Name:  "syncmode",

core/state/dump.go

@@ -21,61 +21,128 @@ import (
 	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
 )
 
+// DumpAccount represents an account in the state
 type DumpAccount struct {
 	Balance  string                 `json:"balance"`
 	Nonce    uint64                 `json:"nonce"`
 	Root     string                 `json:"root"`
 	CodeHash string                 `json:"codeHash"`
 	Code     string                 `json:"code"`
-	Storage  map[string]string `json:"storage"`
+	Storage  map[common.Hash]string `json:"storage"`
+	Address  *common.Address        `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode
 }
 
+// Dump represents the full dump in a collected format, as one large map
 type Dump struct {
 	Root     string                         `json:"root"`
-	Accounts map[string]DumpAccount `json:"accounts"`
+	Accounts map[common.Address]DumpAccount `json:"accounts"`
 }
 
-func (self *StateDB) RawDump() Dump {
-	dump := Dump{
-		Root:     fmt.Sprintf("%x", self.trie.Hash()),
-		Accounts: make(map[string]DumpAccount),
-	}
+// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
+type iterativeDump json.Encoder
 
+// Collector interface which the state trie calls during iteration
+type collector interface {
+	onRoot(common.Hash)
+	onAccount(common.Address, DumpAccount)
+}
+
+func (self *Dump) onRoot(root common.Hash) {
+	self.Root = fmt.Sprintf("%x", root)
+}
+
+func (self *Dump) onAccount(addr common.Address, account DumpAccount) {
+	self.Accounts[addr] = account
+}
+
+func (self iterativeDump) onAccount(addr common.Address, account DumpAccount) {
+	dumpAccount := &DumpAccount{
+		Balance:  account.Balance,
+		Nonce:    account.Nonce,
+		Root:     account.Root,
+		CodeHash: account.CodeHash,
+		Code:     account.Code,
+		Storage:  account.Storage,
+		Address:  nil,
+	}
+	if addr != (common.Address{}) {
+		dumpAccount.Address = &addr
+	}
+	(*json.Encoder)(&self).Encode(dumpAccount)
+}
+func (self iterativeDump) onRoot(root common.Hash) {
+	(*json.Encoder)(&self).Encode(struct {
+		Root common.Hash `json:"root"`
+	}{root})
+}
+
+func (self *StateDB) dump(c collector, excludeCode, excludeStorage bool) {
+	emptyAddress := (common.Address{})
+	missingPreimages := 0
+	c.onRoot(self.trie.Hash())
 	it := trie.NewIterator(self.trie.NodeIterator(nil))
 	for it.Next() {
-		addr := self.trie.GetKey(it.Key)
+		addr := common.BytesToAddress(self.trie.GetKey(it.Key))
+		if emptyAddress == addr {
+			// We don't have the preimage. All accounts missing preimages
+			// will be 'mapped' and overwrite the same entry, which is quite useless.
+			// Make note and continue
+			missingPreimages++
+			continue
+		}
 		var data Account
 		if err := rlp.DecodeBytes(it.Value, &data); err != nil {
 			panic(err)
 		}
-
-		obj := newObject(nil, common.BytesToAddress(addr), data)
+		obj := newObject(nil, addr, data)
 		account := DumpAccount{
 			Balance:  data.Balance.String(),
 			Nonce:    data.Nonce,
 			Root:     common.Bytes2Hex(data.Root[:]),
 			CodeHash: common.Bytes2Hex(data.CodeHash),
-			Code:     common.Bytes2Hex(obj.Code(self.db)),
-			Storage:  make(map[string]string),
 		}
+		if !excludeCode {
+			account.Code = common.Bytes2Hex(obj.Code(self.db))
+		}
+		if !excludeStorage {
+			account.Storage = make(map[common.Hash]string)
 			storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil))
 			for storageIt.Next() {
-			account.Storage[common.Bytes2Hex(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
+				account.Storage[common.BytesToHash(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value)
 			}
-		dump.Accounts[common.Bytes2Hex(addr)] = account
 		}
-	return dump
+		c.onAccount(addr, account)
+	}
+	if missingPreimages > 0 {
+		log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
+	}
 }
 
-func (self *StateDB) Dump() []byte {
-	json, err := json.MarshalIndent(self.RawDump(), "", "    ")
+// RawDump returns the entire state an a single large object
+func (self *StateDB) RawDump(excludeCode, excludeStorage bool) Dump {
+	dump := &Dump{
+		Accounts: make(map[common.Address]DumpAccount),
+	}
+	self.dump(dump, excludeCode, excludeStorage)
+	return *dump
+}
+
+// Dump returns a JSON string representing the entire state as a single json-object
+func (self *StateDB) Dump(excludeCode, excludeStorage bool) []byte {
+	dump := self.RawDump(excludeCode, excludeStorage)
+	json, err := json.MarshalIndent(dump, "", "  ")
 	if err != nil {
 		fmt.Println("dump err", err)
 	}
-
 	return json
 }
+
+// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
+func (self *StateDB) IterativeDump(excludeCode, excludeStorage bool, output *json.Encoder) {
+	self.dump(iterativeDump(*output), excludeCode, excludeStorage)
+}

core/state/state_test.go

@@ -52,7 +52,7 @@ func (s *StateSuite) TestDump(c *checker.C) {
 	s.state.Commit(false)
 
 	// check that dump contains the state objects that are in trie
-	got := string(s.state.Dump())
+	got := string(s.state.Dump(false, false))
 	want := `{
     "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2",
     "accounts": {

eth/api.go

@@ -266,7 +266,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error
 		// both the pending block as well as the pending state from
 		// the miner and operate on those
 		_, stateDb := api.eth.miner.Pending()
-		return stateDb.RawDump(), nil
+		return stateDb.RawDump(false, false), nil
 	}
 	var block *types.Block
 	if blockNr == rpc.LatestBlockNumber {
@@ -281,7 +281,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error
 	if err != nil {
 		return state.Dump{}, err
 	}
-	return stateDb.RawDump(), nil
+	return stateDb.RawDump(false, false), nil
 }
 
 // PrivateDebugAPI is the collection of Ethereum full node APIs exposed over