diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index bc5d00cfbe..14634c2abe 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -209,7 +209,7 @@ func runCmd(ctx *cli.Context) error { if ctx.GlobalBool(DumpFlag.Name) { statedb.Commit(true) statedb.IntermediateRoot(true) - fmt.Println(string(statedb.Dump())) + fmt.Println(string(statedb.Dump(false, false))) } if memProfilePath := ctx.GlobalString(MemProfileFlag.Name); memProfilePath != "" { diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index b3c69d9b9d..93d6acb968 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -105,7 +105,7 @@ func stateTestCmd(ctx *cli.Context) error { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() if ctx.GlobalBool(DumpFlag.Name) && state != nil { - dump := state.RawDump() + dump := state.RawDump(false, false) result.State = &dump } } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index c91545c7fc..9d681cb0d9 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -162,6 +162,9 @@ Remove blockchain and state databases`, utils.DataDirFlag, utils.CacheFlag, utils.SyncModeFlag, + utils.IterativeOutputFlag, + utils.ExcludeCodeFlag, + utils.ExcludeStorageFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` @@ -287,7 +290,7 @@ func importChain(ctx *cli.Context) error { fmt.Printf("Allocations: %.3f million\n", float64(mem.Mallocs)/1000000) fmt.Printf("GC pause: %v\n\n", time.Duration(mem.PauseTotalNs)) - if ctx.GlobalIsSet(utils.NoCompactionFlag.Name) { + if ctx.GlobalBool(utils.NoCompactionFlag.Name) { return nil } @@ -520,7 +523,13 @@ func dump(ctx *cli.Context) error { if err != nil { utils.Fatalf("could not create new state: %v", err) } - fmt.Printf("%s\n", state.Dump()) + excludeCode := ctx.GlobalBool(utils.ExcludeCodeFlag.Name) + excludeStorage := ctx.GlobalBool(utils.ExcludeStorageFlag.Name) + if ctx.GlobalBool(utils.IterativeOutputFlag.Name) { + state.IterativeDump(excludeCode, excludeStorage, json.NewEncoder(os.Stdout)) + } else { + fmt.Printf("%s\n", state.Dump(excludeCode, excludeStorage)) + } } } chainDb.Close() diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 00809e2e10..264a7a65c5 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -188,6 +188,12 @@ var ( utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, } + + dumpFlags = []cli.Flag{ + utils.IterativeOutputFlag, + utils.ExcludeCodeFlag, + utils.ExcludeStorageFlag, + } ) func init() { @@ -231,6 +237,7 @@ func init() { app.Flags = append(app.Flags, debug.Flags...) app.Flags = append(app.Flags, whisperFlags...) app.Flags = append(app.Flags, metricsFlags...) + app.Flags = append(app.Flags, dumpFlags...) app.Before = func(ctx *cli.Context) error { logdir := "" diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8c5a0c99a0..b4b9ef514e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -196,6 +196,18 @@ var ( Name: "ulc.trusted", Usage: "List of trusted ULC servers", } + IterativeOutputFlag = cli.BoolFlag{ + Name: "dump.iterative", + Usage: "Print streaming JSON iteratively as json objects, delimited by newlines", + } + ExcludeStorageFlag = cli.BoolFlag{ + Name: "dump.nostorage", + Usage: "When set, exclude storage entries (saves db lookups)", + } + ExcludeCodeFlag = cli.BoolFlag{ + Name: "dump.nocode", + Usage: "When set, exclude contract code (saves db lookups)", + } defaultSyncMode = eth.DefaultConfig.SyncMode SyncModeFlag = TextMarshalerFlag{ Name: "syncmode", diff --git a/core/state/dump.go b/core/state/dump.go index 072dbbf053..9efde09369 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -21,61 +21,128 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) +// DumpAccount represents an account in the state type DumpAccount struct { - Balance string `json:"balance"` - Nonce uint64 `json:"nonce"` - Root string `json:"root"` - CodeHash string `json:"codeHash"` - Code string `json:"code"` - Storage map[string]string `json:"storage"` -} - -type Dump struct { + Balance string `json:"balance"` + Nonce uint64 `json:"nonce"` Root string `json:"root"` - Accounts map[string]DumpAccount `json:"accounts"` + CodeHash string `json:"codeHash"` + Code string `json:"code"` + Storage map[common.Hash]string `json:"storage"` + Address *common.Address `json:"address,omitempty"` // Address only present in iterative (line-by-line) mode } -func (self *StateDB) RawDump() Dump { - dump := Dump{ - Root: fmt.Sprintf("%x", self.trie.Hash()), - Accounts: make(map[string]DumpAccount), - } +// Dump represents the full dump in a collected format, as one large map +type Dump struct { + Root string `json:"root"` + Accounts map[common.Address]DumpAccount `json:"accounts"` +} +// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively +type iterativeDump json.Encoder + +// Collector interface which the state trie calls during iteration +type collector interface { + onRoot(common.Hash) + onAccount(common.Address, DumpAccount) +} + +func (self *Dump) onRoot(root common.Hash) { + self.Root = fmt.Sprintf("%x", root) +} + +func (self *Dump) onAccount(addr common.Address, account DumpAccount) { + self.Accounts[addr] = account +} + +func (self iterativeDump) onAccount(addr common.Address, account DumpAccount) { + dumpAccount := &DumpAccount{ + Balance: account.Balance, + Nonce: account.Nonce, + Root: account.Root, + CodeHash: account.CodeHash, + Code: account.Code, + Storage: account.Storage, + Address: nil, + } + if addr != (common.Address{}) { + dumpAccount.Address = &addr + } + (*json.Encoder)(&self).Encode(dumpAccount) +} +func (self iterativeDump) onRoot(root common.Hash) { + (*json.Encoder)(&self).Encode(struct { + Root common.Hash `json:"root"` + }{root}) +} + +func (self *StateDB) dump(c collector, excludeCode, excludeStorage bool) { + emptyAddress := (common.Address{}) + missingPreimages := 0 + c.onRoot(self.trie.Hash()) it := trie.NewIterator(self.trie.NodeIterator(nil)) for it.Next() { - addr := self.trie.GetKey(it.Key) + addr := common.BytesToAddress(self.trie.GetKey(it.Key)) + if emptyAddress == addr { + // We don't have the preimage. All accounts missing preimages + // will be 'mapped' and overwrite the same entry, which is quite useless. + // Make note and continue + missingPreimages++ + continue + } var data Account if err := rlp.DecodeBytes(it.Value, &data); err != nil { panic(err) } - - obj := newObject(nil, common.BytesToAddress(addr), data) + obj := newObject(nil, addr, data) account := DumpAccount{ Balance: data.Balance.String(), Nonce: data.Nonce, Root: common.Bytes2Hex(data.Root[:]), CodeHash: common.Bytes2Hex(data.CodeHash), - Code: common.Bytes2Hex(obj.Code(self.db)), - Storage: make(map[string]string), } - storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil)) - for storageIt.Next() { - account.Storage[common.Bytes2Hex(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value) + if !excludeCode { + account.Code = common.Bytes2Hex(obj.Code(self.db)) } - dump.Accounts[common.Bytes2Hex(addr)] = account + if !excludeStorage { + account.Storage = make(map[common.Hash]string) + storageIt := trie.NewIterator(obj.getTrie(self.db).NodeIterator(nil)) + for storageIt.Next() { + account.Storage[common.BytesToHash(self.trie.GetKey(storageIt.Key))] = common.Bytes2Hex(storageIt.Value) + } + } + c.onAccount(addr, account) + } + if missingPreimages > 0 { + log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) } - return dump } -func (self *StateDB) Dump() []byte { - json, err := json.MarshalIndent(self.RawDump(), "", " ") +// RawDump returns the entire state an a single large object +func (self *StateDB) RawDump(excludeCode, excludeStorage bool) Dump { + dump := &Dump{ + Accounts: make(map[common.Address]DumpAccount), + } + self.dump(dump, excludeCode, excludeStorage) + return *dump +} + +// Dump returns a JSON string representing the entire state as a single json-object +func (self *StateDB) Dump(excludeCode, excludeStorage bool) []byte { + dump := self.RawDump(excludeCode, excludeStorage) + json, err := json.MarshalIndent(dump, "", " ") if err != nil { fmt.Println("dump err", err) } - return json } + +// IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout +func (self *StateDB) IterativeDump(excludeCode, excludeStorage bool, output *json.Encoder) { + self.dump(iterativeDump(*output), excludeCode, excludeStorage) +} diff --git a/core/state/state_test.go b/core/state/state_test.go index 606f2a6f6e..f11391d643 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -52,7 +52,7 @@ func (s *StateSuite) TestDump(c *checker.C) { s.state.Commit(false) // check that dump contains the state objects that are in trie - got := string(s.state.Dump()) + got := string(s.state.Dump(false, false)) want := `{ "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2", "accounts": { diff --git a/eth/api.go b/eth/api.go index bdbbd1ba38..87ebc0bff0 100644 --- a/eth/api.go +++ b/eth/api.go @@ -266,7 +266,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error // both the pending block as well as the pending state from // the miner and operate on those _, stateDb := api.eth.miner.Pending() - return stateDb.RawDump(), nil + return stateDb.RawDump(false, false), nil } var block *types.Block if blockNr == rpc.LatestBlockNumber { @@ -281,7 +281,7 @@ func (api *PublicDebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error if err != nil { return state.Dump{}, err } - return stateDb.RawDump(), nil + return stateDb.RawDump(false, false), nil } // PrivateDebugAPI is the collection of Ethereum full node APIs exposed over