diff --git a/core/state/dump.go b/core/state/dump.go index b7aa04bd76..b60bc13f18 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -28,6 +28,7 @@ import ( "github.com/XinFinOrg/XDPoSChain/trie" ) +// DumpAccount represents an account in the state. type DumpAccount struct { Balance string `json:"balance"` Nonce uint64 `json:"nonce"` @@ -39,16 +40,24 @@ type DumpAccount struct { SecureKey hexutil.Bytes `json:"key,omitempty"` // If we don't have address, we can output the key } +// Dump represents the full dump in a collected format, as one large map. type Dump struct { Root string `json:"root"` Accounts map[common.Address]DumpAccount `json:"accounts"` } -// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively +// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively. type iterativeDump struct { *json.Encoder } +// IteratorDump is an implementation for iterating over data. +type IteratorDump struct { + Root string `json:"root"` + Accounts map[common.Address]DumpAccount `json:"accounts"` + Next []byte `json:"next,omitempty"` // nil if no more accounts +} + // Collector interface which the state trie calls during iteration type collector interface { onRoot(common.Hash) @@ -62,6 +71,13 @@ func (d *Dump) onRoot(root common.Hash) { func (d *Dump) onAccount(addr common.Address, account DumpAccount) { d.Accounts[addr] = account } +func (d *IteratorDump) onRoot(root common.Hash) { + d.Root = fmt.Sprintf("%x", root) +} + +func (d *IteratorDump) onAccount(addr common.Address, account DumpAccount) { + d.Accounts[addr] = account +} func (d iterativeDump) onAccount(addr common.Address, account DumpAccount) { dumpAccount := &DumpAccount{ @@ -86,11 +102,13 @@ func (d iterativeDump) onRoot(root common.Hash) { }{root}) } -func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) { +func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) { emptyAddress := (common.Address{}) missingPreimages := 0 c.onRoot(s.trie.Hash()) - it := trie.NewIterator(s.trie.NodeIterator(nil)) + + var count int + it := trie.NewIterator(s.trie.NodeIterator(start)) for it.Next() { var data types.StateAccount if err := rlp.DecodeBytes(it.Value, &data); err != nil { @@ -128,10 +146,19 @@ func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingP } } c.onAccount(addr, account) + count++ + if maxResults > 0 && count >= maxResults { + if it.Next() { + nextKey = it.Key + } + break + } } if missingPreimages > 0 { log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages) } + + return nextKey } // RawDump returns the entire state an a single large object @@ -139,7 +166,7 @@ func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages b dump := &Dump{ Accounts: make(map[common.Address]DumpAccount), } - s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages) + s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0) return *dump } @@ -155,5 +182,14 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool // IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) { - s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages) + s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0) +} + +// IteratorDump dumps out a batch of accounts starts with the given start key +func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump { + iterator := &IteratorDump{ + Accounts: make(map[common.Address]DumpAccount), + } + iterator.Next = s.dump(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults) + return *iterator } diff --git a/eth/api.go b/eth/api.go index d1c74638bf..fd74041200 100644 --- a/eth/api.go +++ b/eth/api.go @@ -327,6 +327,52 @@ func (api *DebugAPI) GetBadBlocks(ctx context.Context) ([]core.BadBlockArgs, err return api.eth.BlockChain().BadBlocks() } +// AccountRangeMaxResults is the maximum number of results to be returned per call +const AccountRangeMaxResults = 256 + +// AccountRangeAt enumerates all accounts in the given block and start point in paging request +func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start []byte, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) { + var stateDb *state.StateDB + var err error + + if number, ok := blockNrOrHash.Number(); ok { + if number == rpc.PendingBlockNumber { + // If we're dumping the pending state, we need to request + // both the pending block as well as the pending state from + // the miner and operate on those + _, stateDb = api.eth.miner.Pending() + } else { + var block *types.Block + if number == rpc.LatestBlockNumber { + block = api.eth.blockchain.CurrentBlock() + } else { + block = api.eth.blockchain.GetBlockByNumber(uint64(number)) + } + if block == nil { + return state.IteratorDump{}, fmt.Errorf("block #%d not found", number) + } + stateDb, err = api.eth.BlockChain().StateAt(block.Root()) + if err != nil { + return state.IteratorDump{}, err + } + } + } else if hash, ok := blockNrOrHash.Hash(); ok { + block := api.eth.blockchain.GetBlockByHash(hash) + if block == nil { + return state.IteratorDump{}, fmt.Errorf("block %s not found", hash.Hex()) + } + stateDb, err = api.eth.BlockChain().StateAt(block.Root()) + if err != nil { + return state.IteratorDump{}, err + } + } + + if maxResults > AccountRangeMaxResults || maxResults <= 0 { + maxResults = AccountRangeMaxResults + } + return stateDb.IteratorDump(nocode, nostorage, incompletes, start, maxResults), nil +} + // StorageRangeResult is the result of a debug_storageRangeAt API call. type StorageRangeResult struct { Storage storageMap `json:"storage"` @@ -341,7 +387,7 @@ type storageEntry struct { } // StorageRangeAt returns the storage at the given block height and transaction index. -func (api *DebugAPI) StorageRangeAt(ctx context.Context, blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { +func (api *DebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) { _, _, statedb, err := api.computeTxEnv(blockHash, txIndex, 0) if err != nil { return StorageRangeResult{}, err diff --git a/eth/api_test.go b/eth/api_test.go index 6c20f3d825..5ecb58b500 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -17,11 +17,16 @@ package eth import ( + "bytes" + "fmt" + "math/big" "reflect" + "sort" "testing" "github.com/XinFinOrg/XDPoSChain/core/rawdb" "github.com/XinFinOrg/XDPoSChain/core/types" + "github.com/XinFinOrg/XDPoSChain/crypto" "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/core/state" @@ -30,6 +35,114 @@ import ( var dumper = spew.ConfigState{Indent: " "} +func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump { + result := statedb.IteratorDump(true, true, false, start.Bytes(), requestedNum) + + if len(result.Accounts) != expectedNum { + t.Fatalf("expected %d results, got %d", expectedNum, len(result.Accounts)) + } + for address := range result.Accounts { + if address == (common.Address{}) { + t.Fatalf("empty address returned") + } + if !statedb.Exist(address) { + t.Fatalf("account not found in state %s", address.Hex()) + } + } + return result +} + +type resultHash []common.Hash + +func (h resultHash) Len() int { return len(h) } +func (h resultHash) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h resultHash) Less(i, j int) bool { return bytes.Compare(h[i].Bytes(), h[j].Bytes()) < 0 } + +func TestAccountRange(t *testing.T) { + var ( + statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) + state, _ = state.New(common.Hash{}, statedb) + addrs = [AccountRangeMaxResults * 2]common.Address{} + m = map[common.Address]bool{} + ) + + for i := range addrs { + hash := common.HexToHash(fmt.Sprintf("%x", i)) + addr := common.BytesToAddress(crypto.Keccak256Hash(hash.Bytes()).Bytes()) + addrs[i] = addr + state.SetBalance(addrs[i], big.NewInt(1)) + if _, ok := m[addr]; ok { + t.Fatalf("bad") + } else { + m[addr] = true + } + } + state.Commit(true) + root := state.IntermediateRoot(true) + + trie, err := statedb.OpenTrie(root) + if err != nil { + t.Fatal(err) + } + accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults/2, AccountRangeMaxResults/2) + // test pagination + firstResult := accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults, AccountRangeMaxResults) + secondResult := accountRangeTest(t, &trie, state, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults) + + hList := make(resultHash, 0) + for addr1 := range firstResult.Accounts { + // If address is empty, then it makes no sense to compare + // them as they might be two different accounts. + if addr1 == (common.Address{}) { + continue + } + if _, duplicate := secondResult.Accounts[addr1]; duplicate { + t.Fatalf("pagination test failed: results should not overlap") + } + hList = append(hList, crypto.Keccak256Hash(addr1.Bytes())) + } + // Test to see if it's possible to recover from the middle of the previous + // set and get an even split between the first and second sets. + sort.Sort(hList) + middleH := hList[AccountRangeMaxResults/2] + middleResult := accountRangeTest(t, &trie, state, middleH, AccountRangeMaxResults, AccountRangeMaxResults) + missing, infirst, insecond := 0, 0, 0 + for h := range middleResult.Accounts { + if _, ok := firstResult.Accounts[h]; ok { + infirst++ + } else if _, ok := secondResult.Accounts[h]; ok { + insecond++ + } else { + missing++ + } + } + if missing != 0 { + t.Fatalf("%d hashes in the 'middle' set were neither in the first not the second set", missing) + } + if infirst != AccountRangeMaxResults/2 { + t.Fatalf("Imbalance in the number of first-test results: %d != %d", infirst, AccountRangeMaxResults/2) + } + if insecond != AccountRangeMaxResults/2 { + t.Fatalf("Imbalance in the number of second-test results: %d != %d", insecond, AccountRangeMaxResults/2) + } +} + +func TestEmptyAccountRange(t *testing.T) { + var ( + statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) + state, _ = state.New(common.Hash{}, statedb) + ) + state.Commit(true) + state.IntermediateRoot(true) + results := state.IteratorDump(true, true, true, (common.Hash{}).Bytes(), AccountRangeMaxResults) + if bytes.Equal(results.Next, (common.Hash{}).Bytes()) { + t.Fatalf("Empty results should not return a second page") + } + if len(results.Accounts) != 0 { + t.Fatalf("Empty state should not return addresses: %v", results.Accounts) + } +} + func TestStorageRangeAt(t *testing.T) { // Create a state where account 0x010000... has a few storage entries. var (