eth: add debug_accountRange API #19645 (#1194)

This new API allows reading accounts and their content by address range.

Co-authored-by: Wenbiao Zheng <delweng@gmail.com>
Co-authored-by: Martin Holst Swende <martin@swende.se>
Co-authored-by: Felix Lange <fjl@twurst.com>

core/state/dump.go

@@ -28,6 +28,7 @@ import (
 	"github.com/XinFinOrg/XDPoSChain/trie"
 )
 
+// DumpAccount represents an account in the state.
 type DumpAccount struct {
 	Balance   string                 `json:"balance"`
 	Nonce     uint64                 `json:"nonce"`
@@ -39,16 +40,24 @@ type DumpAccount struct {
 	SecureKey hexutil.Bytes          `json:"key,omitempty"`     // If we don't have address, we can output the key
 }
 
+// Dump represents the full dump in a collected format, as one large map.
 type Dump struct {
 	Root     string                         `json:"root"`
 	Accounts map[common.Address]DumpAccount `json:"accounts"`
 }
 
-// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively
+// iterativeDump is a 'collector'-implementation which dump output line-by-line iteratively.
 type iterativeDump struct {
 	*json.Encoder
 }
 
+// IteratorDump is an implementation for iterating over data.
+type IteratorDump struct {
+	Root     string                         `json:"root"`
+	Accounts map[common.Address]DumpAccount `json:"accounts"`
+	Next     []byte                         `json:"next,omitempty"` // nil if no more accounts
+}
+
 // Collector interface which the state trie calls during iteration
 type collector interface {
 	onRoot(common.Hash)
@@ -62,6 +71,13 @@ func (d *Dump) onRoot(root common.Hash) {
 func (d *Dump) onAccount(addr common.Address, account DumpAccount) {
 	d.Accounts[addr] = account
 }
+func (d *IteratorDump) onRoot(root common.Hash) {
+	d.Root = fmt.Sprintf("%x", root)
+}
+
+func (d *IteratorDump) onAccount(addr common.Address, account DumpAccount) {
+	d.Accounts[addr] = account
+}
 
 func (d iterativeDump) onAccount(addr common.Address, account DumpAccount) {
 	dumpAccount := &DumpAccount{
@@ -86,11 +102,13 @@ func (d iterativeDump) onRoot(root common.Hash) {
 	}{root})
 }
 
-func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool) {
+func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) (nextKey []byte) {
 	emptyAddress := (common.Address{})
 	missingPreimages := 0
 	c.onRoot(s.trie.Hash())
-	it := trie.NewIterator(s.trie.NodeIterator(nil))
+
+	var count int
+	it := trie.NewIterator(s.trie.NodeIterator(start))
 	for it.Next() {
 		var data types.StateAccount
 		if err := rlp.DecodeBytes(it.Value, &data); err != nil {
@@ -128,10 +146,19 @@ func (s *StateDB) dump(c collector, excludeCode, excludeStorage, excludeMissingP
 			}
 		}
 		c.onAccount(addr, account)
+		count++
+		if maxResults > 0 && count >= maxResults {
+			if it.Next() {
+				nextKey = it.Key
+			}
+			break
+		}
 	}
 	if missingPreimages > 0 {
 		log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
 	}
+
+	return nextKey
 }
 
 // RawDump returns the entire state an a single large object
@@ -139,7 +166,7 @@ func (s *StateDB) RawDump(excludeCode, excludeStorage, excludeMissingPreimages b
 	dump := &Dump{
 		Accounts: make(map[common.Address]DumpAccount),
 	}
-	s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages)
+	s.dump(dump, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
 	return *dump
 }
 
@@ -155,5 +182,14 @@ func (s *StateDB) Dump(excludeCode, excludeStorage, excludeMissingPreimages bool
 
 // IterativeDump dumps out accounts as json-objects, delimited by linebreaks on stdout
 func (s *StateDB) IterativeDump(excludeCode, excludeStorage, excludeMissingPreimages bool, output *json.Encoder) {
-	s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages)
+	s.dump(iterativeDump{output}, excludeCode, excludeStorage, excludeMissingPreimages, nil, 0)
+}
+
+// IteratorDump dumps out a batch of accounts starts with the given start key
+func (s *StateDB) IteratorDump(excludeCode, excludeStorage, excludeMissingPreimages bool, start []byte, maxResults int) IteratorDump {
+	iterator := &IteratorDump{
+		Accounts: make(map[common.Address]DumpAccount),
+	}
+	iterator.Next = s.dump(iterator, excludeCode, excludeStorage, excludeMissingPreimages, start, maxResults)
+	return *iterator
 }

eth/api.go

@@ -327,6 +327,52 @@ func (api *DebugAPI) GetBadBlocks(ctx context.Context) ([]core.BadBlockArgs, err
 	return api.eth.BlockChain().BadBlocks()
 }
 
+// AccountRangeMaxResults is the maximum number of results to be returned per call
+const AccountRangeMaxResults = 256
+
+// AccountRangeAt enumerates all accounts in the given block and start point in paging request
+func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start []byte, maxResults int, nocode, nostorage, incompletes bool) (state.IteratorDump, error) {
+	var stateDb *state.StateDB
+	var err error
+
+	if number, ok := blockNrOrHash.Number(); ok {
+		if number == rpc.PendingBlockNumber {
+			// If we're dumping the pending state, we need to request
+			// both the pending block as well as the pending state from
+			// the miner and operate on those
+			_, stateDb = api.eth.miner.Pending()
+		} else {
+			var block *types.Block
+			if number == rpc.LatestBlockNumber {
+				block = api.eth.blockchain.CurrentBlock()
+			} else {
+				block = api.eth.blockchain.GetBlockByNumber(uint64(number))
+			}
+			if block == nil {
+				return state.IteratorDump{}, fmt.Errorf("block #%d not found", number)
+			}
+			stateDb, err = api.eth.BlockChain().StateAt(block.Root())
+			if err != nil {
+				return state.IteratorDump{}, err
+			}
+		}
+	} else if hash, ok := blockNrOrHash.Hash(); ok {
+		block := api.eth.blockchain.GetBlockByHash(hash)
+		if block == nil {
+			return state.IteratorDump{}, fmt.Errorf("block %s not found", hash.Hex())
+		}
+		stateDb, err = api.eth.BlockChain().StateAt(block.Root())
+		if err != nil {
+			return state.IteratorDump{}, err
+		}
+	}
+
+	if maxResults > AccountRangeMaxResults || maxResults <= 0 {
+		maxResults = AccountRangeMaxResults
+	}
+	return stateDb.IteratorDump(nocode, nostorage, incompletes, start, maxResults), nil
+}
+
 // StorageRangeResult is the result of a debug_storageRangeAt API call.
 type StorageRangeResult struct {
 	Storage storageMap   `json:"storage"`
@@ -341,7 +387,7 @@ type storageEntry struct {
 }
 
 // StorageRangeAt returns the storage at the given block height and transaction index.
-func (api *DebugAPI) StorageRangeAt(ctx context.Context, blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) {
+func (api *DebugAPI) StorageRangeAt(blockHash common.Hash, txIndex int, contractAddress common.Address, keyStart hexutil.Bytes, maxResult int) (StorageRangeResult, error) {
 	_, _, statedb, err := api.computeTxEnv(blockHash, txIndex, 0)
 	if err != nil {
 		return StorageRangeResult{}, err

eth/api_test.go

@@ -17,11 +17,16 @@
 package eth
 
 import (
+	"bytes"
+	"fmt"
+	"math/big"
 	"reflect"
+	"sort"
 	"testing"
 
 	"github.com/XinFinOrg/XDPoSChain/core/rawdb"
 	"github.com/XinFinOrg/XDPoSChain/core/types"
+	"github.com/XinFinOrg/XDPoSChain/crypto"
 
 	"github.com/XinFinOrg/XDPoSChain/common"
 	"github.com/XinFinOrg/XDPoSChain/core/state"
@@ -30,6 +35,114 @@ import (
 
 var dumper = spew.ConfigState{Indent: "    "}
 
+func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, start common.Hash, requestedNum int, expectedNum int) state.IteratorDump {
+	result := statedb.IteratorDump(true, true, false, start.Bytes(), requestedNum)
+
+	if len(result.Accounts) != expectedNum {
+		t.Fatalf("expected %d results, got %d", expectedNum, len(result.Accounts))
+	}
+	for address := range result.Accounts {
+		if address == (common.Address{}) {
+			t.Fatalf("empty address returned")
+		}
+		if !statedb.Exist(address) {
+			t.Fatalf("account not found in state %s", address.Hex())
+		}
+	}
+	return result
+}
+
+type resultHash []common.Hash
+
+func (h resultHash) Len() int           { return len(h) }
+func (h resultHash) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
+func (h resultHash) Less(i, j int) bool { return bytes.Compare(h[i].Bytes(), h[j].Bytes()) < 0 }
+
+func TestAccountRange(t *testing.T) {
+	var (
+		statedb  = state.NewDatabase(rawdb.NewMemoryDatabase())
+		state, _ = state.New(common.Hash{}, statedb)
+		addrs    = [AccountRangeMaxResults * 2]common.Address{}
+		m        = map[common.Address]bool{}
+	)
+
+	for i := range addrs {
+		hash := common.HexToHash(fmt.Sprintf("%x", i))
+		addr := common.BytesToAddress(crypto.Keccak256Hash(hash.Bytes()).Bytes())
+		addrs[i] = addr
+		state.SetBalance(addrs[i], big.NewInt(1))
+		if _, ok := m[addr]; ok {
+			t.Fatalf("bad")
+		} else {
+			m[addr] = true
+		}
+	}
+	state.Commit(true)
+	root := state.IntermediateRoot(true)
+
+	trie, err := statedb.OpenTrie(root)
+	if err != nil {
+		t.Fatal(err)
+	}
+	accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults/2, AccountRangeMaxResults/2)
+	// test pagination
+	firstResult := accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults, AccountRangeMaxResults)
+	secondResult := accountRangeTest(t, &trie, state, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults)
+
+	hList := make(resultHash, 0)
+	for addr1 := range firstResult.Accounts {
+		// If address is empty, then it makes no sense to compare
+		// them as they might be two different accounts.
+		if addr1 == (common.Address{}) {
+			continue
+		}
+		if _, duplicate := secondResult.Accounts[addr1]; duplicate {
+			t.Fatalf("pagination test failed:  results should not overlap")
+		}
+		hList = append(hList, crypto.Keccak256Hash(addr1.Bytes()))
+	}
+	// Test to see if it's possible to recover from the middle of the previous
+	// set and get an even split between the first and second sets.
+	sort.Sort(hList)
+	middleH := hList[AccountRangeMaxResults/2]
+	middleResult := accountRangeTest(t, &trie, state, middleH, AccountRangeMaxResults, AccountRangeMaxResults)
+	missing, infirst, insecond := 0, 0, 0
+	for h := range middleResult.Accounts {
+		if _, ok := firstResult.Accounts[h]; ok {
+			infirst++
+		} else if _, ok := secondResult.Accounts[h]; ok {
+			insecond++
+		} else {
+			missing++
+		}
+	}
+	if missing != 0 {
+		t.Fatalf("%d hashes in the 'middle' set were neither in the first not the second set", missing)
+	}
+	if infirst != AccountRangeMaxResults/2 {
+		t.Fatalf("Imbalance in the number of first-test results: %d != %d", infirst, AccountRangeMaxResults/2)
+	}
+	if insecond != AccountRangeMaxResults/2 {
+		t.Fatalf("Imbalance in the number of second-test results: %d != %d", insecond, AccountRangeMaxResults/2)
+	}
+}
+
+func TestEmptyAccountRange(t *testing.T) {
+	var (
+		statedb  = state.NewDatabase(rawdb.NewMemoryDatabase())
+		state, _ = state.New(common.Hash{}, statedb)
+	)
+	state.Commit(true)
+	state.IntermediateRoot(true)
+	results := state.IteratorDump(true, true, true, (common.Hash{}).Bytes(), AccountRangeMaxResults)
+	if bytes.Equal(results.Next, (common.Hash{}).Bytes()) {
+		t.Fatalf("Empty results should not return a second page")
+	}
+	if len(results.Accounts) != 0 {
+		t.Fatalf("Empty state should not return addresses: %v", results.Accounts)
+	}
+}
+
 func TestStorageRangeAt(t *testing.T) {
 	// Create a state where account 0x010000... has a few storage entries.
 	var (