forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-04-03 08:35:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

core/state: introduce state iterator interface (#33102)

Browse source 
In this PR, the Database interface in `core/state` has been extended
with one more function:

```go
	// Iteratee returns a state iteratee associated with the specified state root,
	// through which the account iterator and storage iterator can be created.
	Iteratee(root common.Hash) (Iteratee, error)
```

With this additional abstraction layer, the implementation details can be hidden
behind the interface. For example, state traversal can now operate directly on 
the flat state for Verkle or binary trees, which do not natively support traversal.

Moreover, state dumping will now prefer using the flat state iterator as
the primary option, offering better efficiency.


Edit: this PR also fixes a tiny issue in the state dump, marshalling the
next field in the correct way.
This commit is contained in:
rjl493456442 2026-04-03 10:35:32 +08:00 committed by GitHub
parent bcb0efd756
commit 0ba4314321
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 763 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
core/state/database.go
View file

@ -39,6 +39,10 @@ type Database interface {
// Reader returns a state reader associated with the specified state root.
Reader(root common.Hash) (Reader, error)
// Iteratee returns a state iteratee associated with the specified state root,
// through which the account iterator and storage iterator can be created.
Iteratee(root common.Hash) (Iteratee, error)
// OpenTrie opens the main account trie.
OpenTrie(root common.Hash) (Trie, error)
@ -48,9 +52,6 @@ type Database interface {
// TrieDB returns the underlying trie database for managing trie nodes.
TrieDB() *triedb.Database
// Snapshot returns the underlying state snapshot.
Snapshot() *snapshot.Tree
// Commit flushes all pending writes and finalizes the state transition,
// committing the changes to the underlying storage. It returns an error
// if the commit fails.
@ -310,6 +311,12 @@ func (db *CachingDB) Commit(update *stateUpdate) error {
return db.triedb.Update(update.root, update.originRoot, update.blockNumber, update.nodes, update.stateSet())
}
// Iteratee returns a state iteratee associated with the specified state root,
// through which the account iterator and storage iterator can be created.
func (db *CachingDB) Iteratee(root common.Hash) (Iteratee, error) {
return newStateIteratee(!db.triedb.IsVerkle(), root, db.triedb, db.snap)
}
// mustCopyTrie returns a deep-copied trie.
func mustCopyTrie(t Trie) Trie {
switch t := t.(type) {

12
core/state/database_history.go
View file

@ -22,7 +22,6 @@ import (
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
@ -289,14 +288,15 @@ func (db *HistoricDB) TrieDB() *triedb.Database {
return db.triedb
}
// Snapshot returns the underlying state snapshot.
func (db *HistoricDB) Snapshot() *snapshot.Tree {
return nil
}
// Commit flushes all pending writes and finalizes the state transition,
// committing the changes to the underlying storage. It returns an error
// if the commit fails.
func (db *HistoricDB) Commit(update *stateUpdate) error {
return errors.New("not implemented")
}
// Iteratee returns a state iteratee associated with the specified state root,
// through which the account iterator and storage iterator can be created.
func (db *HistoricDB) Iteratee(root common.Hash) (Iteratee, error) {
return nil, errors.New("not implemented")
}

435
core/state/database_iterator.go Normal file
View file

@ -0,0 +1,435 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package state
import (
"errors"
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/triedb"
)
// Iterator is an iterator to step over all the accounts or the specific
// storage in the specific state.
type Iterator interface {
// Next steps the iterator forward one element. It returns false if the iterator
// is exhausted or if an error occurs. Any error encountered is retained and
// can be retrieved via Error().
Next() bool
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit.
Error() error
// Hash returns the hash of the account or storage slot the iterator is
// currently at.
Hash() common.Hash
// Release releases associated resources. Release should always succeed and
// can be called multiple times without causing error.
Release()
}
// AccountIterator is an iterator to step over all the accounts in the
// specific state.
type AccountIterator interface {
Iterator
// Address returns the raw account address the iterator is currently at.
// An error will be returned if the preimage is not available.
Address() (common.Address, error)
// Account returns the RLP encoded account the iterator is currently at.
// An error will be retained if the iterator becomes invalid.
Account() []byte
}
// StorageIterator is an iterator to step over the specific storage in the
// specific state.
type StorageIterator interface {
Iterator
// Key returns the raw storage slot key the iterator is currently at.
// An error will be returned if the preimage is not available.
Key() (common.Hash, error)
// Slot returns the storage slot the iterator is currently at. An error will
// be retained if the iterator becomes invalid.
Slot() []byte
}
// Iteratee wraps the NewIterator methods for traversing the accounts and
// storages of the specific state.
type Iteratee interface {
// NewAccountIterator creates an account iterator for the state specified by
// the given root. It begins at a specified starting position, corresponding
// to a particular initial key (or the next key if the specified one does
// not exist).
//
// The starting position here refers to the hash of the account address.
NewAccountIterator(start common.Hash) (AccountIterator, error)
// NewStorageIterator creates a storage iterator for the state specified by
// the address hash. It begins at a specified starting position, corresponding
// to a particular initial key (or the next key if the specified one does
// not exist).
//
// The starting position here refers to the hash of the slot key.
NewStorageIterator(addressHash common.Hash, start common.Hash) (StorageIterator, error)
}
// PreimageReader wraps the function Preimage for accessing the preimage of
// a given hash.
type PreimageReader interface {
// Preimage returns the preimage of associated hash.
Preimage(hash common.Hash) []byte
}
// flatAccountIterator is a wrapper around the underlying flat state iterator.
// Before returning data from the iterator, it performs an additional conversion
// to bridge the slim encoding with the full encoding format.
type flatAccountIterator struct {
err error
it snapshot.AccountIterator
preimage PreimageReader
}
// newFlatAccountIterator constructs the account iterator with the provided
// flat state iterator.
func newFlatAccountIterator(it snapshot.AccountIterator, preimage PreimageReader) *flatAccountIterator {
return &flatAccountIterator{it: it, preimage: preimage}
}
// Next steps the iterator forward one element. It returns false if the iterator
// is exhausted or if an error occurs. Any error encountered is retained and
// can be retrieved via Error().
func (ai *flatAccountIterator) Next() bool {
if ai.err != nil {
return false
}
return ai.it.Next()
}
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit.
func (ai *flatAccountIterator) Error() error {
if ai.err != nil {
return ai.err
}
return ai.it.Error()
}
// Hash returns the hash of the account or storage slot the iterator is
// currently at.
func (ai *flatAccountIterator) Hash() common.Hash {
return ai.it.Hash()
}
// Release releases associated resources. Release should always succeed and
// can be called multiple times without causing error.
func (ai *flatAccountIterator) Release() {
ai.it.Release()
}
// Address returns the raw account address the iterator is currently at.
// An error will be returned if the preimage is not available.
func (ai *flatAccountIterator) Address() (common.Address, error) {
if ai.preimage == nil {
return common.Address{}, errors.New("account address is not available")
}
preimage := ai.preimage.Preimage(ai.Hash())
if preimage == nil {
return common.Address{}, errors.New("account address is not available")
}
return common.BytesToAddress(preimage), nil
}
// Account returns the account data the iterator is currently at. The account
// data is encoded as slim format from the underlying iterator, the conversion
// is required.
func (ai *flatAccountIterator) Account() []byte {
data, err := types.FullAccountRLP(ai.it.Account())
if err != nil {
ai.err = err
return nil
}
return data
}
// flatStorageIterator is a wrapper around the underlying flat state iterator.
type flatStorageIterator struct {
it snapshot.StorageIterator
preimage PreimageReader
}
// newFlatStorageIterator constructs the storage iterator with the provided
// flat state iterator.
func newFlatStorageIterator(it snapshot.StorageIterator, preimage PreimageReader) *flatStorageIterator {
return &flatStorageIterator{it: it, preimage: preimage}
}
// Next steps the iterator forward one element. It returns false if the iterator
// is exhausted or if an error occurs. Any error encountered is retained and
// can be retrieved via Error().
func (si *flatStorageIterator) Next() bool {
return si.it.Next()
}
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit.
func (si *flatStorageIterator) Error() error {
return si.it.Error()
}
// Hash returns the hash of the account or storage slot the iterator is
// currently at.
func (si *flatStorageIterator) Hash() common.Hash {
return si.it.Hash()
}
// Release releases associated resources. Release should always succeed and
// can be called multiple times without causing error.
func (si *flatStorageIterator) Release() {
si.it.Release()
}
// Key returns the raw storage slot key the iterator is currently at.
// An error will be returned if the preimage is not available.
func (si *flatStorageIterator) Key() (common.Hash, error) {
if si.preimage == nil {
return common.Hash{}, errors.New("slot key is not available")
}
preimage := si.preimage.Preimage(si.Hash())
if preimage == nil {
return common.Hash{}, errors.New("slot key is not available")
}
return common.BytesToHash(preimage), nil
}
// Slot returns the storage slot data the iterator is currently at.
func (si *flatStorageIterator) Slot() []byte {
return si.it.Slot()
}
// merkleIterator implements the Iterator interface, providing functions to traverse
// the accounts or storages with the manner of Merkle-Patricia-Trie.
type merkleIterator struct {
tr Trie
it *trie.Iterator
account bool
}
// newMerkleTrieIterator constructs the iterator with the given trie and starting position.
func newMerkleTrieIterator(tr Trie, start common.Hash, account bool) (*merkleIterator, error) {
it, err := tr.NodeIterator(start.Bytes())
if err != nil {
return nil, err
}
return &merkleIterator{
tr: tr,
it: trie.NewIterator(it),
account: account,
}, nil
}
// Next steps the iterator forward one element. It returns false if the iterator
// is exhausted or if an error occurs. Any error encountered is retained and
// can be retrieved via Error().
func (ti *merkleIterator) Next() bool {
return ti.it.Next()
}
// Error returns any failure that occurred during iteration, which might have
// caused a premature iteration exit.
func (ti *merkleIterator) Error() error {
return ti.it.Err
}
// Hash returns the hash of the account or storage slot the iterator is
// currently at.
func (ti *merkleIterator) Hash() common.Hash {
return common.BytesToHash(ti.it.Key)
}
// Release releases associated resources. Release should always succeed and
// can be called multiple times without causing error.
func (ti *merkleIterator) Release() {}
// Address returns the raw account address the iterator is currently at.
// An error will be returned if the preimage is not available.
func (ti *merkleIterator) Address() (common.Address, error) {
if !ti.account {
return common.Address{}, errors.New("account address is not available")
}
preimage := ti.tr.GetKey(ti.it.Key)
if preimage == nil {
return common.Address{}, errors.New("account address is not available")
}
return common.BytesToAddress(preimage), nil
}
// Account returns the account data the iterator is currently at.
func (ti *merkleIterator) Account() []byte {
if !ti.account {
return nil
}
return ti.it.Value
}
// Key returns the raw storage slot key the iterator is currently at.
// An error will be returned if the preimage is not available.
func (ti *merkleIterator) Key() (common.Hash, error) {
if ti.account {
return common.Hash{}, errors.New("slot key is not available")
}
preimage := ti.tr.GetKey(ti.it.Key)
if preimage == nil {
return common.Hash{}, errors.New("slot key is not available")
}
return common.BytesToHash(preimage), nil
}
// Slot returns the storage slot the iterator is currently at.
func (ti *merkleIterator) Slot() []byte {
if ti.account {
return nil
}
return ti.it.Value
}
// stateIteratee implements Iteratee interface, providing the state traversal
// functionalities of a specific state.
type stateIteratee struct {
merkle bool
root common.Hash
triedb *triedb.Database
snap *snapshot.Tree
}
func newStateIteratee(merkle bool, root common.Hash, triedb *triedb.Database, snap *snapshot.Tree) (*stateIteratee, error) {
return &stateIteratee{
merkle: merkle,
root: root,
triedb: triedb,
snap: snap,
}, nil
}
// NewAccountIterator creates an account iterator for the state specified by
// the given root. It begins at a specified starting position, corresponding
// to a particular initial key (or the next key if the specified one does
// not exist).
//
// The starting position here refers to the hash of the account address.
func (si *stateIteratee) NewAccountIterator(start common.Hash) (AccountIterator, error) {
// If the external snapshot is available (hash scheme), try to initialize
// the account iterator from there first.
if si.snap != nil {
it, err := si.snap.AccountIterator(si.root, start)
if err == nil {
return newFlatAccountIterator(it, si.triedb), nil
}
}
// If the external snapshot is not available, try to initialize the
// account iterator from the trie database (path scheme)
it, err := si.triedb.AccountIterator(si.root, start)
if err == nil {
return newFlatAccountIterator(it, si.triedb), nil
}
if !si.merkle {
return nil, fmt.Errorf("state %x is not available for account traversal", si.root)
}
// The snapshot is not usable so far, construct the account iterator from
// the trie as the fallback. It's not as efficient as the flat state iterator.
tr, err := trie.NewStateTrie(trie.StateTrieID(si.root), si.triedb)
if err != nil {
return nil, err
}
return newMerkleTrieIterator(tr, start, true)
}
// NewStorageIterator creates a storage iterator for the state specified by
// the address hash. It begins at a specified starting position, corresponding
// to a particular initial key (or the next key if the specified one does not exist).
//
// The starting position here refers to the hash of the slot key.
func (si *stateIteratee) NewStorageIterator(addressHash common.Hash, start common.Hash) (StorageIterator, error) {
// If the external snapshot is available (hash scheme), try to initialize
// the storage iterator from there first.
if si.snap != nil {
it, err := si.snap.StorageIterator(si.root, addressHash, start)
if err == nil {
return newFlatStorageIterator(it, si.triedb), nil
}
}
// If the external snapshot is not available, try to initialize the
// storage iterator from the trie database (path scheme)
it, err := si.triedb.StorageIterator(si.root, addressHash, start)
if err == nil {
return newFlatStorageIterator(it, si.triedb), nil
}
if !si.merkle {
return nil, fmt.Errorf("state %x is not available for storage traversal", si.root)
}
// The snapshot is not usable so far, construct the storage iterator from
// the trie as the fallback. It's not as efficient as the flat state iterator.
tr, err := trie.NewStateTrie(trie.StateTrieID(si.root), si.triedb)
if err != nil {
return nil, err
}
acct, err := tr.GetAccountByHash(addressHash)
if err != nil {
return nil, err
}
if acct == nil || acct.Root == types.EmptyRootHash {
return &exhaustedIterator{}, nil
}
storageTr, err := trie.NewStateTrie(trie.StorageTrieID(si.root, addressHash, acct.Root), si.triedb)
if err != nil {
return nil, err
}
return newMerkleTrieIterator(storageTr, start, false)
}
type exhaustedIterator struct{}
func (e exhaustedIterator) Next() bool {
return false
}
func (e exhaustedIterator) Error() error {
return nil
}
func (e exhaustedIterator) Hash() common.Hash {
return common.Hash{}
}
func (e exhaustedIterator) Release() {
}
func (e exhaustedIterator) Key() (common.Hash, error) {
return common.Hash{}, nil
}
func (e exhaustedIterator) Slot() []byte {
return nil
}

262
core/state/database_iterator_test.go Normal file
View file

@ -0,0 +1,262 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package state
import (
"bytes"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
)
// TestExhaustedIterator verifies the exhaustedIterator sentinel: Next is false,
// Error is nil, Hash/Key are zero, Slot is nil, and double Release is safe.
func TestExhaustedIterator(t *testing.T) {
var it exhaustedIterator
if it.Next() {
t.Fatal("Next() returned true")
}
if err := it.Error(); err != nil {
t.Fatalf("Error() = %v, want nil", err)
}
if hash := it.Hash(); hash != (common.Hash{}) {
t.Fatalf("Hash() = %x, want zero", hash)
}
if key, err := it.Key(); key != (common.Hash{}) || err != nil {
t.Fatalf("Key() = %x, %v; want zero, nil", key, err)
}
if slot := it.Slot(); slot != nil {
t.Fatalf("Slot() = %x, want nil", slot)
}
it.Release()
it.Release()
}
// TestAccountIterator tests the account iterator: correct count, ascending
// hash order, valid full-format RLP, data integrity, address preimage
// resolution, and seek behavior.
func TestAccountIterator(t *testing.T) {
testAccountIterator(t, rawdb.HashScheme)
testAccountIterator(t, rawdb.PathScheme)
}
func testAccountIterator(t *testing.T, scheme string) {
_, sdb, ndb, root, accounts := makeTestState(scheme)
ndb.Commit(root, false)
iteratee, err := sdb.Iteratee(root)
if err != nil {
t.Fatalf("(%s) failed to create iteratee: %v", scheme, err)
}
// Build lookups from address hash.
addrByHash := make(map[common.Hash]*testAccount)
for _, acc := range accounts {
addrByHash[crypto.Keccak256Hash(acc.address.Bytes())] = acc
}
// --- Full iteration: count, ordering, RLP validity, data integrity, address resolution ---
acctIt, err := iteratee.NewAccountIterator(common.Hash{})
if err != nil {
t.Fatalf("(%s) failed to create account iterator: %v", scheme, err)
}
var (
hashes []common.Hash
prevHash common.Hash
)
for acctIt.Next() {
hash := acctIt.Hash()
if hash == (common.Hash{}) {
t.Fatalf("(%s) zero hash at position %d", scheme, len(hashes))
}
if len(hashes) > 0 && bytes.Compare(prevHash.Bytes(), hash.Bytes()) >= 0 {
t.Fatalf("(%s) hashes not ascending: %x >= %x", scheme, prevHash, hash)
}
prevHash = hash
hashes = append(hashes, hash)
// Decode and verify account data.
blob := acctIt.Account()
if blob == nil {
t.Fatalf("(%s) nil account at %x", scheme, hash)
}
var decoded types.StateAccount
if err := rlp.DecodeBytes(blob, &decoded); err != nil {
t.Fatalf("(%s) bad RLP at %x: %v", scheme, hash, err)
}
acc := addrByHash[hash]
if decoded.Nonce != acc.nonce {
t.Fatalf("(%s) nonce %x: got %d, want %d", scheme, hash, decoded.Nonce, acc.nonce)
}
if decoded.Balance.Cmp(acc.balance) != 0 {
t.Fatalf("(%s) balance %x: got %v, want %v", scheme, hash, decoded.Balance, acc.balance)
}
// Verify address preimage resolution.
addr, err := acctIt.Address()
if err != nil {
t.Fatalf("(%s) failed to address: %v", scheme, err)
}
if addr != acc.address {
t.Fatalf("(%s) Address() = %x, want %x", scheme, addr, acc.address)
}
}
acctIt.Release()
if err := acctIt.Error(); err != nil {
t.Fatalf("(%s) iteration error: %v", scheme, err)
}
if len(hashes) != len(accounts) {
t.Fatalf("(%s) iterated %d accounts, want %d", scheme, len(hashes), len(accounts))
}
// --- Seek: starting from midpoint should skip earlier entries ---
mid := hashes[len(hashes)/2]
seekIt, err := iteratee.NewAccountIterator(mid)
if err != nil {
t.Fatalf("(%s) failed to create seeked iterator: %v", scheme, err)
}
seekCount := 0
for seekIt.Next() {
if bytes.Compare(seekIt.Hash().Bytes(), mid.Bytes()) < 0 {
t.Fatalf("(%s) seeked iterator returned hash before start", scheme)
}
seekCount++
}
seekIt.Release()
if seekCount != len(hashes)/2 {
t.Fatalf("(%s) unexpected seeked count, %d != %d", scheme, seekCount, len(hashes)/2)
}
}
// TestStorageIterator tests the storage iterator: correct slot counts against
// the trie, ascending hash order, non-nil slot data, key preimage resolution,
// seek behavior, and empty-storage accounts.
func TestStorageIterator(t *testing.T) {
testStorageIterator(t, rawdb.HashScheme)
testStorageIterator(t, rawdb.PathScheme)
}
func testStorageIterator(t *testing.T, scheme string) {
_, sdb, ndb, root, accounts := makeTestState(scheme)
ndb.Commit(root, false)
iteratee, err := sdb.Iteratee(root)
if err != nil {
t.Fatalf("(%s) failed to create iteratee: %v", scheme, err)
}
// --- Slot count and ordering for every account ---
var withStorage common.Hash // remember an account that has storage for seek test
for _, acc := range accounts {
addrHash := crypto.Keccak256Hash(acc.address.Bytes())
expected := countStorageSlots(t, scheme, sdb, root, addrHash)
storageIt, err := iteratee.NewStorageIterator(addrHash, common.Hash{})
if err != nil {
t.Fatalf("(%s) failed to create storage iterator for %x: %v", scheme, acc.address, err)
}
count := 0
var prevHash common.Hash
for storageIt.Next() {
hash := storageIt.Hash()
if count > 0 && bytes.Compare(prevHash.Bytes(), hash.Bytes()) >= 0 {
t.Fatalf("(%s) storage hashes not ascending for %x", scheme, acc.address)
}
prevHash = hash
if storageIt.Slot() == nil {
t.Fatalf("(%s) nil slot at %x", scheme, hash)
}
// Check key preimage resolution on first slot.
if _, err := storageIt.Key(); err != nil {
t.Fatalf("(%s) Key() failed to resolve", scheme)
}
count++
}
if err := storageIt.Error(); err != nil {
t.Fatalf("(%s) storage iteration error for %x: %v", scheme, acc.address, err)
}
storageIt.Release()
if count != expected {
t.Fatalf("(%s) account %x: %d slots, want %d", scheme, acc.address, count, expected)
}
if count > 0 {
withStorage = addrHash
}
}
// --- Seek: starting from second slot should skip the first ---
if withStorage == (common.Hash{}) {
t.Fatalf("(%s) no account with storage found", scheme)
}
fullIt, err := iteratee.NewStorageIterator(withStorage, common.Hash{})
if err != nil {
t.Fatalf("(%s) failed to create full storage iterator: %v", scheme, err)
}
var slotHashes []common.Hash
for fullIt.Next() {
slotHashes = append(slotHashes, fullIt.Hash())
}
fullIt.Release()
seekIt, err := iteratee.NewStorageIterator(withStorage, slotHashes[1])
if err != nil {
t.Fatalf("(%s) failed to create seeked storage iterator: %v", scheme, err)
}
seekCount := 0
for seekIt.Next() {
if bytes.Compare(seekIt.Hash().Bytes(), slotHashes[1].Bytes()) < 0 {
t.Fatalf("(%s) seeked storage iterator returned hash before start", scheme)
}
seekCount++
}
seekIt.Release()
if seekCount != len(slotHashes)-1 {
t.Fatalf("(%s) unexpected seeked storage count %d != %d", scheme, seekCount, len(slotHashes)-1)
}
}
// countStorageSlots counts storage slots for an account by opening the
// storage trie directly.
func countStorageSlots(t *testing.T, scheme string, sdb Database, root common.Hash, addrHash common.Hash) int {
t.Helper()
accTrie, err := trie.NewStateTrie(trie.StateTrieID(root), sdb.TrieDB())
if err != nil {
t.Fatalf("(%s) failed to open account trie: %v", scheme, err)
}
acct, err := accTrie.GetAccountByHash(addrHash)
if err != nil || acct == nil || acct.Root == types.EmptyRootHash {
return 0
}
storageTrie, err := trie.NewStateTrie(trie.StorageTrieID(root, addrHash, acct.Root), sdb.TrieDB())
if err != nil {
t.Fatalf("(%s) failed to open storage trie for %x: %v", scheme, addrHash, err)
}
it := trie.NewIterator(storageTrie.MustNodeIterator(nil))
count := 0
for it.Next() {
count++
}
return count
}

65
core/state/dump.go
View file

@ -27,7 +27,6 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/trie/bintrie"
)
@ -45,6 +44,7 @@ type DumpConfig struct {
type DumpCollector interface {
// OnRoot is called with the state root
OnRoot(common.Hash)
// OnAccount is called once for each account in the trie
OnAccount(*common.Address, DumpAccount)
}
@ -65,9 +65,10 @@ type DumpAccount struct {
type Dump struct {
Root string `json:"root"`
Accounts map[string]DumpAccount `json:"accounts"`
// Next can be set to represent that this dump is only partial, and Next
// is where an iterator should be positioned in order to continue the dump.
Next []byte `json:"next,omitempty"` // nil if no more accounts
Next hexutil.Bytes `json:"next,omitempty"` // nil if no more accounts
}
// OnRoot implements DumpCollector interface
@ -114,9 +115,6 @@ func (d iterativeDump) OnRoot(root common.Hash) {
// DumpToCollector iterates the state according to the given options and inserts
// the items into a collector for aggregation or serialization.
//
// The state iterator is still trie-based and can be converted to snapshot-based
// once the state snapshot is fully integrated into database. TODO(rjl493456442).
func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []byte) {
// Sanitize the input to allow nil configs
if conf == nil {
@ -131,20 +129,23 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []
log.Info("Trie dumping started", "root", s.originalRoot)
c.OnRoot(s.originalRoot)
tr, err := s.db.OpenTrie(s.originalRoot)
iteratee, err := s.db.Iteratee(s.originalRoot)
if err != nil {
return nil
}
trieIt, err := tr.NodeIterator(conf.Start)
var startHash common.Hash
if conf.Start != nil {
startHash = common.BytesToHash(conf.Start)
}
acctIt, err := iteratee.NewAccountIterator(startHash)
if err != nil {
log.Error("Trie dumping error", "err", err)
return nil
}
it := trie.NewIterator(trieIt)
defer acctIt.Release()
for it.Next() {
for acctIt.Next() {
var data types.StateAccount
if err := rlp.DecodeBytes(it.Value, &data); err != nil {
if err := rlp.DecodeBytes(acctIt.Account(), &data); err != nil {
panic(err)
}
var (
@ -153,63 +154,55 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []
Nonce: data.Nonce,
Root: data.Root[:],
CodeHash: data.CodeHash,
AddressHash: it.Key,
AddressHash: acctIt.Hash().Bytes(),
}
address *common.Address
addr common.Address
addrBytes = tr.GetKey(it.Key)
address *common.Address
)
if addrBytes == nil {
addrBytes, err := acctIt.Address()
if err != nil {
missingPreimages++
if conf.OnlyWithAddresses {
continue
}
} else {
addr = common.BytesToAddress(addrBytes)
address = &addr
address = &addrBytes
account.Address = address
}
obj := newObject(s, addr, &data)
obj := newObject(s, addrBytes, &data)
if !conf.SkipCode {
account.Code = obj.Code()
}
if !conf.SkipStorage {
account.Storage = make(map[common.Hash]string)
storageTr, err := s.db.OpenStorageTrie(s.originalRoot, addr, obj.Root(), tr)
storageIt, err := iteratee.NewStorageIterator(acctIt.Hash(), common.Hash{})
if err != nil {
log.Error("Failed to load storage trie", "err", err)
continue
}
trieIt, err := storageTr.NodeIterator(nil)
if err != nil {
log.Error("Failed to create trie iterator", "err", err)
continue
}
storageIt := trie.NewIterator(trieIt)
for storageIt.Next() {
_, content, _, err := rlp.Split(storageIt.Value)
_, content, _, err := rlp.Split(storageIt.Slot())
if err != nil {
log.Error("Failed to decode the value returned by iterator", "error", err)
continue
}
key := storageTr.GetKey(storageIt.Key)
if key == nil {
key, err := storageIt.Key()
if err != nil {
continue
}
account.Storage[common.BytesToHash(key)] = common.Bytes2Hex(content)
account.Storage[key] = common.Bytes2Hex(content)
}
storageIt.Release()
}
c.OnAccount(address, account)
accounts++
if time.Since(logged) > 8*time.Second {
log.Info("Trie dumping in progress", "at", common.Bytes2Hex(it.Key), "accounts", accounts,
"elapsed", common.PrettyDuration(time.Since(start)))
log.Info("Trie dumping in progress", "at", acctIt.Hash().Hex(), "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start)))
logged = time.Now()
}
if conf.Max > 0 && accounts >= conf.Max {
if it.Next() {
nextKey = it.Key
if acctIt.Next() {
nextKey = acctIt.Hash().Bytes()
}
break
}
@ -217,9 +210,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey []
if missingPreimages > 0 {
log.Warn("Dump incomplete due to missing preimages", "missing", missingPreimages)
}
log.Info("Trie dumping complete", "accounts", accounts,
"elapsed", common.PrettyDuration(time.Since(start)))
log.Info("Trie dumping complete", "accounts", accounts, "elapsed", common.PrettyDuration(time.Since(start)))
return nextKey
}

98
core/state/statedb.go
View file

@ -28,7 +28,6 @@ import (
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/stateless"
"github.com/ethereum/go-ethereum/core/tracing"
"github.com/ethereum/go-ethereum/core/types"
@ -1039,31 +1038,32 @@ func (s *StateDB) clearJournalAndRefund() {
s.refund = 0
}
// fastDeleteStorage is the function that efficiently deletes the storage trie
// of a specific account. It leverages the associated state snapshot for fast
// storage iteration and constructs trie node deletion markers by creating
// stack trie with iterated slots.
func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) {
iter, err := snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{})
if err != nil {
return nil, nil, nil, err
}
defer iter.Release()
// deleteStorage is designed to delete the storage trie of a designated account.
func (s *StateDB) deleteStorage(addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) {
var (
nodes = trienode.NewNodeSet(addrHash) // the set for trie node mutations (value is nil)
storages = make(map[common.Hash][]byte) // the set for storage mutations (value is nil)
storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot
)
iteratee, err := s.db.Iteratee(s.originalRoot)
if err != nil {
return nil, nil, nil, err
}
it, err := iteratee.NewStorageIterator(addrHash, common.Hash{})
if err != nil {
return nil, nil, nil, err
}
defer it.Release()
stack := trie.NewStackTrie(func(path []byte, hash common.Hash, blob []byte) {
nodes.AddNode(path, trienode.NewDeletedWithPrev(blob))
})
for iter.Next() {
slot := common.CopyBytes(iter.Slot())
if err := iter.Error(); err != nil { // error might occur after Slot function
for it.Next() {
slot := common.CopyBytes(it.Slot())
if err := it.Error(); err != nil { // error might occur after Slot function
return nil, nil, nil, err
}
key := iter.Hash()
key := it.Hash()
storages[key] = nil
storageOrigins[key] = slot
@ -1071,7 +1071,7 @@ func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash,
return nil, nil, nil, err
}
}
if err := iter.Error(); err != nil { // error might occur during iteration
if err := it.Error(); err != nil { // error might occur during iteration
return nil, nil, nil, err
}
if stack.Hash() != root {
@ -1080,68 +1080,6 @@ func (s *StateDB) fastDeleteStorage(snaps *snapshot.Tree, addrHash common.Hash,
return storages, storageOrigins, nodes, nil
}
// slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage,"
// employed when the associated state snapshot is not available. It iterates the
// storage slots along with all internal trie nodes via trie directly.
func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) {
tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err)
}
it, err := tr.NodeIterator(nil)
if err != nil {
return nil, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err)
}
var (
nodes = trienode.NewNodeSet(addrHash) // the set for trie node mutations (value is nil)
storages = make(map[common.Hash][]byte) // the set for storage mutations (value is nil)
storageOrigins = make(map[common.Hash][]byte) // the set for tracking the original value of slot
)
for it.Next(true) {
if it.Leaf() {
key := common.BytesToHash(it.LeafKey())
storages[key] = nil
storageOrigins[key] = common.CopyBytes(it.LeafBlob())
continue
}
if it.Hash() == (common.Hash{}) {
continue
}
nodes.AddNode(it.Path(), trienode.NewDeletedWithPrev(it.NodeBlob()))
}
if err := it.Error(); err != nil {
return nil, nil, nil, err
}
return storages, storageOrigins, nodes, nil
}
// deleteStorage is designed to delete the storage trie of a designated account.
// The function will make an attempt to utilize an efficient strategy if the
// associated state snapshot is reachable; otherwise, it will resort to a less
// efficient approach.
func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, map[common.Hash][]byte, *trienode.NodeSet, error) {
var (
err error
nodes *trienode.NodeSet // the set for trie node mutations (value is nil)
storages map[common.Hash][]byte // the set for storage mutations (value is nil)
storageOrigins map[common.Hash][]byte // the set for tracking the original value of slot
)
// The fast approach can be failed if the snapshot is not fully
// generated, or it's internally corrupted. Fallback to the slow
// one just in case.
snaps := s.db.Snapshot()
if snaps != nil {
storages, storageOrigins, nodes, err = s.fastDeleteStorage(snaps, addrHash, root)
}
if snaps == nil || err != nil {
storages, storageOrigins, nodes, err = s.slowDeleteStorage(addr, addrHash, root)
}
if err != nil {
return nil, nil, nil, err
}
return storages, storageOrigins, nodes, nil
}
// handleDestruction processes all destruction markers and deletes the account
// and associated storage slots if necessary. There are four potential scenarios
// as following:
@ -1192,7 +1130,7 @@ func (s *StateDB) handleDestruction(noStorageWiping bool) (map[common.Hash]*acco
return nil, nil, fmt.Errorf("unexpected storage wiping, %x", addr)
}
// Remove storage slots belonging to the account.
storages, storagesOrigin, set, err := s.deleteStorage(addr, addrHash, prev.Root)
storages, storagesOrigin, set, err := s.deleteStorage(addrHash, prev.Root)
if err != nil {
return nil, nil, fmt.Errorf("failed to delete storage, err: %w", err)
}

4
core/state/statedb_test.go
View file

@ -1296,12 +1296,12 @@ func TestDeleteStorage(t *testing.T) {
obj := fastState.getOrNewStateObject(addr)
storageRoot := obj.data.Root
_, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
_, _, fastNodes, err := fastState.deleteStorage(crypto.Keccak256Hash(addr[:]), storageRoot)
if err != nil {
t.Fatal(err)
}
_, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
_, _, slowNodes, err := slowState.deleteStorage(crypto.Keccak256Hash(addr[:]), storageRoot)
if err != nil {
t.Fatal(err)
}

2
eth/api_debug.go
View file

@ -236,6 +236,8 @@ func storageRangeAt(statedb *state.StateDB, root common.Hash, address common.Add
if storageRoot == types.EmptyRootHash || storageRoot == (common.Hash{}) {
return StorageRangeResult{}, nil // empty storage
}
// TODO(rjl493456442) it's problematic for traversing the state with in-memory
// state mutations, specifically txIndex != 0.
id := trie.StorageTrieID(root, crypto.Keccak256Hash(address.Bytes()), storageRoot)
tr, err := trie.NewStateTrie(id, statedb.Database().TrieDB())
if err != nil {