forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-20 10:52:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

add necessary changes for building stateless block witnesses. Expose a debug api method to build witness for a given block. Add blockchain test suite which builds witnesses for each imported block. Stateless execution changes and cross-validation logic omitted to minimize the code footprint of this PR

Browse source
This commit is contained in:
Jared Wasinger 2024-02-19 17:12:21 -08:00
parent ac0ff04460
commit 9100e2e566
18 changed files with 958 additions and 143 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
consensus/beacon/consensus.go
View file

@ -30,7 +30,6 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie"
"github.com/holiman/uint256"
)
// Proof-of-stake protocol constants.
@ -353,13 +352,7 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.
beacon.ethone.Finalize(chain, header, state, txs, uncles, nil)
return
}
// Withdrawals processing.
for _, w := range withdrawals {
// Convert amount from gwei to wei.
amount := new(uint256.Int).SetUint64(w.Amount)
amount = amount.Mul(amount, uint256.NewInt(params.GWei))
state.AddBalance(w.Address, amount)
}
state.ApplyWithdrawals(withdrawals)
// No block reward which is issued by consensus layer instead.
}

7
core/state/database.go
View file

@ -126,6 +126,13 @@ type Trie interface {
// be created with new root and updated trie database for following usage
Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error)
// CommitAndObtainAccessList does the same thing as Commit and returns an
// access list map of trie nodes read from the database.
CommitAndObtainAccessList(collectLeaf bool) (common.Hash, *trienode.NodeSet, map[string][]byte, error)
// AccessList returns a map of trie node read from the database.
AccessList() map[string][]byte
// NodeIterator returns an iterator that returns nodes of the trie. Iteration
// starts at the key after the given start key. And error will be returned
// if fails to create node iterator.

18
core/state/state_object.go
View file

@ -194,6 +194,13 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash {
err error
value common.Hash
)
if s.db.witness != nil && s.db.snap != nil && s.origin != nil {
// when building a witness with snapshot enabled, prefetch all read slots to be collected
// and included in the witness when the block root hash is committed (intermediateroot/commit?)
s.db.readPrefetcher.prefetch(s.addrHash, s.origin.Root, s.address, [][]byte{key[:]})
}
if s.db.snap != nil {
start := time.Now()
enc, err = s.db.snap.Storage(s.addrHash, crypto.Keccak256Hash(key.Bytes()))
@ -217,6 +224,7 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash {
return common.Hash{}
}
val, err := tr.GetStorage(s.address, key.Bytes())
//fmt.Printf("trie access list is %v\n", tr.AccessList())
if metrics.EnabledExpensive {
s.db.StorageReads += time.Since(start)
}
@ -379,11 +387,11 @@ func (s *stateObject) updateRoot() {
// commit obtains a set of dirty storage trie nodes and updates the account data.
// The returned set can be nil if nothing to commit. This function assumes all
// storage mutations have already been flushed into trie by updateRoot.
func (s *stateObject) commit() (*trienode.NodeSet, error) {
func (s *stateObject) commit() (*trienode.NodeSet, map[string][]byte, error) {
// Short circuit if trie is not even loaded, don't bother with committing anything
if s.trie == nil {
s.origin = s.data.Copy()
return nil, nil
return nil, nil, nil
}
// Track the amount of time wasted on committing the storage trie
if metrics.EnabledExpensive {
@ -392,15 +400,15 @@ func (s *stateObject) commit() (*trienode.NodeSet, error) {
// The trie is currently in an open state and could potentially contain
// cached mutations. Call commit to acquire a set of nodes that have been
// modified, the set can be nil if nothing to commit.
root, nodes, err := s.trie.Commit(false)
root, nodes, accessList, err := s.trie.CommitAndObtainAccessList(false)
if err != nil {
return nil, err
return nil, nil, err
}
s.data.Root = root
// Update original account data after commit
s.origin = s.data.Copy()
return nodes, nil
return nodes, accessList, nil
}
// AddBalance adds amount to s's balance.

411
core/state/state_witness.go Normal file
View file

@ -0,0 +1,411 @@
package state
import (
"bytes"
"crypto/sha256"
"encoding/json"
"fmt"
"math/big"
"os"
"path/filepath"
"sort"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
)
type Witness struct {
Block *types.Block
blockHashes map[uint64]common.Hash
codes map[common.Hash]Code
root common.Hash
lists map[common.Hash]map[string][]byte
}
func (w *Witness) GetBlockHash(num uint64) common.Hash {
return w.blockHashes[num]
}
func (w *Witness) Root() common.Hash {
return w.root
}
type rlpWitness struct {
EncBlock []byte
Root common.Hash
Owners []common.Hash
AllPaths [][]string
AllNodes [][][]byte
BlockNums []uint64
BlockHashes []common.Hash
Codes []Code
CodeHashes []common.Hash
}
func (e *rlpWitness) ToWitness() (*Witness, error) {
res := NewWitness(e.Root)
if err := rlp.DecodeBytes(e.EncBlock, &res.Block); err != nil {
return nil, err
}
for i := 0; i < len(e.Codes); i++ {
res.codes[e.CodeHashes[i]] = e.Codes[i]
}
for i, owner := range e.Owners {
pathMap := make(map[string][]byte)
for j := 0; j < len(e.AllPaths[i]); j++ {
pathMap[e.AllPaths[i][j]] = e.AllNodes[i][j]
}
res.lists[owner] = pathMap
}
for i, blockNum := range e.BlockNums {
res.blockHashes[blockNum] = e.BlockHashes[i]
}
return res, nil
}
func DecodeWitnessRLP(b []byte) (*Witness, error) {
var res rlpWitness
if err := rlp.DecodeBytes(b, &res); err != nil {
return nil, err
}
if wit, err := res.ToWitness(); err != nil {
return nil, err
} else {
return wit, nil
}
}
func (w *Witness) EncodeRLP() ([]byte, error) {
var encWit rlpWitness
var encBlock bytes.Buffer
if err := w.Block.EncodeRLPWithZeroRoot(&encBlock); err != nil {
return nil, err
}
encWit.EncBlock = encBlock.Bytes()
for owner, nodeMap := range w.lists {
encWit.Owners = append(encWit.Owners, owner)
var ownerPaths []string
var ownerNodes [][]byte
for path, node := range nodeMap {
ownerPaths = append(ownerPaths, path)
ownerNodes = append(ownerNodes, node)
}
encWit.AllPaths = append(encWit.AllPaths, ownerPaths)
encWit.AllNodes = append(encWit.AllNodes, ownerNodes)
}
for codeHash, code := range w.codes {
encWit.CodeHashes = append(encWit.CodeHashes, codeHash)
encWit.Codes = append(encWit.Codes, code)
}
for blockNum, blockHash := range w.blockHashes {
encWit.BlockNums = append(encWit.BlockNums, blockNum)
encWit.BlockHashes = append(encWit.BlockHashes, blockHash)
}
res, err := rlp.EncodeToBytes(&encWit)
if err != nil {
return nil, err
}
return res, nil
}
// addAccessList associates a map of raw trie nodes keyed by path to an owner
// in the witness. the witness takes ownership of the passed map.
func (w *Witness) addAccessList(owner common.Hash, list map[string][]byte) {
var stateNodes map[string][]byte
if len(list) == 0 {
return
}
stateNodes, ok := w.lists[owner]
if !ok {
stateNodes = make(map[string][]byte)
w.lists[owner] = stateNodes
}
for path, node := range list {
stateNodes[path] = node
}
}
// AddBlockHash adds a block hash/number to the witness
func (w *Witness) AddBlockHash(hash common.Hash, num uint64) {
w.blockHashes[num] = hash
}
// AddCode associates a hash with EVM bytecode in the witness. It does
// nothing if there is already a code associated with the given hash.
// The witness takes ownership over the passed code slice.
func (w *Witness) AddCode(hash common.Hash, code Code) {
if code, ok := w.codes[hash]; ok && len(code) > 0 {
return
}
w.codes[hash] = code
}
// AddCodeHash adds a code hash to the witness
// TODO bug: adding a code hash before executing the same account later would result in the account's code
// not being added to the witness. this should be covered in state tests?
func (w *Witness) AddCodeHash(hash common.Hash) {
if _, ok := w.codes[hash]; ok {
return
}
w.codes[hash] = []byte{}
}
// Summary prints a human-readable summary containing the total size of the
// witness and the sizes of the underlying components
func (w *Witness) Summary() string {
b := new(bytes.Buffer)
xx, err := rlp.EncodeToBytes(w.Block)
if err != nil {
panic(err)
}
totBlock := len(xx)
yy, _ := w.EncodeRLP()
totWit := len(yy)
totCode := 0
for _, c := range w.codes {
totCode += len(c)
}
totNodes := 0
totPaths := 0
nodePathCount := 0
for _, ownerPaths := range w.lists {
for path, node := range ownerPaths {
nodePathCount++
totNodes += len(node)
totPaths += len(path)
}
}
fmt.Fprintf(b, "%4d hashes: %v\n", len(w.blockHashes), common.StorageSize(len(w.blockHashes)*32))
fmt.Fprintf(b, "%4d owners: %v\n", len(w.lists), common.StorageSize(len(w.lists)*32))
fmt.Fprintf(b, "%4d nodes: %v\n", nodePathCount, common.StorageSize(totNodes))
fmt.Fprintf(b, "%4d paths: %v\n", nodePathCount, common.StorageSize(totPaths))
fmt.Fprintf(b, "%4d codes: %v\n", len(w.codes), common.StorageSize(totCode))
fmt.Fprintf(b, "%4d codeHashes: %v\n", len(w.codes), common.StorageSize(len(w.codes)*32))
fmt.Fprintf(b, "block (%4d txs): %v\n", len(w.Block.Transactions()), common.StorageSize(totBlock))
fmt.Fprintf(b, "Total size: %v\n ", common.StorageSize(totWit))
return b.String()
}
// Copy deep-copies the witness object. Witness.Block isn't deep-copied as it
// is never mutated by Witness
func (w *Witness) Copy() *Witness {
var res Witness
res.Block = w.Block //
for blockNr, blockHash := range w.blockHashes {
res.blockHashes[blockNr] = blockHash
}
for codeHash, code := range w.codes {
cpy := make([]byte, len(code))
copy(cpy, code)
res.codes[codeHash] = cpy
}
res.root = w.root
for owner, owned := range w.lists {
res.lists[owner] = make(map[string][]byte)
for path, node := range owned {
cpy := make([]byte, len(node))
copy(cpy, node)
res.lists[owner][path] = cpy
}
}
return &res
}
// sortedWitness encodes returns an rlpWitness where hash-map items are sorted lexicographically by key
// in the encoder object to ensure that the encoded bytes are always the same for a given witness.
func (w *Witness) sortedWitness() *rlpWitness {
var sortedCodeHashes []common.Hash
for key, _ := range w.codes {
sortedCodeHashes = append(sortedCodeHashes, key)
}
sort.Slice(sortedCodeHashes, func(i, j int) bool {
return bytes.Compare(sortedCodeHashes[i][:], sortedCodeHashes[j][:]) > 0
})
// sort the list of owners
var owners []common.Hash
for owner, _ := range w.lists {
owners = append(owners, owner)
}
sort.Slice(owners, func(i, j int) bool {
return bytes.Compare(owners[i][:], owners[j][:]) > 0
})
var ownersPaths [][]string
var ownersNodes [][][]byte
// sort the nodes of each owner by path
for _, owner := range owners {
nodes := w.lists[owner]
var ownerPaths []string
for path, _ := range nodes {
ownerPaths = append(ownerPaths, path)
}
sort.Strings(ownerPaths)
var ownerNodes [][]byte
for _, path := range ownerPaths {
ownerNodes = append(ownerNodes, nodes[path])
}
ownersPaths = append(ownersPaths, ownerPaths)
ownersNodes = append(ownersNodes, ownerNodes)
}
var blockNrs []uint64
var blockHashes []common.Hash
for blockNr, blockHash := range w.blockHashes {
blockNrs = append(blockNrs, blockNr)
blockHashes = append(blockHashes, blockHash)
}
var codeHashes []common.Hash
var codes []Code
for codeHash, _ := range w.codes {
codeHashes = append(codeHashes, codeHash)
}
sort.Slice(codeHashes, func(i, j int) bool {
return bytes.Compare(codeHashes[i][:], codeHashes[j][:]) > 0
})
for _, codeHash := range codeHashes {
codes = append(codes, w.codes[codeHash])
}
encBlock, _ := rlp.EncodeToBytes(w.Block)
return &rlpWitness{
EncBlock: encBlock,
Root: common.Hash{},
Owners: owners,
AllPaths: ownersPaths,
AllNodes: ownersNodes,
BlockNums: blockNrs,
BlockHashes: blockHashes,
Codes: codes,
CodeHashes: codeHashes,
}
}
// PrettyPrint displays the contents of a witness object in a human-readable format to standard output.
func (w *Witness) PrettyPrint() string {
sorted := w.sortedWitness()
b := new(bytes.Buffer)
fmt.Fprintf(b, "block: %+v\n", w.Block)
fmt.Fprintf(b, "root: %x\n", sorted.Root)
fmt.Fprint(b, "owners:\n")
for i, owner := range sorted.Owners {
if owner == (common.Hash{}) {
fmt.Fprintf(b, "\troot:\n")
} else {
fmt.Fprintf(b, "\t%x:\n", owner)
}
ownerPaths := sorted.AllPaths[i]
ownerNodes := sorted.AllNodes[i]
for j, path := range ownerPaths {
fmt.Fprintf(b, "\t\t%x:%x\n", []byte(path), ownerNodes[j])
}
}
fmt.Fprintf(b, "block hashes:\n")
for i, blockNum := range sorted.BlockNums {
blockHash := sorted.BlockHashes[i]
fmt.Fprintf(b, "\t%d:%x\n", blockNum, blockHash)
}
fmt.Fprintf(b, "codes:\n")
for i, codeHash := range sorted.CodeHashes {
code := sorted.Codes[i]
fmt.Fprintf(b, "\t%x:%x\n", codeHash, code)
}
return b.String()
}
// Hash returns the sha256 hash of a witness
func (w *Witness) Hash() common.Hash {
res, err := rlp.EncodeToBytes(w.sortedWitness())
if err != nil {
panic(err)
}
return common.Hash(sha256.Sum256(res[:]))
}
// NewWitness returns a new witness object.
func NewWitness(root common.Hash) *Witness {
return &Witness{
Block: nil,
blockHashes: make(map[uint64]common.Hash),
codes: make(map[common.Hash]Code),
root: root,
lists: make(map[common.Hash]map[string][]byte),
}
}
// DumpBlockWitnessToFile serializes a witness object and writes it and the provided chain config to files on
// a given path.
func DumpBlockWitnessToFile(cfg *params.ChainConfig, w *Witness, path string) error {
enc, _ := w.EncodeRLP()
blockHash := w.Block.Hash()
witnessOutputFName := fmt.Sprintf("%d-%x.rlp", w.Block.NumberU64(), blockHash[0:8])
witnessPath := filepath.Join(path, witnessOutputFName)
err := os.WriteFile(witnessPath, enc, 0644)
if err != nil {
return err
}
cfgOutputFName := fmt.Sprintf("%d-%x-chaincfg.json", w.Block.NumberU64(), blockHash[0:8])
cfgPath := filepath.Join(path, cfgOutputFName)
f, err := os.OpenFile(cfgPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
if err != nil {
return err
}
defer f.Close()
cfgWriter := json.NewEncoder(f)
cfgWriter.Encode(cfg)
return nil
}
// PopulateDB imports trie nodes from the witness
// into the specified backing database.
func (w *Witness) PopulateDB(db ethdb.Database) error {
batch := db.NewBatch()
for owner, nodes := range w.lists {
for path, node := range nodes {
if owner == (common.Hash{}) {
rawdb.WriteAccountTrieNode(batch, []byte(path), node)
} else {
rawdb.WriteStorageTrieNode(batch, owner, []byte(path), node)
}
}
}
for blockNum, blockHash := range w.blockHashes {
fakeHeader := types.Header{}
fakeHeader.ParentHash = blockHash
fakeHeader.Number = new(big.Int).SetUint64(blockNum)
rawdb.WriteHeader(batch, &fakeHeader)
}
for codeHash, code := range w.codes {
rawdb.WriteCode(batch, codeHash, code)
}
if err := batch.Write(); err != nil {
return err
}
return nil
}

186
core/state/statedb.go
View file

@ -138,6 +138,20 @@ type StateDB struct {
// Testing hooks
onCommit func(states *triestate.Set) // Hook invoked when commit is performed
witness *Witness
readPrefetcher *triePrefetcher
}
// NewWithWitnessRecording creates a new state from a given trie. The state is configured to construct a stateless
// block witness which is completed after Commit is called.
func NewWithWitnessRecording(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) {
sdb, err := New(root, db, snaps)
if err != nil {
return nil, err
}
sdb.witness = NewWitness(root)
return sdb, nil
}
// New creates a new state from a given trie.
@ -177,11 +191,20 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error)
// commit phase, most of the needed data is already hot.
func (s *StateDB) StartPrefetcher(namespace string) {
if s.prefetcher != nil {
s.prefetcher.wait()
s.prefetcher.close()
s.prefetcher = nil
}
if s.readPrefetcher != nil {
s.readPrefetcher.wait()
s.readPrefetcher.close()
s.readPrefetcher = nil
}
if s.snap != nil {
s.prefetcher = newTriePrefetcher(s.db, s.originalRoot, namespace)
if s.witness != nil {
s.readPrefetcher = newTriePrefetcher(s.db, s.originalRoot, namespace)
}
}
}
@ -189,9 +212,15 @@ func (s *StateDB) StartPrefetcher(namespace string) {
// from the gathered metrics.
func (s *StateDB) StopPrefetcher() {
if s.prefetcher != nil {
s.prefetcher.wait()
s.prefetcher.close()
s.prefetcher = nil
}
if s.readPrefetcher != nil {
s.readPrefetcher.wait()
s.readPrefetcher.close()
s.readPrefetcher = nil
}
}
// setError remembers the first non-nil error it is called with.
@ -350,7 +379,8 @@ func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash {
func (s *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash {
stateObject := s.getStateObject(addr)
if stateObject != nil {
return stateObject.GetCommittedState(hash)
res := stateObject.GetCommittedState(hash)
return res
}
return common.Hash{}
}
@ -604,9 +634,15 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject {
return nil
}
}
// Insert into the live set
obj := newObject(s, addr, data)
s.setStateObject(obj)
if s.witness != nil && s.snap != nil {
// when building witness with snap enabled, prefetch all read accounts to later be collected and
// included in the witness
s.readPrefetcher.prefetch(common.Hash{}, s.originalRoot, common.Address{}, [][]byte{addr[:]})
}
return obj
}
@ -712,6 +748,9 @@ func (s *StateDB) Copy() *StateDB {
snaps: s.snaps,
snap: s.snap,
}
if s.witness != nil {
state.witness = s.witness.Copy()
}
// Copy the dirty states, logs, and preimages
for addr := range s.journal.dirties {
// As documented [here](https://github.com/ethereum/go-ethereum/pull/16485#issuecomment-380438527),
@ -866,6 +905,33 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) {
s.clearJournalAndRefund()
}
func (s *StateDB) collectReadStorageAccessLists() {
for _, obj := range s.stateObjects {
// load read storage slots from the finished trie in the prefetcher as these continue to be prefetched
// until commit.
tr := s.readPrefetcher.trie(obj.addrHash, obj.data.Root)
if tr == nil {
continue
}
accessList := tr.AccessList()
if len(accessList) > 0 {
s.witness.addAccessList(obj.addrHash, accessList)
}
}
}
func (s *StateDB) collectReadAccountsAccessLists() {
tr := s.readPrefetcher.trie(common.Hash{}, s.originalRoot)
if tr == nil {
// TODO: ensure this case is b/c of empty block
return
}
accessList := tr.AccessList()
if len(accessList) > 0 {
s.witness.addAccessList(common.Hash{}, accessList)
}
}
// IntermediateRoot computes the current root hash of the state trie.
// It is called in between transactions to get the root hash that
// goes into transaction receipts.
@ -873,20 +939,24 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
// Finalise all the dirty storage states and write them into the tries
s.Finalise(deleteEmptyObjects)
// If there was a trie prefetcher operating, it gets aborted and irrevocably
// modified after we start retrieving tries. Remove it from the statedb after
// this round of use.
//
// This is weird pre-byzantium since the first tx runs with a prefetcher and
// the remainder without, but pre-byzantium even the initial prefetcher is
// useless, so no sleep lost.
prefetcher := s.prefetcher
if s.prefetcher != nil {
defer func() {
// TODO: need to wait for read accounts to be resolved in prefetcher main trie?
s.prefetcher.wait()
s.prefetcher.close()
s.prefetcher = nil
}()
}
if s.readPrefetcher != nil {
// TODO: move read prefetcher logic into Commit?
s.readPrefetcher.wait()
s.collectReadStorageAccessLists()
s.collectReadAccountsAccessLists()
s.readPrefetcher.close()
s.readPrefetcher = nil
}
// Although naively it makes sense to retrieve the account trie and then do
// the contract storage and account updates sequentially, that short circuits
// the account prefetcher. Instead, let's process all the storage updates
@ -897,6 +967,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
obj.updateRoot()
}
}
// Now we're about to start to write changes to the trie. The trie is so far
// _untouched_. We can check with the prefetcher, if it can give us a trie
// which has the same root, but also has some content loaded into it.
@ -906,16 +977,29 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
}
}
usedAddrs := make([][]byte, 0, len(s.stateObjectsPending))
// perform updates before deletions. In the case where a full node
// has two children, one of them selfdestructs and makes the recipient
// another non-existing sibling, applying deletion before update would
// result in the unecessary premature collapse of the full node into a short node
// for the untouched third sibling.
var deletedObjects []*stateObject
for addr := range s.stateObjectsPending {
if obj := s.stateObjects[addr]; obj.deleted {
s.deleteStateObject(obj)
s.AccountDeleted += 1
} else {
if obj := s.stateObjects[addr]; !obj.deleted {
s.updateStateObject(obj)
s.AccountUpdated += 1
usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure
} else {
deletedObjects = append(deletedObjects, obj)
}
usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure
usedAddrs = append(usedAddrs, common.CopyBytes(addr[:]))
}
for _, deletedObj := range deletedObjects {
s.deleteStateObject(deletedObj)
s.AccountDeleted += 1
usedAddrs = append(usedAddrs, common.CopyBytes(deletedObj.address[:])) // Copy needed for closure
}
if prefetcher != nil {
prefetcher.used(common.Hash{}, s.originalRoot, usedAddrs)
}
@ -929,6 +1013,13 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
return s.trie.Hash()
}
// makes the statedb configure to build a stateless block witness
// this must be called after initializing a statedb and before
// starting prefetchers or applying state changes
func (s *StateDB) EnableWitnessRecording() {
s.witness = NewWitness(s.originalRoot)
}
// SetTxContext sets the current transaction hash and index which are
// used when the EVM emits new state logs. It should be invoked before
// transaction execution.
@ -1153,6 +1244,28 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A
return incomplete, nil
}
// Witness returns a block witness object being constructed or nil if the
// StateDB instance is not configured to record stateless witnesses.
func (s *StateDB) Witness() *Witness {
return s.witness
}
// ApplyWithdrawals credits the balance of each account that is the recipient
// of a withdrawal.
func (s *StateDB) ApplyWithdrawals(withdrawals types.Withdrawals) {
for _, w := range withdrawals {
// Convert amount from gwei to wei.
amount := new(uint256.Int).SetUint64(w.Amount)
amount = amount.Mul(amount, uint256.NewInt(params.GWei))
s.AddBalance(w.Address, amount)
}
if s.witness != nil {
al := s.trie.AccessList()
s.witness.addAccessList(common.Hash{}, al)
}
}
// Commit writes the state to the underlying in-memory trie database.
// Once the state is committed, tries cached in stateDB (including account
// trie, storage tries) will no longer be functional. A new state instance
@ -1177,6 +1290,8 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er
storageTrieNodesDeleted int
nodes = trienode.NewMergedNodeSet()
codeWriter = s.db.DiskDB().NewBatch()
root common.Hash
set *trienode.NodeSet
)
// Handle all state deletions first
incomplete, err := s.handleDestruction(nodes)
@ -1184,6 +1299,31 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er
return common.Hash{}, err
}
// Handle all state updates afterwards
if s.witness != nil {
if s.snap == nil {
// if the snapshot is not in use, all read state values and their intermediate
// nodes are resolved in the statedb/object tries.
//
// we collect access lists after commit because there are circumstances where
// computation of a new trie root hash where a value has been deleted could
// collapse a parent branch node + sibling into a full node for the sibling.
// In this case, the sibling would only be read during root hash computation.
// TODO: verify the above assertion is the case.
for addr := range s.stateObjects {
obj := s.stateObjects[addr]
if _, ok := s.stateObjectsDirty[addr]; ok && !obj.deleted {
// collect dirty object access witness if/when we commit them
continue
}
if obj.trie != nil {
al := obj.trie.AccessList()
s.witness.addAccessList(obj.addrHash, al)
}
}
accessList := s.trie.AccessList()
s.witness.addAccessList(common.Hash{}, accessList)
}
}
for addr := range s.stateObjectsDirty {
obj := s.stateObjects[addr]
if obj.deleted {
@ -1194,11 +1334,19 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er
rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code)
obj.dirtyCode = false
}
// Write any storage changes in the state object to its storage trie
set, err := obj.commit()
set, accessList, err := obj.commit()
if err != nil {
return common.Hash{}, err
}
if s.witness != nil {
// storage trie nodes for writes accrue in the state object's trie instance
// storage trie nodes from read slots are accrued in the prefetcher and
// retrieved in IntermediateRoot
s.witness.addAccessList(obj.addrHash, accessList)
}
// Merge the dirty nodes of storage trie into global set. It is possible
// that the account was destructed and then resurrected in the same block.
// In this case, the node set is shared by both accounts.
@ -1221,7 +1369,14 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er
if metrics.EnabledExpensive {
start = time.Now()
}
root, set, err := s.trie.Commit(true)
if s.witness != nil {
var accessList map[string][]byte
root, set, accessList, err = s.trie.CommitAndObtainAccessList(true)
s.witness.addAccessList(common.Hash{}, accessList)
} else {
root, set, err = s.trie.Commit(true)
}
if err != nil {
return common.Hash{}, err
}
@ -1295,6 +1450,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er
s.storagesOrigin = make(map[common.Address]map[common.Hash][]byte)
s.stateObjectsDirty = make(map[common.Address]struct{})
s.stateObjectsDestruct = make(map[common.Address]*types.StateAccount)
return root, nil
}

161
core/state/trie_prefetcher.go
View file

@ -17,6 +17,7 @@
package state
import (
"fmt"
"sync"
"github.com/ethereum/go-ethereum/common"
@ -70,13 +71,16 @@ func newTriePrefetcher(db Database, root common.Hash, namespace string) *triePre
}
return p
}
func (p *triePrefetcher) wait() {
for _, fetcher := range p.fetchers {
fetcher.wait()
}
}
// close iterates over all the subfetchers, aborts any that were left spinning
// and reports the stats to the metrics subsystem.
func (p *triePrefetcher) close() {
for _, fetcher := range p.fetchers {
fetcher.abort() // safe to do multiple times
if metrics.Enabled {
if fetcher.root == p.root {
p.accountLoadMeter.Mark(int64(len(fetcher.seen)))
@ -123,29 +127,22 @@ func (p *triePrefetcher) copy() *triePrefetcher {
storageSkipMeter: p.storageSkipMeter,
storageWasteMeter: p.storageWasteMeter,
}
// If the prefetcher is already a copy, duplicate the data
if p.fetches != nil {
for root, fetch := range p.fetches {
if fetch == nil {
continue
}
copy.fetches[root] = p.db.CopyTrie(fetch)
}
return copy
}
// Otherwise we're copying an active fetcher, retrieve the current states
for id, fetcher := range p.fetchers {
copy.fetches[id] = fetcher.peek()
}
return copy
}
// prefetch schedules a batch of trie items to prefetch.
// prefetch is called from two locations:
// 1. Finalize of the state-objects storage roots. This happens at the end
// of every transaction, meaning that if several transactions touches
// upon the same contract, the parameters invoking this method may be
// repeated.
// 2. Finalize of the main account trie. This happens only once per block.
func (p *triePrefetcher) prefetch(owner common.Hash, root common.Hash, addr common.Address, keys [][]byte) {
// If the prefetcher is an inactive one, bail out
if p.fetches != nil {
return
}
// Active fetcher, schedule the retrievals
id := p.trieID(owner, root)
fetcher := p.fetchers[id]
@ -175,16 +172,13 @@ func (p *triePrefetcher) trie(owner common.Hash, root common.Hash) Trie {
p.deliveryMissMeter.Mark(1)
return nil
}
// Interrupt the prefetcher if it's by any chance still running and return
// a copy of any pre-loaded trie.
fetcher.abort() // safe to do multiple times
trie := fetcher.peek()
if trie == nil {
// Wait for the fetcher to finish
fetcher.wait() // safe to do multiple times
if fetcher.trie == nil {
p.deliveryMissMeter.Mark(1)
return nil
}
return trie
return fetcher.db.CopyTrie(fetcher.trie)
}
// used marks a batch of state items used to allow creating statistics as to
@ -215,13 +209,12 @@ type subfetcher struct {
addr common.Address // Address of the account that the trie belongs to
trie Trie // Trie being populated with nodes
tasks [][]byte // Items queued up for retrieval
lock sync.Mutex // Lock protecting the task queue
tasks [][]byte // Items queued up for retrieval
lock sync.Mutex // Lock protecting the task queue
closing bool // set to true if the subfetcher is closing
wake chan struct{} // Wake channel if a new task is scheduled
stop chan struct{} // Channel to interrupt processing
term chan struct{} // Channel to signal interruption
copy chan chan Trie // Channel to request a copy of the current trie
wake chan bool // Wake channel if a new task is scheduled, true if the subfetcher should continue running when there are no pending tasks
term chan struct{} // Channel to signal interruption
seen map[string]struct{} // Tracks the entries already loaded
dups int // Number of duplicate preload tasks
@ -237,10 +230,8 @@ func newSubfetcher(db Database, state common.Hash, owner common.Hash, root commo
owner: owner,
root: root,
addr: addr,
wake: make(chan struct{}, 1),
stop: make(chan struct{}),
wake: make(chan bool, 1),
term: make(chan struct{}),
copy: make(chan chan Trie),
seen: make(map[string]struct{}),
}
go sf.loop()
@ -251,42 +242,30 @@ func newSubfetcher(db Database, state common.Hash, owner common.Hash, root commo
func (sf *subfetcher) schedule(keys [][]byte) {
// Append the tasks to the current queue
sf.lock.Lock()
sf.tasks = append(sf.tasks, keys...)
sf.lock.Unlock()
// Notify the prefetcher, it's fine if it's already terminated
// Notify the prefetcher. The wake-chan is buffered, so this is async.
select {
case sf.wake <- struct{}{}:
case sf.wake <- true:
default:
}
}
// peek tries to retrieve a deep copy of the fetcher's trie in whatever form it
// is currently.
func (sf *subfetcher) peek() Trie {
ch := make(chan Trie)
select {
case sf.copy <- ch:
// Subfetcher still alive, return copy from it
return <-ch
case <-sf.term:
// Subfetcher already terminated, return a copy directly
if sf.trie == nil {
return nil
}
return sf.db.CopyTrie(sf.trie)
}
}
// abort interrupts the subfetcher immediately. It is safe to call abort multiple
// wait waits for the subfetcher to finish it's task. It is safe to call wait multiple
// times but it is not thread safe.
func (sf *subfetcher) abort() {
select {
case <-sf.stop:
default:
close(sf.stop)
func (sf *subfetcher) wait() {
// Signal termination by nil tasks
sf.lock.Lock()
if sf.closing {
sf.lock.Unlock()
return // already exiting
}
sf.closing = true
sf.lock.Unlock()
// Notify the prefetcher. The wake-chan is buffered, so this is async.
sf.wake <- false
// Wait for it to terminate
<-sf.term
}
@ -316,50 +295,32 @@ func (sf *subfetcher) loop() {
}
// Trie opened successfully, keep prefetching items
for {
select {
case <-sf.wake:
// Subfetcher was woken up, retrieve any tasks to avoid spinning the lock
sf.lock.Lock()
tasks := sf.tasks
sf.tasks = nil
sf.lock.Unlock()
keepRunning := <-sf.wake
if !keepRunning {
return
}
// Subfetcher was woken up, retrieve any tasks to avoid spinning the lock
sf.lock.Lock()
tasks := sf.tasks
sf.tasks = nil
sf.lock.Unlock()
// Prefetch any tasks until the loop is interrupted
for i, task := range tasks {
select {
case <-sf.stop:
// If termination is requested, add any leftover back and return
sf.lock.Lock()
sf.tasks = append(sf.tasks, tasks[i:]...)
sf.lock.Unlock()
return
case ch := <-sf.copy:
// Somebody wants a copy of the current trie, grant them
ch <- sf.db.CopyTrie(sf.trie)
default:
// No termination request yet, prefetch the next entry
if _, ok := sf.seen[string(task)]; ok {
sf.dups++
} else {
if len(task) == common.AddressLength {
sf.trie.GetAccount(common.BytesToAddress(task))
} else {
sf.trie.GetStorage(sf.addr, task)
}
sf.seen[string(task)] = struct{}{}
}
// Prefetch all tasks
for _, task := range tasks {
if _, ok := sf.seen[string(task)]; ok {
sf.dups++
continue
}
if len(task) == common.AddressLength {
sf.trie.GetAccount(common.BytesToAddress(task))
} else {
_, err := sf.trie.GetStorage(sf.addr, task)
if err != nil {
// TODO: see what needs to be done in this case
fmt.Printf("prefetch storage failed: %+v\n", err)
}
}
case ch := <-sf.copy:
// Somebody wants a copy of the current trie, grant them
ch <- sf.db.CopyTrie(sf.trie)
case <-sf.stop:
// Termination is requested, abort and leave remaining tasks
return
sf.seen[string(task)] = struct{}{}
}
}
}

12
core/types/block.go
View file

@ -319,6 +319,18 @@ func (b *Block) DecodeRLP(s *rlp.Stream) error {
return nil
}
// EncodeRLPWithZeroRoot encodes a block (with header state root set to 0x00...0) to RLP
func (b *Block) EncodeRLPWithZeroRoot(w io.Writer) error {
old := b.header.Root
b.header.Root = common.Hash{}
err := b.EncodeRLP(w)
b.header.Root = old
if err != nil {
return err
}
return nil
}
// EncodeRLP serializes a block as RLP.
func (b *Block) EncodeRLP(w io.Writer) error {
return rlp.Encode(w, &extblock{

16
core/vm/evm.go
View file

@ -181,6 +181,7 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas
if evm.depth > int(params.CallCreateDepth) {
return nil, gas, ErrDepth
}
// Fail if we're trying to transfer more than the available balance
if !value.IsZero() && !evm.Context.CanTransfer(evm.StateDB, caller.Address(), value) {
return nil, gas, ErrInsufficientBalance
@ -188,7 +189,6 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas
snapshot := evm.StateDB.Snapshot()
p, isPrecompile := evm.precompile(addr)
debug := evm.Config.Tracer != nil
if !evm.StateDB.Exist(addr) {
if !isPrecompile && evm.chainRules.IsEIP158 && value.IsZero() {
// Calling a non existing account, don't do anything, but ping the tracer
@ -229,6 +229,11 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas
// Initialise a new contract and set the code that is to be used by the EVM.
// The contract is a scoped environment for this execution context only.
code := evm.StateDB.GetCode(addr)
codeCopy := make([]byte, len(code))
copy(codeCopy[:], code[:])
if witness := evm.StateDB.Witness(); witness != nil {
witness.AddCode(evm.StateDB.GetCodeHash(addr), codeCopy)
}
if len(code) == 0 {
ret, err = nil, nil // gas is unchanged
} else {
@ -293,6 +298,9 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte,
// Initialise a new contract and set the code that is to be used by the EVM.
// The contract is a scoped environment for this execution context only.
contract := NewContract(caller, AccountRef(caller.Address()), value, gas)
if witness := evm.StateDB.Witness(); witness != nil {
witness.AddCode(evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy))
}
contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy))
ret, err = evm.interpreter.Run(contract, input, false)
gas = contract.Gas
@ -337,6 +345,9 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by
addrCopy := addr
// Initialise a new contract and make initialise the delegate values
contract := NewContract(caller, AccountRef(caller.Address()), nil, gas).AsDelegate()
if witness := evm.StateDB.Witness(); witness != nil {
witness.AddCode(evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy))
}
contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy))
ret, err = evm.interpreter.Run(contract, input, false)
gas = contract.Gas
@ -390,6 +401,9 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
// Initialise a new contract and set the code that is to be used by the EVM.
// The contract is a scoped environment for this execution context only.
contract := NewContract(caller, AccountRef(addrCopy), new(uint256.Int), gas)
if witness := evm.StateDB.Witness(); witness != nil {
witness.AddCode(evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy))
}
contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy))
// When an error was returned by the EVM or when setting the creation code
// above we revert to the snapshot and consume any gas remaining. Additionally

30
core/vm/instructions.go
View file

@ -17,8 +17,6 @@
package vm
import (
"math"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
@ -344,7 +342,13 @@ func opReturnDataCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeConte
func opExtCodeSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) {
slot := scope.Stack.peek()
slot.SetUint64(uint64(interpreter.evm.StateDB.GetCodeSize(slot.Bytes20())))
address := slot.Bytes20()
slot.SetUint64(uint64(interpreter.evm.StateDB.GetCodeSize(address)))
if witness := interpreter.evm.StateDB.Witness(); witness != nil {
code := interpreter.evm.StateDB.GetCode(address)
codeHash := interpreter.evm.StateDB.GetCodeHash(address)
witness.AddCode(codeHash, code)
}
return nil, nil
}
@ -361,7 +365,7 @@ func opCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([
)
uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow()
if overflow {
uint64CodeOffset = math.MaxUint64
uint64CodeOffset = 0xffffffffffffffff
}
codeCopy := getData(scope.Contract.Code, uint64CodeOffset, length.Uint64())
scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy)
@ -379,9 +383,13 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext)
)
uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow()
if overflow {
uint64CodeOffset = math.MaxUint64
uint64CodeOffset = 0xffffffffffffffff
}
addr := common.Address(a.Bytes20())
if witness := interpreter.evm.StateDB.Witness(); witness != nil {
witness.AddCode(interpreter.evm.StateDB.GetCodeHash(addr), interpreter.evm.StateDB.GetCode(addr))
}
codeCopy := getData(interpreter.evm.StateDB.GetCode(addr), uint64CodeOffset, length.Uint64())
scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy)
@ -420,6 +428,10 @@ func opExtCodeHash(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext)
if interpreter.evm.StateDB.Empty(address) {
slot.Clear()
} else {
_ = interpreter.evm.StateDB.GetCode(address) // ensure the account leaf is fetched and included in the witness
if witness := interpreter.evm.StateDB.Witness(); witness != nil {
witness.AddCodeHash(interpreter.evm.StateDB.GetCodeHash(address))
}
slot.SetBytes(interpreter.evm.StateDB.GetCodeHash(address).Bytes())
}
return nil, nil
@ -446,7 +458,13 @@ func opBlockhash(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) (
lower = upper - 256
}
if num64 >= lower && num64 < upper {
num.SetBytes(interpreter.evm.Context.GetHash(num64).Bytes())
res := interpreter.evm.Context.GetHash(num64).Bytes()
if witness := interpreter.evm.StateDB.Witness(); witness != nil {
var bh common.Hash
copy(bh[:], res[:])
witness.AddBlockHash(bh, num64)
}
num.SetBytes(res[:])
} else {
num.Clear()
}

4
core/vm/interface.go
View file

@ -19,6 +19,8 @@ package vm
import (
"math/big"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
@ -79,6 +81,8 @@ type StateDB interface {
AddLog(*types.Log)
AddPreimage(common.Hash, []byte)
Witness() *state.Witness
}
// CallContext provides a basic interface for the EVM calling conventions. The EVM

49
eth/api_debug.go
View file

@ -20,8 +20,13 @@ import (
"context"
"errors"
"fmt"
"github.com/ethereum/go-ethereum/eth/tracers/logger"
"os"
"time"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/rawdb"
@ -443,3 +448,47 @@ func (api *DebugAPI) GetTrieFlushInterval() (string, error) {
}
return api.eth.blockchain.GetTrieFlushInterval().String(), nil
}
func BuildProof(number uint64, bc *core.BlockChain) ([]byte, error) {
if number == 0 {
panic("cannot build genesis block proof")
}
parent := bc.GetBlockByNumber(number - 1)
db, err := bc.StateAt(parent.Header().Root)
if err != nil {
return nil, err
}
db.EnableWitnessRecording()
db.StartPrefetcher("apidebug")
block := bc.GetBlockByNumber(number)
logconfig := &logger.Config{
EnableMemory: false,
DisableStack: false,
DisableStorage: false,
EnableReturnData: true,
Debug: true,
}
tracer := logger.NewJSONLogger(logconfig, os.Stdout)
_ = tracer
stateProcessor := core.NewStateProcessor(bc.Config(), bc, bc.Engine())
_, _, _, err = stateProcessor.Process(block, db, vm.Config{})
if err != nil {
return nil, err
}
if _, err = db.Commit(block.NumberU64(), true); err != nil {
return nil, err
}
proof := db.Witness()
proof.Block = block
enc, err := proof.EncodeRLP()
if err != nil {
return nil, err
}
return enc, nil
}
func (api *DebugAPI) BuildProof(num rpc.BlockNumber) ([]byte, error) {
return BuildProof(uint64(num), api.eth.blockchain)
}

5
internal/web3ext/web3ext.go
View file

@ -501,6 +501,11 @@ web3._extend({
call: 'debug_getTrieFlushInterval',
params: 0
}),
new web3._extend.Method({
name: 'buildProof',
call: 'debug_buildProof',
params: 1
}),
],
properties: []
});

109
tests/block_test.go
View file

@ -61,8 +61,92 @@ func TestBlockchain(t *testing.T) {
// which run natively, so there's no reason to run them here.
}
// TestExecutionSpecBlocktests runs the test fixtures from execution-spec-tests.
func TestExecutionSpecBlocktests(t *testing.T) {
func networkPostMerge(network string) bool {
switch network {
case "Frontier":
return false
case "EIP150":
return false
case "EIP158":
return false
case "Byzantium":
return false
case "Constantinople":
return false
case "ConstantinopleFix":
return false
case "Istanbul":
return false
case "MuirGlacier":
return false
case "Berlin":
return false
case "London":
return false
case "ArrowGlacier":
return false
case "GreyGlacier":
return false
case "Merge":
return true
case "Shanghai":
return true
case "Cancun":
return true
}
return false
}
func TestStatelessBlockchain(t *testing.T) {
bt := new(testMatcher)
// Skip random failures due to selfish mining test
bt.skipLoad(`.*bcForgedTest/bcForkUncle\.json`)
// Slow tests
bt.slow(`.*bcExploitTest/DelegateCallSpam.json`)
bt.slow(`.*bcExploitTest/ShanghaiLove.json`)
bt.slow(`.*bcExploitTest/SuicideIssue.json`)
bt.slow(`.*/bcForkStressTest/`)
bt.slow(`.*/bcGasPricerTest/RPC_API_Test.json`)
bt.slow(`.*/bcWalletTest/`)
// Very slow test
bt.skipLoad(`.*/stTimeConsuming/.*`)
// test takes a lot for time and goes easily OOM because of sha3 calculation on a huge range,
// using 4.6 TGas
bt.skipLoad(`.*randomStatetest94.json.*`)
// skip uncle tests for stateless
bt.skipLoad(`.*/UnclePopulation.json`)
// skip this test in stateless because it uses 5000 blocks and the
// historical state of older blocks is unavailable for stateless
// test verification after importing the test set.
bt.skipLoad(`.*/bcWalletTest/walletReorganizeOwners.json`)
bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) {
if runtime.GOARCH == "386" && runtime.GOOS == "windows" && rand.Int63()%2 == 0 {
t.Skip("test (randomly) skipped on 32-bit windows")
}
config, ok := Forks[test.json.Network]
if !ok {
t.Fatalf("test malformed: doesn't have chain config embedded")
}
isMerged := config.TerminalTotalDifficulty != nil && config.TerminalTotalDifficulty.BitLen() == 0
if isMerged {
execBlockTestStateless(t, bt, test)
} else {
t.Skip("skipping pre-merge test")
}
})
// There is also a LegacyTests folder, containing blockchain tests generated
// prior to Istanbul. However, they are all derived from GeneralStateTests,
// which run natively, so there's no reason to run them here.
}
// TestExecutionSpec runs the test fixtures from execution-spec-tests.
func TestExecutionSpec(t *testing.T) {
if !common.FileExist(executionSpecBlockchainTestDir) {
t.Skipf("directory %s does not exist", executionSpecBlockchainTestDir)
}
@ -91,3 +175,24 @@ func execBlockTest(t *testing.T, bt *testMatcher, test *BlockTest) {
return
}
}
func execBlockTestStateless(t *testing.T, bt *testMatcher, test *BlockTest) {
if err := bt.checkFailure(t, test.RunStateless(false, rawdb.HashScheme, nil, nil)); err != nil {
t.Errorf("test in hash mode without snapshotter failed: %v", err)
return
}
if err := bt.checkFailure(t, test.RunStateless(true, rawdb.HashScheme, nil, nil)); err != nil {
t.Errorf("test in hash mode with snapshotter failed: %v", err)
return
}
if err := bt.checkFailure(t, test.RunStateless(false, rawdb.PathScheme, nil, nil)); err != nil {
t.Errorf("test in path mode without snapshotter failed: %v", err)
return
}
if err := bt.checkFailure(t, test.RunStateless(true, rawdb.PathScheme, nil, nil)); err != nil {
t.Errorf("test in path mode with snapshotter failed: %v", err)
return
}
}

30
tests/block_test_util.go
View file

@ -26,6 +26,8 @@ import (
"os"
"reflect"
"github.com/ethereum/go-ethereum/eth"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/common/math"
@ -57,8 +59,8 @@ func (t *BlockTest) UnmarshalJSON(in []byte) error {
type btJSON struct {
Blocks []btBlock `json:"blocks"`
Genesis btHeader `json:"genesisBlockHeader"`
Pre types.GenesisAlloc `json:"pre"`
Post types.GenesisAlloc `json:"postState"`
Pre core.GenesisAlloc `json:"pre"`
Post core.GenesisAlloc `json:"postState"`
BestBlock common.UnprefixedHash `json:"lastblockhash"`
Network string `json:"network"`
SealEngine string `json:"sealEngine"`
@ -109,7 +111,15 @@ type btHeaderMarshaling struct {
ExcessBlobGas *math.HexOrDecimal64
}
func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, postCheck func(error, *core.BlockChain)) (result error) {
func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, postCheck func(error, *core.BlockChain)) error {
return t.run(false, snapshotter, scheme, tracer, postCheck)
}
func (t *BlockTest) RunStateless(snapshotter bool, scheme string, tracer vm.EVMLogger, postCheck func(error, *core.BlockChain)) error {
return t.run(true, snapshotter, scheme, tracer, postCheck)
}
func (t *BlockTest) run(stateless bool, snapshotter bool, scheme string, tracer vm.EVMLogger, postCheck func(error, *core.BlockChain)) (result error) {
config, ok := Forks[t.json.Network]
if !ok {
return UnsupportedForkError{t.json.Network}
@ -126,6 +136,7 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, po
} else {
tconf.HashDB = hashdb.Defaults
}
// Commit genesis state
gspec := t.genesis(config)
triedb := triedb.NewDatabase(db, tconf)
@ -144,7 +155,7 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, po
// Wrap the original engine within the beacon-engine
engine := beacon.New(ethash.NewFaker())
cache := &core.CacheConfig{TrieCleanLimit: 0, StateScheme: scheme, Preimages: true}
cache := &core.CacheConfig{TrieCleanLimit: 0, StateScheme: scheme, Preimages: true, TrieDirtyDisabled: true}
if snapshotter {
cache.SnapshotLimit = 1
cache.SnapshotWait = true
@ -183,6 +194,17 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer vm.EVMLogger, po
return err
}
}
if stateless {
for _, blk := range validBlocks {
_, err := eth.BuildProof(blk.BlockHeader.Number.Uint64(), chain)
if err != nil {
return fmt.Errorf("failed to build proof: %v", err)
}
// TODO: decode and execute the stateless proof, verify the produced root
// matches the block root.
}
}
return t.validateImportedHeaders(chain, validBlocks)
}

17
trie/secure_trie.go
View file

@ -213,6 +213,23 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte {
}
return t.db.Preimage(common.BytesToHash(shaKey))
}
func (t *StateTrie) AccessList() map[string][]byte {
return t.trie.AccessList()
}
func (t *StateTrie) CommitAndObtainAccessList(collectLeaf bool) (common.Hash, *trienode.NodeSet, map[string][]byte, error) {
// Write all the pre-images to the actual disk database
if len(t.getSecKeyCache()) > 0 {
preimages := make(map[common.Hash][]byte)
for hk, key := range t.secKeyCache {
preimages[common.BytesToHash([]byte(hk))] = key
}
t.db.InsertPreimage(preimages)
t.secKeyCache = make(map[string][]byte)
}
// Commit the trie and return its modified nodeset.
return t.trie.CommitAndObtainAccessList(collectLeaf)
}
// Commit collects all dirty nodes in the trie and replaces them with the
// corresponding node hash. All collected nodes (including dirty leaves if

24
trie/trie.go
View file

@ -107,7 +107,7 @@ func NewEmpty(db database.Database) *Trie {
}
// MustNodeIterator is a wrapper of NodeIterator and will omit any encountered
// error but just print out an error message.
// error but just printg out an error message.
func (t *Trie) MustNodeIterator(start []byte) NodeIterator {
it, err := t.NodeIterator(start)
if err != nil {
@ -581,6 +581,10 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) {
return n, nil
}
// TODO: for resolveAndTrack, differentiate between hash node resolve failure in stateless
// vs normal execution. In normal mode, it represents an error with database
// consistency. In stateless execution, it means that the witness is incomplete.
// resolveAndTrack loads node from the underlying store with the given node hash
// and path prefix and also tracks the loaded node blob in tracer treated as the
// node's original value. The rlp-encoded blob is preferred to be loaded from
@ -602,6 +606,22 @@ func (t *Trie) Hash() common.Hash {
return common.BytesToHash(hash.(hashNode))
}
func (t *Trie) AccessList() map[string][]byte {
return t.tracer.accessList
}
func (t *Trie) CommitAndObtainAccessList(collectLeaf bool) (common.Hash, *trienode.NodeSet, map[string][]byte, error) {
accessList := t.tracer.accessList
// Commit will reset the tracer accessList, so after this
// operation, we have full ownership of the map (hence: no need to
// deep-copy or even copy).
rootHash, nodes, err := t.Commit(collectLeaf)
if err != nil {
return rootHash, nodes, nil, err
}
return rootHash, nodes, accessList, err
}
// Commit collects all dirty nodes in the trie and replaces them with the
// corresponding node hash. All collected nodes (including dirty leaves if
// collectLeaf is true) will be encapsulated into a nodeset for return.
@ -609,8 +629,8 @@ func (t *Trie) Hash() common.Hash {
// Once the trie is committed, it's not usable anymore. A new trie must
// be created with new root and updated trie database for following usage
func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) {
defer t.tracer.reset()
defer func() {
t.tracer.reset()
t.committed = true
}()
// Trie is empty and can be classified into two types of situations:

8
trie/verkle.go
View file

@ -216,6 +216,14 @@ func (t *VerkleTrie) Hash() common.Hash {
return t.root.Commit().Bytes()
}
func (t *VerkleTrie) AccessList() map[string][]byte {
panic("not implemented")
}
func (t *VerkleTrie) CommitAndObtainAccessList(collectLeaf bool) (common.Hash, *trienode.NodeSet, map[string][]byte, error) {
panic("not implemented")
}
// Commit writes all nodes to the tree's memory database.
func (t *VerkleTrie) Commit(_ bool) (common.Hash, *trienode.NodeSet, error) {
root, ok := t.root.(*verkle.InternalNode)

5
triedb/database.go
View file

@ -45,6 +45,11 @@ var HashDefaults = &Config{
HashDB: hashdb.Defaults,
}
var PathDefaults = &Config{
Preimages: false,
PathDB: pathdb.Defaults,
}
// backend defines the methods needed to access/update trie nodes in different
// state scheme.
type backend interface {