Merge branch 'ethereum:master' into portal

.github/CODEOWNERS

@@ -4,14 +4,18 @@
 accounts/usbwallet              @karalabe
 accounts/scwallet               @gballet
 accounts/abi                    @gballet @MariusVanDerWijden
+beacon/engine                   @lightclient
 cmd/clef                        @holiman
+cmd/evm                         @holiman @MariusVanDerWijden @lightclient
 consensus                       @karalabe
 core/                           @karalabe @holiman @rjl493456442
 eth/                            @karalabe @holiman @rjl493456442
-eth/catalyst/                   @gballet
+eth/catalyst/                   @gballet @lightclient
 eth/tracers/                    @s1na
 core/tracing/                   @s1na
 graphql/                        @s1na
+internal/ethapi                 @lightclient
+internal/era                    @lightclient
 les/                            @zsfelfoldi @rjl493456442
 light/                          @zsfelfoldi @rjl493456442
 node/                           @fjl

accounts/keystore/account_cache_test.go

@@ -325,11 +325,7 @@ func TestUpdatedKeyfileContents(t *testing.T) {
 	t.Parallel()
 
 	// Create a temporary keystore to test with
-	dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-updatedkeyfilecontents-test-%d-%d", os.Getpid(), rand.Int()))
+	dir := t.TempDir()
-
-	// Create the directory
-	os.MkdirAll(dir, 0700)
-	defer os.RemoveAll(dir)
 
 	ks := NewKeyStore(dir, LightScryptN, LightScryptP)
 

cmd/blsync/main.go

@@ -19,39 +19,21 @@ package main
 import (
 	"context"
 	"fmt"
-	"io"
 	"os"
 
 	"github.com/ethereum/go-ethereum/beacon/blsync"
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/mattn/go-colorable"
-	"github.com/mattn/go-isatty"
 	"github.com/urfave/cli/v2"
 )
 
-var (
-	verbosityFlag = &cli.IntFlag{
-		Name:     "verbosity",
-		Usage:    "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail",
-		Value:    3,
-		Category: flags.LoggingCategory,
-	}
-	vmoduleFlag = &cli.StringFlag{
-		Name:     "vmodule",
-		Usage:    "Per-module verbosity: comma-separated list of <pattern>=<level> (e.g. eth/*=5,p2p=4)",
-		Value:    "",
-		Hidden:   true,
-		Category: flags.LoggingCategory,
-	}
-)
-
 func main() {
 	app := flags.NewApp("beacon light syncer tool")
-	app.Flags = []cli.Flag{
+	app.Flags = flags.Merge([]cli.Flag{
 		utils.BeaconApiFlag,
 		utils.BeaconApiHeaderFlag,
 		utils.BeaconThresholdFlag,
@@ -66,8 +48,16 @@ func main() {
 		utils.GoerliFlag,
 		utils.BlsyncApiFlag,
 		utils.BlsyncJWTSecretFlag,
-		verbosityFlag,
+	},
-		vmoduleFlag,
+		debug.Flags,
+	)
+	app.Before = func(ctx *cli.Context) error {
+		flags.MigrateGlobalFlags(ctx)
+		return debug.Setup(ctx)
+	}
+	app.After = func(ctx *cli.Context) error {
+		debug.Exit()
+		return nil
 	}
 	app.Action = sync
 
@@ -78,14 +68,6 @@ func main() {
 }
 
 func sync(ctx *cli.Context) error {
-	usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb"
-	output := io.Writer(os.Stderr)
-	if usecolor {
-		output = colorable.NewColorable(os.Stderr)
-	}
-	verbosity := log.FromLegacyLevel(ctx.Int(verbosityFlag.Name))
-	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(output, verbosity, usecolor)))
-
 	// set up blsync
 	client := blsync.NewClient(ctx)
 	client.SetEngineRPC(makeRPCClient(ctx))

cmd/evm/blockrunner.go

@@ -86,7 +86,7 @@ func blockTestCmd(ctx *cli.Context) error {
 			continue
 		}
 		test := tests[name]
-		if err := test.Run(false, rawdb.HashScheme, tracer, func(res error, chain *core.BlockChain) {
+		if err := test.Run(false, rawdb.HashScheme, false, tracer, func(res error, chain *core.BlockChain) {
 			if ctx.Bool(DumpFlag.Name) {
 				if state, _ := chain.State(); state != nil {
 					fmt.Println(string(state.Dump(nil)))

cmd/evm/internal/t8ntool/execution.go

@@ -306,7 +306,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 				if tracer.Hooks.OnTxEnd != nil {
 					tracer.Hooks.OnTxEnd(receipt, nil)
 				}
-				writeTraceResult(tracer, traceOutput)
+				if err = writeTraceResult(tracer, traceOutput); err != nil {
+					log.Warn("Error writing tracer output", "err", err)
+				}
 			}
 		}
 
@@ -323,7 +325,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		var (
 			blockReward = big.NewInt(miningReward)
 			minerReward = new(big.Int).Set(blockReward)
-			perOmmer    = new(big.Int).Div(blockReward, big.NewInt(32))
+			perOmmer    = new(big.Int).Rsh(blockReward, 5)
 		)
 		for _, ommer := range pre.Env.Ommers {
 			// Add 1/32th for each ommer included
@@ -332,7 +334,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 			reward := big.NewInt(8)
 			reward.Sub(reward, new(big.Int).SetUint64(ommer.Delta))
 			reward.Mul(reward, blockReward)
-			reward.Div(reward, big.NewInt(8))
+			reward.Rsh(reward, 3)
 			statedb.AddBalance(ommer.Address, uint256.MustFromBig(reward), tracing.BalanceIncreaseRewardMineUncle)
 		}
 		statedb.AddBalance(pre.Env.Coinbase, uint256.MustFromBig(minerReward), tracing.BalanceIncreaseRewardMineBlock)

consensus/ethash/consensus.go

@@ -562,12 +562,6 @@ func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) {
 	return hash
 }
 
-// Some weird constants to avoid constant memory allocs for them.
-var (
-	u256_8  = uint256.NewInt(8)
-	u256_32 = uint256.NewInt(32)
-)
-
 // accumulateRewards credits the coinbase of the given block with the mining
 // reward. The total reward consists of the static block reward and rewards for
 // included uncles. The coinbase of each uncle block is also rewarded.
@@ -589,10 +583,10 @@ func accumulateRewards(config *params.ChainConfig, stateDB *state.StateDB, heade
 		r.AddUint64(uNum, 8)
 		r.Sub(r, hNum)
 		r.Mul(r, blockReward)
-		r.Div(r, u256_8)
+		r.Rsh(r, 3)
 		stateDB.AddBalance(uncle.Coinbase, r, tracing.BalanceIncreaseRewardMineUncle)
 
-		r.Div(blockReward, u256_32)
+		r.Rsh(blockReward, 5)
 		reward.Add(reward, r)
 	}
 	stateDB.AddBalance(header.Coinbase, reward, tracing.BalanceIncreaseRewardMineBlock)

core/block_validator.go

@@ -20,8 +20,10 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/stateless"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
@@ -34,14 +36,12 @@ import (
 type BlockValidator struct {
 	config *params.ChainConfig // Chain configuration options
 	bc     *BlockChain         // Canonical block chain
-	engine consensus.Engine    // Consensus engine used for validating
 }
 
 // NewBlockValidator returns a new block validator which is safe for re-use
-func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine) *BlockValidator {
+func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain) *BlockValidator {
 	validator := &BlockValidator{
 		config: config,
-		engine: engine,
 		bc:     blockchain,
 	}
 	return validator
@@ -59,7 +59,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	// Header validity is known at this point. Here we verify that uncles, transactions
 	// and withdrawals given in the block body match the header.
 	header := block.Header()
-	if err := v.engine.VerifyUncles(v.bc, block); err != nil {
+	if err := v.bc.engine.VerifyUncles(v.bc, block); err != nil {
 		return err
 	}
 	if hash := types.CalcUncleHash(block.Uncles()); hash != header.UncleHash {
@@ -121,7 +121,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 
 // ValidateState validates the various changes that happen after a state transition,
 // such as amount of used gas, the receipt roots and the state root itself.
-func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateDB, receipts types.Receipts, usedGas uint64) error {
+func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateDB, receipts types.Receipts, usedGas uint64, stateless bool) error {
 	header := block.Header()
 	if block.GasUsed() != usedGas {
 		return fmt.Errorf("invalid gas used (remote: %d local: %d)", block.GasUsed(), usedGas)
@@ -132,6 +132,11 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
 	if rbloom != header.Bloom {
 		return fmt.Errorf("invalid bloom (remote: %x  local: %x)", header.Bloom, rbloom)
 	}
+	// In stateless mode, return early because the receipt and state root are not
+	// provided through the witness, rather the cross validator needs to return it.
+	if stateless {
+		return nil
+	}
 	// The receipt Trie's root (R = (Tr [[H1, R1], ... [Hn, Rn]]))
 	receiptSha := types.DeriveSha(receipts, trie.NewStackTrie(nil))
 	if receiptSha != header.ReceiptHash {
@@ -145,6 +150,28 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
 	return nil
 }
 
+// ValidateWitness cross validates a block execution with stateless remote clients.
+//
+// Normally we'd distribute the block witness to remote cross validators, wait
+// for them to respond and then merge the results. For now, however, it's only
+// Geth, so do an internal stateless run.
+func (v *BlockValidator) ValidateWitness(witness *stateless.Witness, receiptRoot common.Hash, stateRoot common.Hash) error {
+	// Run the cross client stateless execution
+	// TODO(karalabe): Self-stateless for now, swap with other clients
+	crossReceiptRoot, crossStateRoot, err := ExecuteStateless(v.config, witness)
+	if err != nil {
+		return fmt.Errorf("stateless execution failed: %v", err)
+	}
+	// Stateless cross execution suceeeded, validate the withheld computed fields
+	if crossReceiptRoot != receiptRoot {
+		return fmt.Errorf("cross validator receipt root mismatch (cross: %x local: %x)", crossReceiptRoot, receiptRoot)
+	}
+	if crossStateRoot != stateRoot {
+		return fmt.Errorf("cross validator state root mismatch (cross: %x local: %x)", crossStateRoot, stateRoot)
+	}
+	return nil
+}
+
 // CalcGasLimit computes the gas limit of the next block after parent. It aims
 // to keep the baseline gas close to the provided target, and increase it towards
 // the target if the baseline gas is lower.

core/blockchain.go

@@ -37,6 +37,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
+	"github.com/ethereum/go-ethereum/core/stateless"
 	"github.com/ethereum/go-ethereum/core/tracing"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
@@ -302,18 +303,18 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 		vmConfig:      vmConfig,
 		logger:        vmConfig.Tracer,
 	}
-	bc.flushInterval.Store(int64(cacheConfig.TrieTimeLimit))
-	bc.forker = NewForkChoice(bc, shouldPreserve)
-	bc.stateCache = state.NewDatabaseWithNodeDB(bc.db, bc.triedb)
-	bc.validator = NewBlockValidator(chainConfig, bc, engine)
-	bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine)
-	bc.processor = NewStateProcessor(chainConfig, bc, engine)
-
 	var err error
 	bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.insertStopped)
 	if err != nil {
 		return nil, err
 	}
+	bc.flushInterval.Store(int64(cacheConfig.TrieTimeLimit))
+	bc.forker = NewForkChoice(bc, shouldPreserve)
+	bc.stateCache = state.NewDatabaseWithNodeDB(bc.db, bc.triedb)
+	bc.validator = NewBlockValidator(chainConfig, bc)
+	bc.prefetcher = newStatePrefetcher(chainConfig, bc.hc)
+	bc.processor = NewStateProcessor(chainConfig, bc.hc)
+
 	bc.genesisBlock = bc.GetBlockByNumber(0)
 	if bc.genesisBlock == nil {
 		return nil, ErrNoGenesis
@@ -1809,7 +1810,14 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 		// while processing transactions. Before Byzantium the prefetcher is mostly
 		// useless due to the intermediate root hashing after each transaction.
 		if bc.chainConfig.IsByzantium(block.Number()) {
-			statedb.StartPrefetcher("chain", !bc.vmConfig.EnableWitnessCollection)
+			var witness *stateless.Witness
+			if bc.vmConfig.EnableWitnessCollection {
+				witness, err = stateless.NewWitness(bc, block)
+				if err != nil {
+					return it.index, err
+				}
+			}
+			statedb.StartPrefetcher("chain", witness)
 		}
 		activeState = statedb
 
@@ -1924,11 +1932,18 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s
 	ptime := time.Since(pstart)
 
 	vstart := time.Now()
-	if err := bc.validator.ValidateState(block, statedb, receipts, usedGas); err != nil {
+	if err := bc.validator.ValidateState(block, statedb, receipts, usedGas, false); err != nil {
 		bc.reportBlock(block, receipts, err)
 		return nil, err
 	}
 	vtime := time.Since(vstart)
+
+	if witness := statedb.Witness(); witness != nil {
+		if err = bc.validator.ValidateWitness(witness, block.ReceiptHash(), block.Root()); err != nil {
+			bc.reportBlock(block, receipts, err)
+			return nil, fmt.Errorf("cross verification failed: %v", err)
+		}
+	}
 	proctime := time.Since(start) // processing + validation
 
 	// Update the metrics touched during block processing and validation

core/blockchain_test.go

@@ -168,8 +168,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
 			blockchain.reportBlock(block, receipts, err)
 			return err
 		}
-		err = blockchain.validator.ValidateState(block, statedb, receipts, usedGas)
+		if err = blockchain.validator.ValidateState(block, statedb, receipts, usedGas, false); err != nil {
-		if err != nil {
 			blockchain.reportBlock(block, receipts, err)
 			return err
 		}

core/state/database.go

@@ -125,6 +125,10 @@ type Trie interface {
 	// be created with new root and updated trie database for following usage
 	Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet)
 
+	// Witness returns a set containing all trie nodes that have been accessed.
+	// The returned map could be nil if the witness is empty.
+	Witness() map[string]struct{}
+
 	// NodeIterator returns an iterator that returns nodes of the trie. Iteration
 	// starts at the key after the given start key. And error will be returned
 	// if fails to create node iterator.

core/state/state_object.go

@@ -323,11 +323,17 @@ func (s *stateObject) finalise() {
 //
 // It assumes all the dirty storage slots have been finalized before.
 func (s *stateObject) updateTrie() (Trie, error) {
-	// Short circuit if nothing changed, don't bother with hashing anything
+	// Short circuit if nothing was accessed, don't trigger a prefetcher warning
 	if len(s.uncommittedStorage) == 0 {
-		return s.trie, nil
+		// Nothing was written, so we could stop early. Unless we have both reads
+		// and witness collection enabled, in which case we need to fetch the trie.
+		if s.db.witness == nil || len(s.originStorage) == 0 {
+			return s.trie, nil
+		}
 	}
-	// Retrieve a pretecher populated trie, or fall back to the database
+	// Retrieve a pretecher populated trie, or fall back to the database. This will
+	// block until all prefetch tasks are done, which are needed for witnesses even
+	// for unmodified state objects.
 	tr := s.getPrefetchedTrie()
 	if tr != nil {
 		// Prefetcher returned a live trie, swap it out for the current one
@@ -341,6 +347,10 @@ func (s *stateObject) updateTrie() (Trie, error) {
 			return nil, err
 		}
 	}
+	// Short circuit if nothing changed, don't bother with hashing anything
+	if len(s.uncommittedStorage) == 0 {
+		return s.trie, nil
+	}
 	// Perform trie updates before deletions. This prevents resolution of unnecessary trie nodes
 	// in circumstances similar to the following:
 	//

core/state/statedb.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
+	"github.com/ethereum/go-ethereum/core/stateless"
 	"github.com/ethereum/go-ethereum/core/tracing"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -105,7 +106,7 @@ type StateDB struct {
 	// resurrection. The account value is tracked as the original value
 	// before the transition. This map is populated at the transaction
 	// boundaries.
-	stateObjectsDestruct map[common.Address]*types.StateAccount
+	stateObjectsDestruct map[common.Address]*stateObject
 
 	// This map tracks the account mutations that occurred during the
 	// transition. Uncommitted mutations belonging to the same account
@@ -146,6 +147,9 @@ type StateDB struct {
 	validRevisions []revision
 	nextRevisionId int
 
+	// State witness if cross validation is needed
+	witness *stateless.Witness
+
 	// Measurements gathered during execution for debugging purposes
 	AccountReads         time.Duration
 	AccountHashes        time.Duration
@@ -177,7 +181,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error)
 		originalRoot:         root,
 		snaps:                snaps,
 		stateObjects:         make(map[common.Address]*stateObject),
-		stateObjectsDestruct: make(map[common.Address]*types.StateAccount),
+		stateObjectsDestruct: make(map[common.Address]*stateObject),
 		mutations:            make(map[common.Address]*mutation),
 		logs:                 make(map[common.Hash][]*types.Log),
 		preimages:            make(map[common.Hash][]byte),
@@ -200,14 +204,19 @@ func (s *StateDB) SetLogger(l *tracing.Hooks) {
 // StartPrefetcher initializes a new trie prefetcher to pull in nodes from the
 // state trie concurrently while the state is mutated so that when we reach the
 // commit phase, most of the needed data is already hot.
-func (s *StateDB) StartPrefetcher(namespace string, noreads bool) {
+func (s *StateDB) StartPrefetcher(namespace string, witness *stateless.Witness) {
+	// Terminate any previously running prefetcher
 	if s.prefetcher != nil {
 		s.prefetcher.terminate(false)
 		s.prefetcher.report()
 		s.prefetcher = nil
 	}
+	// Enable witness collection if requested
+	s.witness = witness
+
+	// If snapshots are enabled, start prefethers explicitly
 	if s.snap != nil {
-		s.prefetcher = newTriePrefetcher(s.db, s.originalRoot, namespace, noreads)
+		s.prefetcher = newTriePrefetcher(s.db, s.originalRoot, namespace, witness == nil)
 
 		// With the switch to the Proof-of-Stake consensus algorithm, block production
 		// rewards are now handled at the consensus layer. Consequently, a block may
@@ -582,7 +591,6 @@ func (s *StateDB) getStateObject(addr common.Address) *stateObject {
 		start := time.Now()
 		acc, err := s.snap.Account(crypto.HashData(s.hasher, addr.Bytes()))
 		s.SnapshotAccountReads += time.Since(start)
-
 		if err == nil {
 			if acc == nil {
 				return nil
@@ -683,7 +691,7 @@ func (s *StateDB) Copy() *StateDB {
 		hasher:               crypto.NewKeccakState(),
 		originalRoot:         s.originalRoot,
 		stateObjects:         make(map[common.Address]*stateObject, len(s.stateObjects)),
-		stateObjectsDestruct: maps.Clone(s.stateObjectsDestruct),
+		stateObjectsDestruct: make(map[common.Address]*stateObject, len(s.stateObjectsDestruct)),
 		mutations:            make(map[common.Address]*mutation, len(s.mutations)),
 		dbErr:                s.dbErr,
 		refund:               s.refund,
@@ -703,10 +711,17 @@ func (s *StateDB) Copy() *StateDB {
 		snaps: s.snaps,
 		snap:  s.snap,
 	}
+	if s.witness != nil {
+		state.witness = s.witness.Copy()
+	}
 	// Deep copy cached state objects.
 	for addr, obj := range s.stateObjects {
 		state.stateObjects[addr] = obj.deepCopy(state)
 	}
+	// Deep copy destructed state objects.
+	for addr, obj := range s.stateObjectsDestruct {
+		state.stateObjectsDestruct[addr] = obj.deepCopy(state)
+	}
 	// Deep copy the object state markers.
 	for addr, op := range s.mutations {
 		state.mutations[addr] = op.copy()
@@ -788,7 +803,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) {
 			// set indefinitely). Note only the first occurred self-destruct
 			// event is tracked.
 			if _, ok := s.stateObjectsDestruct[obj.address]; !ok {
-				s.stateObjectsDestruct[obj.address] = obj.origin
+				s.stateObjectsDestruct[obj.address] = obj
 			}
 		} else {
 			obj.finalise()
@@ -846,9 +861,46 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
 		obj := s.stateObjects[addr] // closure for the task runner below
 		workers.Go(func() error {
 			obj.updateRoot()
+
+			// If witness building is enabled and the state object has a trie,
+			// gather the witnesses for its specific storage trie
+			if s.witness != nil && obj.trie != nil {
+				s.witness.AddState(obj.trie.Witness())
+			}
 			return nil
 		})
 	}
+	// If witness building is enabled, gather all the read-only accesses
+	if s.witness != nil {
+		// Pull in anything that has been accessed before destruction
+		for _, obj := range s.stateObjectsDestruct {
+			// Skip any objects that haven't touched their storage
+			if len(obj.originStorage) == 0 {
+				continue
+			}
+			if trie := obj.getPrefetchedTrie(); trie != nil {
+				s.witness.AddState(trie.Witness())
+			} else if obj.trie != nil {
+				s.witness.AddState(obj.trie.Witness())
+			}
+		}
+		// Pull in only-read and non-destructed trie witnesses
+		for _, obj := range s.stateObjects {
+			// Skip any objects that have been updated
+			if _, ok := s.mutations[obj.address]; ok {
+				continue
+			}
+			// Skip any objects that haven't touched their storage
+			if len(obj.originStorage) == 0 {
+				continue
+			}
+			if trie := obj.getPrefetchedTrie(); trie != nil {
+				s.witness.AddState(trie.Witness())
+			} else if obj.trie != nil {
+				s.witness.AddState(obj.trie.Witness())
+			}
+		}
+	}
 	workers.Wait()
 	s.StorageUpdates += time.Since(start)
 
@@ -904,7 +956,13 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
 	// Track the amount of time wasted on hashing the account trie
 	defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now())
 
-	return s.trie.Hash()
+	hash := s.trie.Hash()
+
+	// If witness building is enabled, gather the account trie witness
+	if s.witness != nil {
+		s.witness.AddState(s.trie.Witness())
+	}
+	return hash
 }
 
 // SetTxContext sets the current transaction hash and index which are
@@ -1060,7 +1118,9 @@ func (s *StateDB) handleDestruction() (map[common.Hash]*accountDelete, []*trieno
 		buf     = crypto.NewKeccakState()
 		deletes = make(map[common.Hash]*accountDelete)
 	)
-	for addr, prev := range s.stateObjectsDestruct {
+	for addr, prevObj := range s.stateObjectsDestruct {
+		prev := prevObj.origin
+
 		// The account was non-existent, and it's marked as destructed in the scope
 		// of block. It can be either case (a) or (b) and will be interpreted as
 		// null->null state transition.
@@ -1239,7 +1299,7 @@ func (s *StateDB) commit(deleteEmptyObjects bool) (*stateUpdate, error) {
 
 	// Clear all internal flags and update state root at the end.
 	s.mutations = make(map[common.Address]*mutation)
-	s.stateObjectsDestruct = make(map[common.Address]*types.StateAccount)
+	s.stateObjectsDestruct = make(map[common.Address]*stateObject)
 
 	origin := s.originalRoot
 	s.originalRoot = root
@@ -1412,3 +1472,8 @@ func (s *StateDB) markUpdate(addr common.Address) {
 func (s *StateDB) PointCache() *utils.PointCache {
 	return s.db.PointCache()
 }
+
+// Witness retrieves the current state witness being collected.
+func (s *StateDB) Witness() *stateless.Witness {
+	return s.witness
+}

core/state_prefetcher.go

@@ -19,7 +19,6 @@ package core
 import (
 	"sync/atomic"
 
-	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
@@ -31,16 +30,14 @@ import (
 // data from disk before the main block processor start executing.
 type statePrefetcher struct {
 	config *params.ChainConfig // Chain configuration options
-	bc     *BlockChain         // Canonical block chain
+	chain  *HeaderChain        // Canonical block chain
-	engine consensus.Engine    // Consensus engine used for block rewards
 }
 
 // newStatePrefetcher initialises a new statePrefetcher.
-func newStatePrefetcher(config *params.ChainConfig, bc *BlockChain, engine consensus.Engine) *statePrefetcher {
+func newStatePrefetcher(config *params.ChainConfig, chain *HeaderChain) *statePrefetcher {
 	return &statePrefetcher{
 		config: config,
-		bc:     bc,
+		chain:  chain,
-		engine: engine,
 	}
 }
 
@@ -51,7 +48,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c
 	var (
 		header       = block.Header()
 		gaspool      = new(GasPool).AddGas(block.GasLimit())
-		blockContext = NewEVMBlockContext(header, p.bc, nil)
+		blockContext = NewEVMBlockContext(header, p.chain, nil)
 		evm          = vm.NewEVM(blockContext, vm.TxContext{}, statedb, p.config, cfg)
 		signer       = types.MakeSigner(p.config, header.Number, header.Time)
 	)

core/state_processor.go

@@ -22,7 +22,6 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/misc"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -37,16 +36,14 @@ import (
 // StateProcessor implements Processor.
 type StateProcessor struct {
 	config *params.ChainConfig // Chain configuration options
-	bc     *BlockChain         // Canonical block chain
+	chain  *HeaderChain        // Canonical header chain
-	engine consensus.Engine    // Consensus engine used for block rewards
 }
 
 // NewStateProcessor initialises a new StateProcessor.
-func NewStateProcessor(config *params.ChainConfig, bc *BlockChain, engine consensus.Engine) *StateProcessor {
+func NewStateProcessor(config *params.ChainConfig, chain *HeaderChain) *StateProcessor {
 	return &StateProcessor{
 		config: config,
-		bc:     bc,
+		chain:  chain,
-		engine: engine,
 	}
 }
 
@@ -73,10 +70,11 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
 		misc.ApplyDAOHardFork(statedb)
 	}
 	var (
-		context = NewEVMBlockContext(header, p.bc, nil)
+		context vm.BlockContext
-		vmenv   = vm.NewEVM(context, vm.TxContext{}, statedb, p.config, cfg)
 		signer  = types.MakeSigner(p.config, header.Number, header.Time)
 	)
+	context = NewEVMBlockContext(header, p.chain, nil)
+	vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, p.config, cfg)
 	if beaconRoot := block.BeaconRoot(); beaconRoot != nil {
 		ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb)
 	}
@@ -101,7 +99,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg
 		return nil, nil, 0, errors.New("withdrawals before shanghai")
 	}
 	// Finalize the block, applying any consensus engine specific extras (e.g. block rewards)
-	p.engine.Finalize(p.bc, header, statedb, block.Body())
+	p.chain.engine.Finalize(p.chain, header, statedb, block.Body())
 
 	return receipts, allLogs, *usedGas, nil
 }

core/stateless.go

@@ -0,0 +1,73 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/lru"
+	"github.com/ethereum/go-ethereum/consensus/beacon"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/stateless"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/triedb"
+)
+
+// ExecuteStateless runs a stateless execution based on a witness, verifies
+// everything it can locally and returns the two computed fields that need the
+// other side to explicitly check.
+//
+// This method is a bit of a sore thumb here, but:
+//   - It cannot be placed in core/stateless, because state.New prodces a circular dep
+//   - It cannot be placed outside of core, because it needs to construct a dud headerchain
+//
+// TODO(karalabe): Would be nice to resolve both issues above somehow and move it.
+func ExecuteStateless(config *params.ChainConfig, witness *stateless.Witness) (common.Hash, common.Hash, error) {
+	// Create and populate the state database to serve as the stateless backend
+	memdb := witness.MakeHashDB()
+
+	db, err := state.New(witness.Root(), state.NewDatabaseWithConfig(memdb, triedb.HashDefaults), nil)
+	if err != nil {
+		return common.Hash{}, common.Hash{}, err
+	}
+	// Create a blockchain that is idle, but can be used to access headers through
+	chain := &HeaderChain{
+		config:      config,
+		chainDb:     memdb,
+		headerCache: lru.NewCache[common.Hash, *types.Header](256),
+		engine:      beacon.New(ethash.NewFaker()),
+	}
+	processor := NewStateProcessor(config, chain)
+	validator := NewBlockValidator(config, nil) // No chain, we only validate the state, not the block
+
+	// Run the stateless blocks processing and self-validate certain fields
+	receipts, _, usedGas, err := processor.Process(witness.Block, db, vm.Config{})
+	if err != nil {
+		return common.Hash{}, common.Hash{}, err
+	}
+	if err = validator.ValidateState(witness.Block, db, receipts, usedGas, true); err != nil {
+		return common.Hash{}, common.Hash{}, err
+	}
+	// Almost everything validated, but receipt and state root needs to be returned
+	receiptRoot := types.DeriveSha(receipts, trie.NewStackTrie(nil))
+	stateRoot := db.IntermediateRoot(config.IsEIP158(witness.Block.Number()))
+
+	return receiptRoot, stateRoot, nil
+}

core/stateless/database.go

@@ -0,0 +1,60 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package stateless
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+// MakeHashDB imports tries, codes and block hashes from a witness into a new
+// hash-based memory db. We could eventually rewrite this into a pathdb, but
+// simple is better for now.
+func (w *Witness) MakeHashDB() ethdb.Database {
+	var (
+		memdb  = rawdb.NewMemoryDatabase()
+		hasher = crypto.NewKeccakState()
+		hash   = make([]byte, 32)
+	)
+	// Inject all the "block hashes" (i.e. headers) into the ephemeral database
+	for _, header := range w.Headers {
+		rawdb.WriteHeader(memdb, header)
+	}
+	// Inject all the bytecodes into the ephemeral database
+	for code := range w.Codes {
+		blob := []byte(code)
+
+		hasher.Reset()
+		hasher.Write(blob)
+		hasher.Read(hash)
+
+		rawdb.WriteCode(memdb, common.BytesToHash(hash), blob)
+	}
+	// Inject all the MPT trie nodes into the ephemeral database
+	for node := range w.State {
+		blob := []byte(node)
+
+		hasher.Reset()
+		hasher.Write(blob)
+		hasher.Read(hash)
+
+		rawdb.WriteLegacyTrieNode(memdb, common.BytesToHash(hash), blob)
+	}
+	return memdb
+}

core/stateless/encoding.go

@@ -0,0 +1,129 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package stateless
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"slices"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+//go:generate go run github.com/fjl/gencodec -type extWitness -field-override extWitnessMarshalling -out gen_encoding_json.go
+
+// toExtWitness converts our internal witness representation to the consensus one.
+func (w *Witness) toExtWitness() *extWitness {
+	ext := &extWitness{
+		Block:   w.Block,
+		Headers: w.Headers,
+	}
+	ext.Codes = make([][]byte, 0, len(w.Codes))
+	for code := range w.Codes {
+		ext.Codes = append(ext.Codes, []byte(code))
+	}
+	slices.SortFunc(ext.Codes, bytes.Compare)
+
+	ext.State = make([][]byte, 0, len(w.State))
+	for node := range w.State {
+		ext.State = append(ext.State, []byte(node))
+	}
+	slices.SortFunc(ext.State, bytes.Compare)
+	return ext
+}
+
+// fromExtWitness converts the consensus witness format into our internal one.
+func (w *Witness) fromExtWitness(ext *extWitness) error {
+	w.Block, w.Headers = ext.Block, ext.Headers
+
+	w.Codes = make(map[string]struct{}, len(ext.Codes))
+	for _, code := range ext.Codes {
+		w.Codes[string(code)] = struct{}{}
+	}
+	w.State = make(map[string]struct{}, len(ext.State))
+	for _, node := range ext.State {
+		w.State[string(node)] = struct{}{}
+	}
+	return w.sanitize()
+}
+
+// MarshalJSON marshals a witness as JSON.
+func (w *Witness) MarshalJSON() ([]byte, error) {
+	return w.toExtWitness().MarshalJSON()
+}
+
+// EncodeRLP serializes a witness as RLP.
+func (w *Witness) EncodeRLP(wr io.Writer) error {
+	return rlp.Encode(wr, w.toExtWitness())
+}
+
+// UnmarshalJSON unmarshals from JSON.
+func (w *Witness) UnmarshalJSON(input []byte) error {
+	var ext extWitness
+	if err := ext.UnmarshalJSON(input); err != nil {
+		return err
+	}
+	return w.fromExtWitness(&ext)
+}
+
+// DecodeRLP decodes a witness from RLP.
+func (w *Witness) DecodeRLP(s *rlp.Stream) error {
+	var ext extWitness
+	if err := s.Decode(&ext); err != nil {
+		return err
+	}
+	return w.fromExtWitness(&ext)
+}
+
+// sanitize checks for some mandatory fields in the witness after decoding so
+// the rest of the code can assume invariants and doesn't have to deal with
+// corrupted data.
+func (w *Witness) sanitize() error {
+	// Verify that the "parent" header (i.e. index 0) is available, and is the
+	// true parent of the block-to-be executed, since we use that to link the
+	// current block to the pre-state.
+	if len(w.Headers) == 0 {
+		return errors.New("parent header (for pre-root hash) missing")
+	}
+	for i, header := range w.Headers {
+		if header == nil {
+			return fmt.Errorf("witness header nil at position %d", i)
+		}
+	}
+	if w.Headers[0].Hash() != w.Block.ParentHash() {
+		return fmt.Errorf("parent hash different: witness %v, block parent %v", w.Headers[0].Hash(), w.Block.ParentHash())
+	}
+	return nil
+}
+
+// extWitness is a witness RLP encoding for transferring across clients.
+type extWitness struct {
+	Block   *types.Block    `json:"block"       gencodec:"required"`
+	Headers []*types.Header `json:"headers"       gencodec:"required"`
+	Codes   [][]byte        `json:"codes"`
+	State   [][]byte        `json:"state"`
+}
+
+// extWitnessMarshalling defines the hex marshalling types for a witness.
+type extWitnessMarshalling struct {
+	Codes []hexutil.Bytes
+	State []hexutil.Bytes
+}

core/stateless/gen_encoding_json.go

@@ -0,0 +1,74 @@
+// Code generated by github.com/fjl/gencodec. DO NOT EDIT.
+
+package stateless
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+var _ = (*extWitnessMarshalling)(nil)
+
+// MarshalJSON marshals as JSON.
+func (e extWitness) MarshalJSON() ([]byte, error) {
+	type extWitness struct {
+		Block   *types.Block    `json:"block"       gencodec:"required"`
+		Headers []*types.Header `json:"headers"       gencodec:"required"`
+		Codes   []hexutil.Bytes `json:"codes"`
+		State   []hexutil.Bytes `json:"state"`
+	}
+	var enc extWitness
+	enc.Block = e.Block
+	enc.Headers = e.Headers
+	if e.Codes != nil {
+		enc.Codes = make([]hexutil.Bytes, len(e.Codes))
+		for k, v := range e.Codes {
+			enc.Codes[k] = v
+		}
+	}
+	if e.State != nil {
+		enc.State = make([]hexutil.Bytes, len(e.State))
+		for k, v := range e.State {
+			enc.State[k] = v
+		}
+	}
+	return json.Marshal(&enc)
+}
+
+// UnmarshalJSON unmarshals from JSON.
+func (e *extWitness) UnmarshalJSON(input []byte) error {
+	type extWitness struct {
+		Block   *types.Block    `json:"block"       gencodec:"required"`
+		Headers []*types.Header `json:"headers"       gencodec:"required"`
+		Codes   []hexutil.Bytes `json:"codes"`
+		State   []hexutil.Bytes `json:"state"`
+	}
+	var dec extWitness
+	if err := json.Unmarshal(input, &dec); err != nil {
+		return err
+	}
+	if dec.Block == nil {
+		return errors.New("missing required field 'block' for extWitness")
+	}
+	e.Block = dec.Block
+	if dec.Headers == nil {
+		return errors.New("missing required field 'headers' for extWitness")
+	}
+	e.Headers = dec.Headers
+	if dec.Codes != nil {
+		e.Codes = make([][]byte, len(dec.Codes))
+		for k, v := range dec.Codes {
+			e.Codes[k] = v
+		}
+	}
+	if dec.State != nil {
+		e.State = make([][]byte, len(dec.State))
+		for k, v := range dec.State {
+			e.State[k] = v
+		}
+	}
+	return nil
+}

core/stateless/witness.go

@@ -0,0 +1,159 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package stateless
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"maps"
+	"slices"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// HeaderReader is an interface to pull in headers in place of block hashes for
+// the witness.
+type HeaderReader interface {
+	// GetHeader retrieves a block header from the database by hash and number,
+	GetHeader(hash common.Hash, number uint64) *types.Header
+}
+
+// Witness encompasses a block, state and any other chain data required to apply
+// a set of transactions and derive a post state/receipt root.
+type Witness struct {
+	Block   *types.Block        // Current block with rootHash and receiptHash zeroed out
+	Headers []*types.Header     // Past headers in reverse order (0=parent, 1=parent's-parent, etc). First *must* be set.
+	Codes   map[string]struct{} // Set of bytecodes ran or accessed
+	State   map[string]struct{} // Set of MPT state trie nodes (account and storage together)
+
+	chain HeaderReader // Chain reader to convert block hash ops to header proofs
+	lock  sync.Mutex   // Lock to allow concurrent state insertions
+}
+
+// NewWitness creates an empty witness ready for population.
+func NewWitness(chain HeaderReader, block *types.Block) (*Witness, error) {
+	// Zero out the result fields to avoid accidentally sending them to the verifier
+	header := block.Header()
+	header.Root = common.Hash{}
+	header.ReceiptHash = common.Hash{}
+
+	// Retrieve the parent header, which will *always* be included to act as a
+	// trustless pre-root hash container
+	parent := chain.GetHeader(block.ParentHash(), block.NumberU64()-1)
+	if parent == nil {
+		return nil, errors.New("failed to retrieve parent header")
+	}
+	// Create the wtness with a reconstructed gutted out block
+	return &Witness{
+		Block:   types.NewBlockWithHeader(header).WithBody(*block.Body()),
+		Codes:   make(map[string]struct{}),
+		State:   make(map[string]struct{}),
+		Headers: []*types.Header{parent},
+		chain:   chain,
+	}, nil
+}
+
+// AddBlockHash adds a "blockhash" to the witness with the designated offset from
+// chain head. Under the hood, this method actually pulls in enough headers from
+// the chain to cover the block being added.
+func (w *Witness) AddBlockHash(number uint64) {
+	// Keep pulling in headers until this hash is populated
+	for int(w.Block.NumberU64()-number) > len(w.Headers) {
+		tail := w.Block.Header()
+		if len(w.Headers) > 0 {
+			tail = w.Headers[len(w.Headers)-1]
+		}
+		w.Headers = append(w.Headers, w.chain.GetHeader(tail.ParentHash, tail.Number.Uint64()-1))
+	}
+}
+
+// AddCode adds a bytecode blob to the witness.
+func (w *Witness) AddCode(code []byte) {
+	if len(code) == 0 {
+		return
+	}
+	w.Codes[string(code)] = struct{}{}
+}
+
+// AddState inserts a batch of MPT trie nodes into the witness.
+func (w *Witness) AddState(nodes map[string]struct{}) {
+	if len(nodes) == 0 {
+		return
+	}
+	w.lock.Lock()
+	defer w.lock.Unlock()
+
+	for node := range nodes {
+		w.State[node] = struct{}{}
+	}
+}
+
+// Copy deep-copies the witness object.  Witness.Block isn't deep-copied as it
+// is never mutated by Witness
+func (w *Witness) Copy() *Witness {
+	return &Witness{
+		Block:   w.Block,
+		Headers: slices.Clone(w.Headers),
+		Codes:   maps.Clone(w.Codes),
+		State:   maps.Clone(w.State),
+	}
+}
+
+// String prints a human-readable summary containing the total size of the
+// witness and the sizes of the underlying components
+func (w *Witness) String() string {
+	blob, _ := rlp.EncodeToBytes(w)
+	bytesTotal := len(blob)
+
+	blob, _ = rlp.EncodeToBytes(w.Block)
+	bytesBlock := len(blob)
+
+	bytesHeaders := 0
+	for _, header := range w.Headers {
+		blob, _ = rlp.EncodeToBytes(header)
+		bytesHeaders += len(blob)
+	}
+	bytesCodes := 0
+	for code := range w.Codes {
+		bytesCodes += len(code)
+	}
+	bytesState := 0
+	for node := range w.State {
+		bytesState += len(node)
+	}
+	buf := new(bytes.Buffer)
+
+	fmt.Fprintf(buf, "Witness #%d: %v\n", w.Block.Number(), common.StorageSize(bytesTotal))
+	fmt.Fprintf(buf, "     block (%4d txs):  %10v\n", len(w.Block.Transactions()), common.StorageSize(bytesBlock))
+	fmt.Fprintf(buf, "%4d headers:      %10v\n", len(w.Headers), common.StorageSize(bytesHeaders))
+	fmt.Fprintf(buf, "%4d trie nodes:   %10v\n", len(w.State), common.StorageSize(bytesState))
+	fmt.Fprintf(buf, "%4d codes:        %10v\n", len(w.Codes), common.StorageSize(bytesCodes))
+
+	return buf.String()
+}
+
+// Root returns the pre-state root from the first header.
+//
+// Note, this method will panic in case of a bad witness (but RLP decoding will
+// sanitize it and fail before that).
+func (w *Witness) Root() common.Hash {
+	return w.Headers[0].Root
+}

core/txpool/blobpool/blobpool.go

@@ -407,7 +407,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres
 	if p.head.ExcessBlobGas != nil {
 		blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(*p.head.ExcessBlobGas))
 	}
-	p.evict = newPriceHeap(basefee, blobfee, &p.index)
+	p.evict = newPriceHeap(basefee, blobfee, p.index)
 
 	// Pool initialized, attach the blob limbo to it to track blobs included
 	// recently but not yet finalized

core/txpool/blobpool/evictheap.go

@@ -17,13 +17,13 @@
 package blobpool
 
 import (
-	"bytes"
 	"container/heap"
 	"math"
-	"sort"
+	"slices"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/holiman/uint256"
+	"golang.org/x/exp/maps"
 )
 
 // evictHeap is a helper data structure to keep track of the cheapest bottleneck
@@ -35,7 +35,7 @@ import (
 // The goal of the heap is to decide which account has the worst bottleneck to
 // evict transactions from.
 type evictHeap struct {
-	metas *map[common.Address][]*blobTxMeta // Pointer to the blob pool's index for price retrievals
+	metas map[common.Address][]*blobTxMeta // Pointer to the blob pool's index for price retrievals
 
 	basefeeJumps float64 // Pre-calculated absolute dynamic fee jumps for the base fee
 	blobfeeJumps float64 // Pre-calculated absolute dynamic fee jumps for the blob fee
@@ -46,23 +46,18 @@ type evictHeap struct {
 
 // newPriceHeap creates a new heap of cheapest accounts in the blob pool to evict
 // from in case of over saturation.
-func newPriceHeap(basefee *uint256.Int, blobfee *uint256.Int, index *map[common.Address][]*blobTxMeta) *evictHeap {
+func newPriceHeap(basefee *uint256.Int, blobfee *uint256.Int, index map[common.Address][]*blobTxMeta) *evictHeap {
 	heap := &evictHeap{
 		metas: index,
-		index: make(map[common.Address]int),
+		index: make(map[common.Address]int, len(index)),
 	}
 	// Populate the heap in account sort order. Not really needed in practice,
 	// but it makes the heap initialization deterministic and less annoying to
 	// test in unit tests.
-	addrs := make([]common.Address, 0, len(*index))
+	heap.addrs = maps.Keys(index)
-	for addr := range *index {
+	slices.SortFunc(heap.addrs, common.Address.Cmp)
-		addrs = append(addrs, addr)
+	for i, addr := range heap.addrs {
-	}
+		heap.index[addr] = i
-	sort.Slice(addrs, func(i, j int) bool { return bytes.Compare(addrs[i][:], addrs[j][:]) < 0 })
-
-	for _, addr := range addrs {
-		heap.index[addr] = len(heap.addrs)
-		heap.addrs = append(heap.addrs, addr)
 	}
 	heap.reinit(basefee, blobfee, true)
 	return heap
@@ -94,8 +89,8 @@ func (h *evictHeap) Len() int {
 // Less implements sort.Interface as part of heap.Interface, returning which of
 // the two requested accounts has a cheaper bottleneck.
 func (h *evictHeap) Less(i, j int) bool {
-	txsI := (*(h.metas))[h.addrs[i]]
+	txsI := h.metas[h.addrs[i]]
-	txsJ := (*(h.metas))[h.addrs[j]]
+	txsJ := h.metas[h.addrs[j]]
 
 	lastI := txsI[len(txsI)-1]
 	lastJ := txsJ[len(txsJ)-1]

core/txpool/blobpool/evictheap_test.go

@@ -37,17 +37,17 @@ func verifyHeapInternals(t *testing.T, evict *evictHeap) {
 	seen := make(map[common.Address]struct{})
 	for i, addr := range evict.addrs {
 		seen[addr] = struct{}{}
-		if _, ok := (*evict.metas)[addr]; !ok {
+		if _, ok := evict.metas[addr]; !ok {
 			t.Errorf("heap contains unexpected address at slot %d: %v", i, addr)
 		}
 	}
-	for addr := range *evict.metas {
+	for addr := range evict.metas {
 		if _, ok := seen[addr]; !ok {
 			t.Errorf("heap is missing required address %v", addr)
 		}
 	}
-	if len(evict.addrs) != len(*evict.metas) {
+	if len(evict.addrs) != len(evict.metas) {
-		t.Errorf("heap size %d mismatches metadata size %d", len(evict.addrs), len(*evict.metas))
+		t.Errorf("heap size %d mismatches metadata size %d", len(evict.addrs), len(evict.metas))
 	}
 	// Ensure that all accounts are present in the heap order index and no extras
 	have := make([]common.Address, len(evict.index))
@@ -159,7 +159,7 @@ func TestPriceHeapSorting(t *testing.T) {
 			}}
 		}
 		// Create a price heap and check the pop order
-		priceheap := newPriceHeap(uint256.NewInt(tt.basefee), uint256.NewInt(tt.blobfee), &index)
+		priceheap := newPriceHeap(uint256.NewInt(tt.basefee), uint256.NewInt(tt.blobfee), index)
 		verifyHeapInternals(t, priceheap)
 
 		for j := 0; j < len(tt.order); j++ {
@@ -218,7 +218,7 @@ func benchmarkPriceHeapReinit(b *testing.B, datacap uint64) {
 		}}
 	}
 	// Create a price heap and reinit it over and over
-	heap := newPriceHeap(uint256.NewInt(rand.Uint64()), uint256.NewInt(rand.Uint64()), &index)
+	heap := newPriceHeap(uint256.NewInt(rand.Uint64()), uint256.NewInt(rand.Uint64()), index)
 
 	basefees := make([]*uint256.Int, b.N)
 	blobfees := make([]*uint256.Int, b.N)
@@ -278,7 +278,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) {
 		}}
 	}
 	// Create a price heap and overflow it over and over
-	evict := newPriceHeap(uint256.NewInt(rand.Uint64()), uint256.NewInt(rand.Uint64()), &index)
+	evict := newPriceHeap(uint256.NewInt(rand.Uint64()), uint256.NewInt(rand.Uint64()), index)
 	var (
 		addrs = make([]common.Address, b.N)
 		metas = make([]*blobTxMeta, b.N)

core/types.go

@@ -19,7 +19,9 @@ package core
 import (
 	"sync/atomic"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/stateless"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 )
@@ -33,7 +35,10 @@ type Validator interface {
 
 	// ValidateState validates the given statedb and optionally the receipts and
 	// gas used.
-	ValidateState(block *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64) error
+	ValidateState(block *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64, stateless bool) error
+
+	// ValidateWitness cross validates a block execution with stateless remote clients.
+	ValidateWitness(witness *stateless.Witness, receiptRoot common.Hash, stateRoot common.Hash) error
 }
 
 // Prefetcher is an interface for pre-caching transaction signatures and state.

core/types/transaction_signing.go

@@ -572,6 +572,6 @@ func deriveChainId(v *big.Int) *big.Int {
 		}
 		return new(big.Int).SetUint64((v - 35) / 2)
 	}
-	v = new(big.Int).Sub(v, big.NewInt(35))
+	v.Sub(v, big.NewInt(35))
-	return v.Div(v, big.NewInt(2))
+	return v.Rsh(v, 1)
 }

core/vm/contracts.go

@@ -296,10 +296,7 @@ type bigModExp struct {
 var (
 	big1      = big.NewInt(1)
 	big3      = big.NewInt(3)
-	big4      = big.NewInt(4)
 	big7      = big.NewInt(7)
-	big8      = big.NewInt(8)
-	big16     = big.NewInt(16)
 	big20     = big.NewInt(20)
 	big32     = big.NewInt(32)
 	big64     = big.NewInt(64)
@@ -325,13 +322,13 @@ func modexpMultComplexity(x *big.Int) *big.Int {
 	case x.Cmp(big1024) <= 0:
 		// (x ** 2 // 4 ) + ( 96 * x - 3072)
 		x = new(big.Int).Add(
-			new(big.Int).Div(new(big.Int).Mul(x, x), big4),
+			new(big.Int).Rsh(new(big.Int).Mul(x, x), 2),
 			new(big.Int).Sub(new(big.Int).Mul(big96, x), big3072),
 		)
 	default:
 		// (x ** 2 // 16) + (480 * x - 199680)
 		x = new(big.Int).Add(
-			new(big.Int).Div(new(big.Int).Mul(x, x), big16),
+			new(big.Int).Rsh(new(big.Int).Mul(x, x), 4),
 			new(big.Int).Sub(new(big.Int).Mul(big480, x), big199680),
 		)
 	}
@@ -369,7 +366,7 @@ func (c *bigModExp) RequiredGas(input []byte) uint64 {
 	adjExpLen := new(big.Int)
 	if expLen.Cmp(big32) > 0 {
 		adjExpLen.Sub(expLen, big32)
-		adjExpLen.Mul(big8, adjExpLen)
+		adjExpLen.Lsh(adjExpLen, 3)
 	}
 	adjExpLen.Add(adjExpLen, big.NewInt(int64(msb)))
 	// Calculate the gas cost of the operation
@@ -383,8 +380,8 @@ func (c *bigModExp) RequiredGas(input []byte) uint64 {
 		//    ceiling(x/8)^2
 		//
 		//where is x is max(length_of_MODULUS, length_of_BASE)
-		gas = gas.Add(gas, big7)
+		gas.Add(gas, big7)
-		gas = gas.Div(gas, big8)
+		gas.Rsh(gas, 3)
 		gas.Mul(gas, gas)
 
 		gas.Mul(gas, math.BigMax(adjExpLen, big1))

core/vm/evm.go

@@ -231,6 +231,9 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas
 		// Initialise a new contract and set the code that is to be used by the EVM.
 		// The contract is a scoped environment for this execution context only.
 		code := evm.StateDB.GetCode(addr)
+		if witness := evm.StateDB.Witness(); witness != nil {
+			witness.AddCode(code)
+		}
 		if len(code) == 0 {
 			ret, err = nil, nil // gas is unchanged
 		} else {
@@ -298,6 +301,9 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte,
 		// Initialise a new contract and set the code that is to be used by the EVM.
 		// The contract is a scoped environment for this execution context only.
 		contract := NewContract(caller, AccountRef(caller.Address()), value, gas)
+		if witness := evm.StateDB.Witness(); witness != nil {
+			witness.AddCode(evm.StateDB.GetCode(addrCopy))
+		}
 		contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy))
 		ret, err = evm.interpreter.Run(contract, input, false)
 		gas = contract.Gas
@@ -345,6 +351,9 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by
 		addrCopy := addr
 		// Initialise a new contract and make initialise the delegate values
 		contract := NewContract(caller, AccountRef(caller.Address()), nil, gas).AsDelegate()
+		if witness := evm.StateDB.Witness(); witness != nil {
+			witness.AddCode(evm.StateDB.GetCode(addrCopy))
+		}
 		contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy))
 		ret, err = evm.interpreter.Run(contract, input, false)
 		gas = contract.Gas
@@ -400,6 +409,9 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte
 		// Initialise a new contract and set the code that is to be used by the EVM.
 		// The contract is a scoped environment for this execution context only.
 		contract := NewContract(caller, AccountRef(addrCopy), new(uint256.Int), gas)
+		if witness := evm.StateDB.Witness(); witness != nil {
+			witness.AddCode(evm.StateDB.GetCode(addrCopy))
+		}
 		contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy))
 		// When an error was returned by the EVM or when setting the creation code
 		// above we revert to the snapshot and consume any gas remaining. Additionally

core/vm/instructions.go

@@ -340,6 +340,10 @@ func opReturnDataCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeConte
 
 func opExtCodeSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) {
 	slot := scope.Stack.peek()
+	address := slot.Bytes20()
+	if witness := interpreter.evm.StateDB.Witness(); witness != nil {
+		witness.AddCode(interpreter.evm.StateDB.GetCode(address))
+	}
 	slot.SetUint64(uint64(interpreter.evm.StateDB.GetCodeSize(slot.Bytes20())))
 	return nil, nil
 }
@@ -378,7 +382,11 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext)
 		uint64CodeOffset = math.MaxUint64
 	}
 	addr := common.Address(a.Bytes20())
-	codeCopy := getData(interpreter.evm.StateDB.GetCode(addr), uint64CodeOffset, length.Uint64())
+	code := interpreter.evm.StateDB.GetCode(addr)
+	if witness := interpreter.evm.StateDB.Witness(); witness != nil {
+		witness.AddCode(code)
+	}
+	codeCopy := getData(code, uint64CodeOffset, length.Uint64())
 	scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy)
 
 	return nil, nil
@@ -443,7 +451,11 @@ func opBlockhash(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) (
 		lower = upper - 256
 	}
 	if num64 >= lower && num64 < upper {
-		num.SetBytes(interpreter.evm.Context.GetHash(num64).Bytes())
+		res := interpreter.evm.Context.GetHash(num64)
+		if witness := interpreter.evm.StateDB.Witness(); witness != nil {
+			witness.AddBlockHash(num64)
+		}
+		num.SetBytes(res[:])
 	} else {
 		num.Clear()
 	}

core/vm/interface.go

@@ -20,6 +20,7 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/stateless"
 	"github.com/ethereum/go-ethereum/core/tracing"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/params"
@@ -87,6 +88,8 @@ type StateDB interface {
 
 	AddLog(*types.Log)
 	AddPreimage(common.Hash, []byte)
+
+	Witness() *stateless.Witness
 }
 
 // CallContext provides a basic interface for the EVM calling conventions. The EVM

eth/tracers/live/supply.go

@@ -204,12 +204,10 @@ func (s *supply) internalTxsHandler(call *supplyTxCallstack) {
 		s.delta.Burn.Misc.Add(s.delta.Burn.Misc, call.burn)
 	}
 
-	if len(call.calls) > 0 {
+	// Recursively handle internal calls
-		// Recursively handle internal calls
+	for _, call := range call.calls {
-		for _, call := range call.calls {
+		callCopy := call
-			callCopy := call
+		s.internalTxsHandler(&callCopy)
-			s.internalTxsHandler(&callCopy)
-		}
 	}
 }
 

eth/tracers/native/call_flat.go

@@ -274,16 +274,14 @@ func flatFromNested(input *callFrame, traceAddress []int, convertErrs bool, ctx
 	}
 
 	output = append(output, *frame)
-	if len(input.Calls) > 0 {
+	for i, childCall := range input.Calls {
-		for i, childCall := range input.Calls {
+		childAddr := childTraceAddress(traceAddress, i)
-			childAddr := childTraceAddress(traceAddress, i)
+		childCallCopy := childCall
-			childCallCopy := childCall
+		flat, err := flatFromNested(&childCallCopy, childAddr, convertErrs, ctx)
-			flat, err := flatFromNested(&childCallCopy, childAddr, convertErrs, ctx)
+		if err != nil {
-			if err != nil {
+			return nil, err
-				return nil, err
-			}
-			output = append(output, flat...)
 		}
+		output = append(output, flat...)
 	}
 
 	return output, nil

internal/debug/flags.go

@@ -231,9 +231,9 @@ func Setup(ctx *cli.Context) error {
 	case ctx.Bool(logjsonFlag.Name):
 		// Retain backwards compatibility with `--log.json` flag if `--log.format` not set
 		defer log.Warn("The flag '--log.json' is deprecated, please use '--log.format=json' instead")
-		handler = log.JSONHandlerWithLevel(output, log.LevelInfo)
+		handler = log.JSONHandler(output)
 	case logFmtFlag == "json":
-		handler = log.JSONHandlerWithLevel(output, log.LevelInfo)
+		handler = log.JSONHandler(output)
 	case logFmtFlag == "logfmt":
 		handler = log.LogfmtHandler(output)
 	case logFmtFlag == "", logFmtFlag == "terminal":

rpc/handler.go

@@ -17,8 +17,11 @@
 package rpc
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
+	"fmt"
 	"reflect"
 	"strconv"
 	"strings"
@@ -468,16 +471,16 @@ func (h *handler) handleCallMsg(ctx *callProc, msg *jsonrpcMessage) *jsonrpcMess
 
 	case msg.isCall():
 		resp := h.handleCall(ctx, msg)
-		var ctx []interface{}
+		var logctx []any
-		ctx = append(ctx, "reqid", idForLog{msg.ID}, "duration", time.Since(start))
+		logctx = append(logctx, "reqid", idForLog{msg.ID}, "duration", time.Since(start))
 		if resp.Error != nil {
-			ctx = append(ctx, "err", resp.Error.Message)
+			logctx = append(logctx, "err", resp.Error.Message)
 			if resp.Error.Data != nil {
-				ctx = append(ctx, "errdata", resp.Error.Data)
+				logctx = append(logctx, "errdata", formatErrorData(resp.Error.Data))
 			}
-			h.log.Warn("Served "+msg.Method, ctx...)
+			h.log.Warn("Served "+msg.Method, logctx...)
 		} else {
-			h.log.Debug("Served "+msg.Method, ctx...)
+			h.log.Debug("Served "+msg.Method, logctx...)
 		}
 		return resp
 
@@ -591,3 +594,33 @@ func (id idForLog) String() string {
 	}
 	return string(id.RawMessage)
 }
+
+var errTruncatedOutput = errors.New("truncated output")
+
+type limitedBuffer struct {
+	output []byte
+	limit  int
+}
+
+func (buf *limitedBuffer) Write(data []byte) (int, error) {
+	avail := max(buf.limit, len(buf.output))
+	if len(data) < avail {
+		buf.output = append(buf.output, data...)
+		return len(data), nil
+	}
+	buf.output = append(buf.output, data[:avail]...)
+	return avail, errTruncatedOutput
+}
+
+func formatErrorData(v any) string {
+	buf := limitedBuffer{limit: 1024}
+	err := json.NewEncoder(&buf).Encode(v)
+	switch {
+	case err == nil:
+		return string(bytes.TrimRight(buf.output, "\n"))
+	case errors.Is(err, errTruncatedOutput):
+		return fmt.Sprintf("%s... (truncated)", buf.output)
+	default:
+		return fmt.Sprintf("bad error data (err=%v)", err)
+	}
+}

tests/block_test.go

@@ -26,11 +26,11 @@ import (
 
 func TestBlockchain(t *testing.T) {
 	bt := new(testMatcher)
-	// General state tests are 'exported' as blockchain tests, but we can run them natively.
+
-	// For speedier CI-runs, the line below can be uncommented, so those are skipped.
+	// We are running most of GeneralStatetests to tests witness support, even
-	// For now, in hardfork-times (Berlin), we run the tests both as StateTests and
+	// though they are ran as state tests too. Still, the performance tests are
-	// as blockchain tests, since the latter also covers things like receipt root
+	// less about state andmore about EVM number crunching, so skip those.
-	bt.skipLoad(`^GeneralStateTests/`)
+	bt.skipLoad(`^GeneralStateTests/VMTests/vmPerformance`)
 
 	// Skip random failures due to selfish mining test
 	bt.skipLoad(`.*bcForgedTest/bcForkUncle\.json`)
@@ -70,33 +70,25 @@ func TestExecutionSpecBlocktests(t *testing.T) {
 }
 
 func execBlockTest(t *testing.T, bt *testMatcher, test *BlockTest) {
-	// If -short flag is used, we don't execute all four permutations, only one.
+	// Define all the different flag combinations we should run the tests with,
-	executionMask := 0xf
+	// picking only one for short tests.
+	//
+	// Note, witness building and self-testing is always enabled as it's a very
+	// good test to ensure that we don't break it.
+	var (
+		snapshotConf = []bool{false, true}
+		dbschemeConf = []string{rawdb.HashScheme, rawdb.PathScheme}
+	)
 	if testing.Short() {
-		executionMask = (1 << (rand.Int63() & 4))
+		snapshotConf = []bool{snapshotConf[rand.Int()%2]}
+		dbschemeConf = []string{dbschemeConf[rand.Int()%2]}
 	}
-	if executionMask&0x1 != 0 {
+	for _, snapshot := range snapshotConf {
-		if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil, nil)); err != nil {
+		for _, dbscheme := range dbschemeConf {
-			t.Errorf("test in hash mode without snapshotter failed: %v", err)
+			if err := bt.checkFailure(t, test.Run(snapshot, dbscheme, true, nil, nil)); err != nil {
-			return
+				t.Errorf("test with config {snapshotter:%v, scheme:%v} failed: %v", snapshot, dbscheme, err)
-		}
+				return
-	}
+			}
-	if executionMask&0x2 != 0 {
-		if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil, nil)); err != nil {
-			t.Errorf("test in hash mode with snapshotter failed: %v", err)
-			return
-		}
-	}
-	if executionMask&0x4 != 0 {
-		if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil, nil)); err != nil {
-			t.Errorf("test in path mode without snapshotter failed: %v", err)
-			return
-		}
-	}
-	if executionMask&0x8 != 0 {
-		if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil, nil)); err != nil {
-			t.Errorf("test in path mode with snapshotter failed: %v", err)
-			return
 		}
 	}
 }

tests/block_test_util.go

@@ -110,7 +110,7 @@ type btHeaderMarshaling struct {
 	ExcessBlobGas *math.HexOrDecimal64
 }
 
-func (t *BlockTest) Run(snapshotter bool, scheme string, tracer *tracing.Hooks, postCheck func(error, *core.BlockChain)) (result error) {
+func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *tracing.Hooks, postCheck func(error, *core.BlockChain)) (result error) {
 	config, ok := Forks[t.json.Network]
 	if !ok {
 		return UnsupportedForkError{t.json.Network}
@@ -151,7 +151,8 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer *tracing.Hooks,
 		cache.SnapshotWait = true
 	}
 	chain, err := core.NewBlockChain(db, cache, gspec, nil, engine, vm.Config{
-		Tracer: tracer,
+		Tracer:                  tracer,
+		EnableWitnessCollection: witness,
 	}, nil, nil)
 	if err != nil {
 		return err

trie/secure_trie.go

@@ -24,6 +24,16 @@ import (
 	"github.com/ethereum/go-ethereum/triedb/database"
 )
 
+// preimageStore wraps the methods of a backing store for reading and writing
+// trie node preimages.
+type preimageStore interface {
+	// Preimage retrieves the preimage of the specified hash.
+	Preimage(hash common.Hash) []byte
+
+	// InsertPreimage commits a set of preimages along with their hashes.
+	InsertPreimage(preimages map[common.Hash][]byte)
+}
+
 // SecureTrie is the old name of StateTrie.
 // Deprecated: use StateTrie.
 type SecureTrie = StateTrie
@@ -52,6 +62,7 @@ func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db da
 type StateTrie struct {
 	trie             Trie
 	db               database.Database
+	preimages        preimageStore
 	hashKeyBuf       [common.HashLength]byte
 	secKeyCache      map[string][]byte
 	secKeyCacheOwner *StateTrie // Pointer to self, replace the key cache on mismatch
@@ -70,7 +81,14 @@ func NewStateTrie(id *ID, db database.Database) (*StateTrie, error) {
 	if err != nil {
 		return nil, err
 	}
-	return &StateTrie{trie: *trie, db: db}, nil
+	tr := &StateTrie{trie: *trie, db: db}
+
+	// link the preimage store if it's supported
+	preimages, ok := db.(preimageStore)
+	if ok {
+		tr.preimages = preimages
+	}
+	return tr, nil
 }
 
 // MustGet returns the value for key stored in the trie.
@@ -211,7 +229,15 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte {
 	if key, ok := t.getSecKeyCache()[string(shaKey)]; ok {
 		return key
 	}
-	return t.db.Preimage(common.BytesToHash(shaKey))
+	if t.preimages == nil {
+		return nil
+	}
+	return t.preimages.Preimage(common.BytesToHash(shaKey))
+}
+
+// Witness returns a set containing all trie nodes that have been accessed.
+func (t *StateTrie) Witness() map[string]struct{} {
+	return t.trie.Witness()
 }
 
 // Commit collects all dirty nodes in the trie and replaces them with the
@@ -228,7 +254,9 @@ func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
 		for hk, key := range t.secKeyCache {
 			preimages[common.BytesToHash([]byte(hk))] = key
 		}
-		t.db.InsertPreimage(preimages)
+		if t.preimages != nil {
+			t.preimages.InsertPreimage(preimages)
+		}
 		t.secKeyCache = make(map[string][]byte)
 	}
 	// Commit the trie and return its modified nodeset.

trie/trie.go

@@ -661,6 +661,18 @@ func (t *Trie) hashRoot() (node, node) {
 	return hashed, cached
 }
 
+// Witness returns a set containing all trie nodes that have been accessed.
+func (t *Trie) Witness() map[string]struct{} {
+	if len(t.tracer.accessList) == 0 {
+		return nil
+	}
+	witness := make(map[string]struct{})
+	for _, node := range t.tracer.accessList {
+		witness[string(node)] = struct{}{}
+	}
+	return witness
+}
+
 // Reset drops the referenced root node and cleans all internal state.
 func (t *Trie) Reset() {
 	t.root = nil

trie/trie_reader.go

@@ -20,7 +20,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/trie/triestate"
 	"github.com/ethereum/go-ethereum/triedb/database"
 )
 
@@ -72,23 +71,3 @@ func (r *trieReader) node(path []byte, hash common.Hash) ([]byte, error) {
 	}
 	return blob, nil
 }
-
-// MerkleLoader implements triestate.TrieLoader for constructing tries.
-type MerkleLoader struct {
-	db database.Database
-}
-
-// NewMerkleLoader creates the merkle trie loader.
-func NewMerkleLoader(db database.Database) *MerkleLoader {
-	return &MerkleLoader{db: db}
-}
-
-// OpenTrie opens the main account trie.
-func (l *MerkleLoader) OpenTrie(root common.Hash) (triestate.Trie, error) {
-	return New(TrieID(root), l.db)
-}
-
-// OpenStorageTrie opens the storage trie of an account.
-func (l *MerkleLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) {
-	return New(StorageTrieID(stateRoot, addrHash, root), l.db)
-}

trie/trienode/node.go

@@ -139,16 +139,14 @@ func (set *NodeSet) Size() (int, int) {
 func (set *NodeSet) Summary() string {
 	var out = new(strings.Builder)
 	fmt.Fprintf(out, "nodeset owner: %v\n", set.Owner)
-	if set.Nodes != nil {
+	for path, n := range set.Nodes {
-		for path, n := range set.Nodes {
+		// Deletion
-			// Deletion
+		if n.IsDeleted() {
-			if n.IsDeleted() {
+			fmt.Fprintf(out, "  [-]: %x\n", path)
-				fmt.Fprintf(out, "  [-]: %x\n", path)
+			continue
-				continue
-			}
-			// Insertion or update
-			fmt.Fprintf(out, "  [+/*]: %x -> %v \n", path, n.Hash)
 		}
+		// Insertion or update
+		fmt.Fprintf(out, "  [+/*]: %x -> %v \n", path, n.Hash)
 	}
 	for _, n := range set.Leaves {
 		fmt.Fprintf(out, "[leaf]: %v\n", n)

trie/triestate/state.go

@@ -16,43 +16,7 @@
 
 package triestate
 
-import (
+import "github.com/ethereum/go-ethereum/common"
-	"errors"
-	"fmt"
-	"sync"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/trie/trienode"
-)
-
-// Trie is an Ethereum state trie, can be implemented by Ethereum Merkle Patricia
-// tree or Verkle tree.
-type Trie interface {
-	// Get returns the value for key stored in the trie.
-	Get(key []byte) ([]byte, error)
-
-	// Update associates key with value in the trie.
-	Update(key, value []byte) error
-
-	// Delete removes any existing value for key from the trie.
-	Delete(key []byte) error
-
-	// Commit the trie and returns a set of dirty nodes generated along with
-	// the new root hash.
-	Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet)
-}
-
-// TrieLoader wraps functions to load tries.
-type TrieLoader interface {
-	// OpenTrie opens the main account trie.
-	OpenTrie(root common.Hash) (Trie, error)
-
-	// OpenStorageTrie opens the storage trie of an account.
-	OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error)
-}
 
 // Set represents a collection of mutated states during a state transition.
 // The value refers to the original content of state before the transition
@@ -87,177 +51,3 @@ func (s *Set) Size() common.StorageSize {
 	}
 	return s.size
 }
-
-// context wraps all fields for executing state diffs.
-type context struct {
-	prevRoot    common.Hash
-	postRoot    common.Hash
-	accounts    map[common.Address][]byte
-	storages    map[common.Address]map[common.Hash][]byte
-	accountTrie Trie
-	nodes       *trienode.MergedNodeSet
-}
-
-// Apply traverses the provided state diffs, apply them in the associated
-// post-state and return the generated dirty trie nodes. The state can be
-// loaded via the provided trie loader.
-func Apply(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte, loader TrieLoader) (map[common.Hash]map[string]*trienode.Node, error) {
-	tr, err := loader.OpenTrie(postRoot)
-	if err != nil {
-		return nil, err
-	}
-	ctx := &context{
-		prevRoot:    prevRoot,
-		postRoot:    postRoot,
-		accounts:    accounts,
-		storages:    storages,
-		accountTrie: tr,
-		nodes:       trienode.NewMergedNodeSet(),
-	}
-	for addr, account := range accounts {
-		var err error
-		if len(account) == 0 {
-			err = deleteAccount(ctx, loader, addr)
-		} else {
-			err = updateAccount(ctx, loader, addr)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("failed to revert state, err: %w", err)
-		}
-	}
-	root, result := tr.Commit(false)
-	if root != prevRoot {
-		return nil, fmt.Errorf("failed to revert state, want %#x, got %#x", prevRoot, root)
-	}
-	if err := ctx.nodes.Merge(result); err != nil {
-		return nil, err
-	}
-	return ctx.nodes.Flatten(), nil
-}
-
-// updateAccount the account was present in prev-state, and may or may not
-// existent in post-state. Apply the reverse diff and verify if the storage
-// root matches the one in prev-state account.
-func updateAccount(ctx *context, loader TrieLoader, addr common.Address) error {
-	// The account was present in prev-state, decode it from the
-	// 'slim-rlp' format bytes.
-	h := newHasher()
-	defer h.release()
-
-	addrHash := h.hash(addr.Bytes())
-	prev, err := types.FullAccount(ctx.accounts[addr])
-	if err != nil {
-		return err
-	}
-	// The account may or may not existent in post-state, try to
-	// load it and decode if it's found.
-	blob, err := ctx.accountTrie.Get(addrHash.Bytes())
-	if err != nil {
-		return err
-	}
-	post := types.NewEmptyStateAccount()
-	if len(blob) != 0 {
-		if err := rlp.DecodeBytes(blob, &post); err != nil {
-			return err
-		}
-	}
-	// Apply all storage changes into the post-state storage trie.
-	st, err := loader.OpenStorageTrie(ctx.postRoot, addrHash, post.Root)
-	if err != nil {
-		return err
-	}
-	for key, val := range ctx.storages[addr] {
-		var err error
-		if len(val) == 0 {
-			err = st.Delete(key.Bytes())
-		} else {
-			err = st.Update(key.Bytes(), val)
-		}
-		if err != nil {
-			return err
-		}
-	}
-	root, result := st.Commit(false)
-	if root != prev.Root {
-		return errors.New("failed to reset storage trie")
-	}
-	// The returned set can be nil if storage trie is not changed
-	// at all.
-	if result != nil {
-		if err := ctx.nodes.Merge(result); err != nil {
-			return err
-		}
-	}
-	// Write the prev-state account into the main trie
-	full, err := rlp.EncodeToBytes(prev)
-	if err != nil {
-		return err
-	}
-	return ctx.accountTrie.Update(addrHash.Bytes(), full)
-}
-
-// deleteAccount the account was not present in prev-state, and is expected
-// to be existent in post-state. Apply the reverse diff and verify if the
-// account and storage is wiped out correctly.
-func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error {
-	// The account must be existent in post-state, load the account.
-	h := newHasher()
-	defer h.release()
-
-	addrHash := h.hash(addr.Bytes())
-	blob, err := ctx.accountTrie.Get(addrHash.Bytes())
-	if err != nil {
-		return err
-	}
-	if len(blob) == 0 {
-		return fmt.Errorf("account is non-existent %#x", addrHash)
-	}
-	var post types.StateAccount
-	if err := rlp.DecodeBytes(blob, &post); err != nil {
-		return err
-	}
-	st, err := loader.OpenStorageTrie(ctx.postRoot, addrHash, post.Root)
-	if err != nil {
-		return err
-	}
-	for key, val := range ctx.storages[addr] {
-		if len(val) != 0 {
-			return errors.New("expect storage deletion")
-		}
-		if err := st.Delete(key.Bytes()); err != nil {
-			return err
-		}
-	}
-	root, result := st.Commit(false)
-	if root != types.EmptyRootHash {
-		return errors.New("failed to clear storage trie")
-	}
-	// The returned set can be nil if storage trie is not changed
-	// at all.
-	if result != nil {
-		if err := ctx.nodes.Merge(result); err != nil {
-			return err
-		}
-	}
-	// Delete the post-state account from the main trie.
-	return ctx.accountTrie.Delete(addrHash.Bytes())
-}
-
-// hasher is used to compute the sha256 hash of the provided data.
-type hasher struct{ sha crypto.KeccakState }
-
-var hasherPool = sync.Pool{
-	New: func() interface{} { return &hasher{sha: crypto.NewKeccakState()} },
-}
-
-func newHasher() *hasher {
-	return hasherPool.Get().(*hasher)
-}
-
-func (h *hasher) hash(data []byte) common.Hash {
-	return crypto.HashData(h.sha, data)
-}
-
-func (h *hasher) release() {
-	hasherPool.Put(h)
-}

trie/verkle.go

@@ -144,10 +144,8 @@ func (t *VerkleTrie) UpdateAccount(addr common.Address, acc *types.StateAccount)
 
 	// Encode balance in little-endian
 	bytes := acc.Balance.Bytes()
-	if len(bytes) > 0 {
+	for i, b := range bytes {
-		for i, b := range bytes {
+		balance[len(bytes)-i-1] = b
-			balance[len(bytes)-i-1] = b
-		}
 	}
 	values[utils.BalanceLeafKey] = balance[:]
 
@@ -369,3 +367,8 @@ func (t *VerkleTrie) ToDot() string {
 func (t *VerkleTrie) nodeResolver(path []byte) ([]byte, error) {
 	return t.reader.node(path, common.Hash{})
 }
+
+// Witness returns a set containing all trie nodes that have been accessed.
+func (t *VerkleTrie) Witness() map[string]struct{} {
+	panic("not implemented")
+}

triedb/database.go

@@ -264,14 +264,7 @@ func (db *Database) Recover(target common.Hash) error {
 	if !ok {
 		return errors.New("not supported")
 	}
-	var loader triestate.TrieLoader
+	return pdb.Recover(target)
-	if db.config.IsVerkle {
-		// TODO define verkle loader
-		log.Crit("Verkle loader is not defined")
-	} else {
-		loader = trie.NewMerkleLoader(db)
-	}
-	return pdb.Recover(target, loader)
 }
 
 // Recoverable returns the indicator if the specified state is enabled to be

triedb/database/database.go

@@ -16,9 +16,7 @@
 
 package database
 
-import (
+import "github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common"
-)
 
 // Reader wraps the Node method of a backing trie reader.
 type Reader interface {
@@ -31,20 +29,8 @@ type Reader interface {
 	Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error)
 }
 
-// PreimageStore wraps the methods of a backing store for reading and writing
-// trie node preimages.
-type PreimageStore interface {
-	// Preimage retrieves the preimage of the specified hash.
-	Preimage(hash common.Hash) []byte
-
-	// InsertPreimage commits a set of preimages along with their hashes.
-	InsertPreimage(preimages map[common.Hash][]byte)
-}
-
 // Database wraps the methods of a backing trie store.
 type Database interface {
-	PreimageStore
-
 	// Reader returns a node reader associated with the specific state.
 	// An error will be returned if the specified state is not available.
 	Reader(stateRoot common.Hash) (Reader, error)

triedb/pathdb/database.go

@@ -345,7 +345,7 @@ func (db *Database) Enable(root common.Hash) error {
 // Recover rollbacks the database to a specified historical point.
 // The state is supported as the rollback destination only if it's
 // canonical state and the corresponding trie histories are existent.
-func (db *Database) Recover(root common.Hash, loader triestate.TrieLoader) error {
+func (db *Database) Recover(root common.Hash) error {
 	db.lock.Lock()
 	defer db.lock.Unlock()
 
@@ -371,7 +371,7 @@ func (db *Database) Recover(root common.Hash, loader triestate.TrieLoader) error
 		if err != nil {
 			return err
 		}
-		dl, err = dl.revert(h, loader)
+		dl, err = dl.revert(h)
 		if err != nil {
 			return err
 		}

triedb/pathdb/database_test.go

@@ -29,24 +29,31 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/internal/testrand"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 	"github.com/ethereum/go-ethereum/trie/triestate"
 	"github.com/holiman/uint256"
 )
 
-func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[common.Hash][]byte) (common.Hash, *trienode.NodeSet) {
+func updateTrie(db *Database, stateRoot common.Hash, addrHash common.Hash, root common.Hash, dirties map[common.Hash][]byte) (common.Hash, *trienode.NodeSet) {
-	h, err := newTestHasher(addrHash, root, cleans)
+	var id *trie.ID
+	if addrHash == (common.Hash{}) {
+		id = trie.StateTrieID(stateRoot)
+	} else {
+		id = trie.StorageTrieID(stateRoot, addrHash, root)
+	}
+	tr, err := trie.New(id, db)
 	if err != nil {
-		panic(fmt.Errorf("failed to create hasher, err: %w", err))
+		panic(fmt.Errorf("failed to load trie, err: %w", err))
 	}
 	for key, val := range dirties {
 		if len(val) == 0 {
-			h.Delete(key.Bytes())
+			tr.Delete(key.Bytes())
 		} else {
-			h.Update(key.Bytes(), val)
+			tr.Update(key.Bytes(), val)
 		}
 	}
-	return h.Commit(false)
+	return tr.Commit(false)
 }
 
 func generateAccount(storageRoot common.Hash) types.StateAccount {
@@ -66,6 +73,7 @@ const (
 )
 
 type genctx struct {
+	stateRoot     common.Hash
 	accounts      map[common.Hash][]byte
 	storages      map[common.Hash]map[common.Hash][]byte
 	accountOrigin map[common.Address][]byte
@@ -73,8 +81,9 @@ type genctx struct {
 	nodes         *trienode.MergedNodeSet
 }
 
-func newCtx() *genctx {
+func newCtx(stateRoot common.Hash) *genctx {
 	return &genctx{
+		stateRoot:     stateRoot,
 		accounts:      make(map[common.Hash][]byte),
 		storages:      make(map[common.Hash]map[common.Hash][]byte),
 		accountOrigin: make(map[common.Address][]byte),
@@ -112,7 +121,7 @@ func newTester(t *testing.T, historyLimit uint64) *tester {
 			snapStorages: make(map[common.Hash]map[common.Hash]map[common.Hash][]byte),
 		}
 	)
-	for i := 0; i < 8; i++ {
+	for i := 0; i < 12; i++ {
 		var parent = types.EmptyRootHash
 		if len(obj.roots) != 0 {
 			parent = obj.roots[len(obj.roots)-1]
@@ -151,7 +160,7 @@ func (t *tester) generateStorage(ctx *genctx, addr common.Address) common.Hash {
 		storage[hash] = v
 		origin[hash] = nil
 	}
-	root, set := updateTrie(addrHash, types.EmptyRootHash, storage, nil)
+	root, set := updateTrie(t.db, ctx.stateRoot, addrHash, types.EmptyRootHash, storage)
 
 	ctx.storages[addrHash] = storage
 	ctx.storageOrigin[addr] = origin
@@ -180,7 +189,7 @@ func (t *tester) mutateStorage(ctx *genctx, addr common.Address, root common.Has
 		storage[hash] = v
 		origin[hash] = nil
 	}
-	root, set := updateTrie(crypto.Keccak256Hash(addr.Bytes()), root, storage, t.storages[addrHash])
+	root, set := updateTrie(t.db, ctx.stateRoot, crypto.Keccak256Hash(addr.Bytes()), root, storage)
 
 	ctx.storages[addrHash] = storage
 	ctx.storageOrigin[addr] = origin
@@ -198,7 +207,7 @@ func (t *tester) clearStorage(ctx *genctx, addr common.Address, root common.Hash
 		origin[hash] = val
 		storage[hash] = nil
 	}
-	root, set := updateTrie(addrHash, root, storage, t.storages[addrHash])
+	root, set := updateTrie(t.db, ctx.stateRoot, addrHash, root, storage)
 	if root != types.EmptyRootHash {
 		panic("failed to clear storage trie")
 	}
@@ -210,7 +219,7 @@ func (t *tester) clearStorage(ctx *genctx, addr common.Address, root common.Hash
 
 func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNodeSet, *triestate.Set) {
 	var (
-		ctx     = newCtx()
+		ctx     = newCtx(parent)
 		dirties = make(map[common.Hash]struct{})
 	)
 	for i := 0; i < 20; i++ {
@@ -271,7 +280,7 @@ func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNode
 			ctx.accountOrigin[addr] = account
 		}
 	}
-	root, set := updateTrie(common.Hash{}, parent, ctx.accounts, t.accounts)
+	root, set := updateTrie(t.db, parent, common.Hash{}, parent, ctx.accounts)
 	ctx.nodes.Merge(set)
 
 	// Save state snapshot before commit
@@ -297,6 +306,9 @@ func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNode
 				t.storages[addrHash][sHash] = slot
 			}
 		}
+		if len(t.storages[addrHash]) == 0 {
+			delete(t.storages, addrHash)
+		}
 	}
 	return root, ctx.nodes, triestate.New(ctx.accountOrigin, ctx.storageOrigin)
 }
@@ -310,25 +322,31 @@ func (t *tester) lastHash() common.Hash {
 }
 
 func (t *tester) verifyState(root common.Hash) error {
-	reader, err := t.db.Reader(root)
+	tr, err := trie.New(trie.StateTrieID(root), t.db)
 	if err != nil {
 		return err
 	}
-	_, err = reader.Node(common.Hash{}, nil, root)
-	if err != nil {
-		return errors.New("root node is not available")
-	}
 	for addrHash, account := range t.snapAccounts[root] {
-		path := crypto.Keccak256(addrHash.Bytes())
+		blob, err := tr.Get(addrHash.Bytes())
-		blob, err := reader.Node(common.Hash{}, path, crypto.Keccak256Hash(account))
 		if err != nil || !bytes.Equal(blob, account) {
 			return fmt.Errorf("account is mismatched: %w", err)
 		}
 	}
 	for addrHash, slots := range t.snapStorages[root] {
+		blob := t.snapAccounts[root][addrHash]
+		if len(blob) == 0 {
+			return fmt.Errorf("account %x is missing", addrHash)
+		}
+		account := new(types.StateAccount)
+		if err := rlp.DecodeBytes(blob, account); err != nil {
+			return err
+		}
+		storageIt, err := trie.New(trie.StorageTrieID(root, addrHash, account.Root), t.db)
+		if err != nil {
+			return err
+		}
 		for hash, slot := range slots {
-			path := crypto.Keccak256(hash.Bytes())
+			blob, err := storageIt.Get(hash.Bytes())
-			blob, err := reader.Node(addrHash, path, crypto.Keccak256Hash(slot))
 			if err != nil || !bytes.Equal(blob, slot) {
 				return fmt.Errorf("slot is mismatched: %w", err)
 			}
@@ -395,13 +413,11 @@ func TestDatabaseRollback(t *testing.T) {
 	}
 	// Revert database from top to bottom
 	for i := tester.bottomIndex(); i >= 0; i-- {
-		root := tester.roots[i]
 		parent := types.EmptyRootHash
 		if i > 0 {
 			parent = tester.roots[i-1]
 		}
-		loader := newHashLoader(tester.snapAccounts[root], tester.snapStorages[root])
+		if err := tester.db.Recover(parent); err != nil {
-		if err := tester.db.Recover(parent, loader); err != nil {
 			t.Fatalf("Failed to revert db, err: %v", err)
 		}
 		if i > 0 {

triedb/pathdb/disklayer.go

@@ -219,7 +219,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) {
 }
 
 // revert applies the given state history and return a reverted disk layer.
-func (dl *diskLayer) revert(h *history, loader triestate.TrieLoader) (*diskLayer, error) {
+func (dl *diskLayer) revert(h *history) (*diskLayer, error) {
 	if h.meta.root != dl.rootHash() {
 		return nil, errUnexpectedHistory
 	}
@@ -229,7 +229,7 @@ func (dl *diskLayer) revert(h *history, loader triestate.TrieLoader) (*diskLayer
 	// Apply the reverse state changes upon the current state. This must
 	// be done before holding the lock in order to access state in "this"
 	// layer.
-	nodes, err := triestate.Apply(h.meta.parent, h.meta.root, h.accounts, h.storages, loader)
+	nodes, err := apply(dl.db, h.meta.parent, h.meta.root, h.accounts, h.storages)
 	if err != nil {
 		return nil, err
 	}

triedb/pathdb/execute.go

@@ -0,0 +1,186 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
+
+package pathdb
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/trienode"
+	"github.com/ethereum/go-ethereum/triedb/database"
+)
+
+// context wraps all fields for executing state diffs.
+type context struct {
+	prevRoot common.Hash
+	postRoot common.Hash
+	accounts map[common.Address][]byte
+	storages map[common.Address]map[common.Hash][]byte
+	nodes    *trienode.MergedNodeSet
+
+	// TODO (rjl493456442) abstract out the state hasher
+	// for supporting verkle tree.
+	accountTrie *trie.Trie
+}
+
+// apply processes the given state diffs, updates the corresponding post-state
+// and returns the trie nodes that have been modified.
+func apply(db database.Database, prevRoot common.Hash, postRoot common.Hash, accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte) (map[common.Hash]map[string]*trienode.Node, error) {
+	tr, err := trie.New(trie.TrieID(postRoot), db)
+	if err != nil {
+		return nil, err
+	}
+	ctx := &context{
+		prevRoot:    prevRoot,
+		postRoot:    postRoot,
+		accounts:    accounts,
+		storages:    storages,
+		accountTrie: tr,
+		nodes:       trienode.NewMergedNodeSet(),
+	}
+	for addr, account := range accounts {
+		var err error
+		if len(account) == 0 {
+			err = deleteAccount(ctx, db, addr)
+		} else {
+			err = updateAccount(ctx, db, addr)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("failed to revert state, err: %w", err)
+		}
+	}
+	root, result := tr.Commit(false)
+	if root != prevRoot {
+		return nil, fmt.Errorf("failed to revert state, want %#x, got %#x", prevRoot, root)
+	}
+	if err := ctx.nodes.Merge(result); err != nil {
+		return nil, err
+	}
+	return ctx.nodes.Flatten(), nil
+}
+
+// updateAccount the account was present in prev-state, and may or may not
+// existent in post-state. Apply the reverse diff and verify if the storage
+// root matches the one in prev-state account.
+func updateAccount(ctx *context, db database.Database, addr common.Address) error {
+	// The account was present in prev-state, decode it from the
+	// 'slim-rlp' format bytes.
+	h := newHasher()
+	defer h.release()
+
+	addrHash := h.hash(addr.Bytes())
+	prev, err := types.FullAccount(ctx.accounts[addr])
+	if err != nil {
+		return err
+	}
+	// The account may or may not existent in post-state, try to
+	// load it and decode if it's found.
+	blob, err := ctx.accountTrie.Get(addrHash.Bytes())
+	if err != nil {
+		return err
+	}
+	post := types.NewEmptyStateAccount()
+	if len(blob) != 0 {
+		if err := rlp.DecodeBytes(blob, &post); err != nil {
+			return err
+		}
+	}
+	// Apply all storage changes into the post-state storage trie.
+	st, err := trie.New(trie.StorageTrieID(ctx.postRoot, addrHash, post.Root), db)
+	if err != nil {
+		return err
+	}
+	for key, val := range ctx.storages[addr] {
+		var err error
+		if len(val) == 0 {
+			err = st.Delete(key.Bytes())
+		} else {
+			err = st.Update(key.Bytes(), val)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	root, result := st.Commit(false)
+	if root != prev.Root {
+		return errors.New("failed to reset storage trie")
+	}
+	// The returned set can be nil if storage trie is not changed
+	// at all.
+	if result != nil {
+		if err := ctx.nodes.Merge(result); err != nil {
+			return err
+		}
+	}
+	// Write the prev-state account into the main trie
+	full, err := rlp.EncodeToBytes(prev)
+	if err != nil {
+		return err
+	}
+	return ctx.accountTrie.Update(addrHash.Bytes(), full)
+}
+
+// deleteAccount the account was not present in prev-state, and is expected
+// to be existent in post-state. Apply the reverse diff and verify if the
+// account and storage is wiped out correctly.
+func deleteAccount(ctx *context, db database.Database, addr common.Address) error {
+	// The account must be existent in post-state, load the account.
+	h := newHasher()
+	defer h.release()
+
+	addrHash := h.hash(addr.Bytes())
+	blob, err := ctx.accountTrie.Get(addrHash.Bytes())
+	if err != nil {
+		return err
+	}
+	if len(blob) == 0 {
+		return fmt.Errorf("account is non-existent %#x", addrHash)
+	}
+	var post types.StateAccount
+	if err := rlp.DecodeBytes(blob, &post); err != nil {
+		return err
+	}
+	st, err := trie.New(trie.StorageTrieID(ctx.postRoot, addrHash, post.Root), db)
+	if err != nil {
+		return err
+	}
+	for key, val := range ctx.storages[addr] {
+		if len(val) != 0 {
+			return errors.New("expect storage deletion")
+		}
+		if err := st.Delete(key.Bytes()); err != nil {
+			return err
+		}
+	}
+	root, result := st.Commit(false)
+	if root != types.EmptyRootHash {
+		return errors.New("failed to clear storage trie")
+	}
+	// The returned set can be nil if storage trie is not changed
+	// at all.
+	if result != nil {
+		if err := ctx.nodes.Merge(result); err != nil {
+			return err
+		}
+	}
+	// Delete the post-state account from the main trie.
+	return ctx.accountTrie.Delete(addrHash.Bytes())
+}

triedb/pathdb/testutils.go

@@ -1,159 +0,0 @@
-// Copyright 2023 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package pathdb
-
-import (
-	"bytes"
-	"fmt"
-	"slices"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/trie/trienode"
-	"github.com/ethereum/go-ethereum/trie/triestate"
-)
-
-// testHasher is a test utility for computing root hash of a batch of state
-// elements. The hash algorithm is to sort all the elements in lexicographical
-// order, concat the key and value in turn, and perform hash calculation on
-// the concatenated bytes. Except the root hash, a nodeset will be returned
-// once Commit is called, which contains all the changes made to hasher.
-type testHasher struct {
-	owner   common.Hash            // owner identifier
-	root    common.Hash            // original root
-	dirties map[common.Hash][]byte // dirty states
-	cleans  map[common.Hash][]byte // clean states
-}
-
-// newTestHasher constructs a hasher object with provided states.
-func newTestHasher(owner common.Hash, root common.Hash, cleans map[common.Hash][]byte) (*testHasher, error) {
-	if cleans == nil {
-		cleans = make(map[common.Hash][]byte)
-	}
-	if got, _ := hash(cleans); got != root {
-		return nil, fmt.Errorf("state root mismatched, want: %x, got: %x", root, got)
-	}
-	return &testHasher{
-		owner:   owner,
-		root:    root,
-		dirties: make(map[common.Hash][]byte),
-		cleans:  cleans,
-	}, nil
-}
-
-// Get returns the value for key stored in the trie.
-func (h *testHasher) Get(key []byte) ([]byte, error) {
-	hash := common.BytesToHash(key)
-	val, ok := h.dirties[hash]
-	if ok {
-		return val, nil
-	}
-	return h.cleans[hash], nil
-}
-
-// Update associates key with value in the trie.
-func (h *testHasher) Update(key, value []byte) error {
-	h.dirties[common.BytesToHash(key)] = common.CopyBytes(value)
-	return nil
-}
-
-// Delete removes any existing value for key from the trie.
-func (h *testHasher) Delete(key []byte) error {
-	h.dirties[common.BytesToHash(key)] = nil
-	return nil
-}
-
-// Commit computes the new hash of the states and returns the set with all
-// state changes.
-func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
-	var (
-		nodes = make(map[common.Hash][]byte)
-		set   = trienode.NewNodeSet(h.owner)
-	)
-	for hash, val := range h.cleans {
-		nodes[hash] = val
-	}
-	for hash, val := range h.dirties {
-		nodes[hash] = val
-		if bytes.Equal(val, h.cleans[hash]) {
-			continue
-		}
-		// Utilize the hash of the state key as the node path to mitigate
-		// potential collisions within the path.
-		path := crypto.Keccak256(hash.Bytes())
-		if len(val) == 0 {
-			set.AddNode(path, trienode.NewDeleted())
-		} else {
-			set.AddNode(path, trienode.New(crypto.Keccak256Hash(val), val))
-		}
-	}
-	root, blob := hash(nodes)
-
-	// Include the dirty root node as well.
-	if root != types.EmptyRootHash && root != h.root {
-		set.AddNode(nil, trienode.New(root, blob))
-	}
-	if root == types.EmptyRootHash && h.root != types.EmptyRootHash {
-		set.AddNode(nil, trienode.NewDeleted())
-	}
-	return root, set
-}
-
-// hash performs the hash computation upon the provided states.
-func hash(states map[common.Hash][]byte) (common.Hash, []byte) {
-	var hs []common.Hash
-	for hash := range states {
-		hs = append(hs, hash)
-	}
-	slices.SortFunc(hs, common.Hash.Cmp)
-
-	var input []byte
-	for _, hash := range hs {
-		if len(states[hash]) == 0 {
-			continue
-		}
-		input = append(input, hash.Bytes()...)
-		input = append(input, states[hash]...)
-	}
-	if len(input) == 0 {
-		return types.EmptyRootHash, nil
-	}
-	return crypto.Keccak256Hash(input), input
-}
-
-type hashLoader struct {
-	accounts map[common.Hash][]byte
-	storages map[common.Hash]map[common.Hash][]byte
-}
-
-func newHashLoader(accounts map[common.Hash][]byte, storages map[common.Hash]map[common.Hash][]byte) *hashLoader {
-	return &hashLoader{
-		accounts: accounts,
-		storages: storages,
-	}
-}
-
-// OpenTrie opens the main account trie.
-func (l *hashLoader) OpenTrie(root common.Hash) (triestate.Trie, error) {
-	return newTestHasher(common.Hash{}, root, l.accounts)
-}
-
-// OpenStorageTrie opens the storage trie of an account.
-func (l *hashLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) {
-	return newTestHasher(addrHash, root, l.storages[addrHash])
-}