From d8ea7ac2b056f94ed77e18493d5d17f90f1c7747 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 25 Jun 2024 03:14:12 -0600 Subject: [PATCH 01/14] cmd/blsync: use debug.Setup for logging configuration (#30065) --- cmd/blsync/main.go | 42 ++++++++++++------------------------------ 1 file changed, 12 insertions(+), 30 deletions(-) diff --git a/cmd/blsync/main.go b/cmd/blsync/main.go index 2aa3d9a24e..854c99703c 100644 --- a/cmd/blsync/main.go +++ b/cmd/blsync/main.go @@ -19,39 +19,21 @@ package main import ( "context" "fmt" - "io" "os" "github.com/ethereum/go-ethereum/beacon/blsync" "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/rpc" - "github.com/mattn/go-colorable" - "github.com/mattn/go-isatty" "github.com/urfave/cli/v2" ) -var ( - verbosityFlag = &cli.IntFlag{ - Name: "verbosity", - Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail", - Value: 3, - Category: flags.LoggingCategory, - } - vmoduleFlag = &cli.StringFlag{ - Name: "vmodule", - Usage: "Per-module verbosity: comma-separated list of = (e.g. eth/*=5,p2p=4)", - Value: "", - Hidden: true, - Category: flags.LoggingCategory, - } -) - func main() { app := flags.NewApp("beacon light syncer tool") - app.Flags = []cli.Flag{ + app.Flags = flags.Merge([]cli.Flag{ utils.BeaconApiFlag, utils.BeaconApiHeaderFlag, utils.BeaconThresholdFlag, @@ -66,8 +48,16 @@ func main() { utils.GoerliFlag, utils.BlsyncApiFlag, utils.BlsyncJWTSecretFlag, - verbosityFlag, - vmoduleFlag, + }, + debug.Flags, + ) + app.Before = func(ctx *cli.Context) error { + flags.MigrateGlobalFlags(ctx) + return debug.Setup(ctx) + } + app.After = func(ctx *cli.Context) error { + debug.Exit() + return nil } app.Action = sync @@ -78,14 +68,6 @@ func main() { } func sync(ctx *cli.Context) error { - usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb" - output := io.Writer(os.Stderr) - if usecolor { - output = colorable.NewColorable(os.Stderr) - } - verbosity := log.FromLegacyLevel(ctx.Int(verbosityFlag.Name)) - log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(output, verbosity, usecolor))) - // set up blsync client := blsync.NewClient(ctx) client.SetEngineRPC(makeRPCClient(ctx)) From 0a651f8972392943b22120d2444adcfe8a480d14 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 25 Jun 2024 03:16:27 -0600 Subject: [PATCH 02/14] .github: add lightclient as codeowner to relevant packages (#30062) --- .github/CODEOWNERS | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 0dabaf4df5..ec2efb10e3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -4,14 +4,18 @@ accounts/usbwallet @karalabe accounts/scwallet @gballet accounts/abi @gballet @MariusVanDerWijden +beacon/engine @lightclient cmd/clef @holiman +cmd/evm @holiman @MariusVanDerWijden @lightclient consensus @karalabe core/ @karalabe @holiman @rjl493456442 eth/ @karalabe @holiman @rjl493456442 -eth/catalyst/ @gballet +eth/catalyst/ @gballet @lightclient eth/tracers/ @s1na core/tracing/ @s1na graphql/ @s1na +internal/ethapi @lightclient +internal/era @lightclient les/ @zsfelfoldi @rjl493456442 light/ @zsfelfoldi @rjl493456442 node/ @fjl From fe0c0b04fee6d7fbaad77540ceaed4df84939ba7 Mon Sep 17 00:00:00 2001 From: Halimao <1065621723@qq.com> Date: Tue, 25 Jun 2024 17:24:33 +0800 Subject: [PATCH 03/14] accounts/keystore: use t.TempDir in test (#30052) --- accounts/keystore/account_cache_test.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index 6bc14f5bb6..41a3002248 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -325,11 +325,7 @@ func TestUpdatedKeyfileContents(t *testing.T) { t.Parallel() // Create a temporary keystore to test with - dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-updatedkeyfilecontents-test-%d-%d", os.Getpid(), rand.Int())) - - // Create the directory - os.MkdirAll(dir, 0700) - defer os.RemoveAll(dir) + dir := t.TempDir() ks := NewKeyStore(dir, LightScryptN, LightScryptP) From 73f7e7c0878b61eceb11923187a34660a7218f47 Mon Sep 17 00:00:00 2001 From: AMIR <31338382+amiremohamadi@users.noreply.github.com> Date: Tue, 25 Jun 2024 13:00:58 +0330 Subject: [PATCH 04/14] internal/debug: remove unnecessary log level assignment (#30044) Log level is specified in L259 so it's unnecessary to specify it for handlers (L234, L236). --- internal/debug/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 361dc6fcca..19222c8325 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -231,9 +231,9 @@ func Setup(ctx *cli.Context) error { case ctx.Bool(logjsonFlag.Name): // Retain backwards compatibility with `--log.json` flag if `--log.format` not set defer log.Warn("The flag '--log.json' is deprecated, please use '--log.format=json' instead") - handler = log.JSONHandlerWithLevel(output, log.LevelInfo) + handler = log.JSONHandler(output) case logFmtFlag == "json": - handler = log.JSONHandlerWithLevel(output, log.LevelInfo) + handler = log.JSONHandler(output) case logFmtFlag == "logfmt": handler = log.LogfmtHandler(output) case logFmtFlag == "", logFmtFlag == "terminal": From ed8fd0ac0919ee954cfa621d16a5d0cfbf30c96d Mon Sep 17 00:00:00 2001 From: jwasinger Date: Tue, 25 Jun 2024 04:48:08 -0700 Subject: [PATCH 05/14] all: stateless witness builder and (self-)cross validator (#29719) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * all: add stateless verifications * all: simplify witness and integrate it into live geth --------- Co-authored-by: Péter Szilágyi --- cmd/evm/blockrunner.go | 2 +- core/block_validator.go | 37 ++++++- core/blockchain.go | 33 ++++-- core/blockchain_test.go | 3 +- core/state/database.go | 4 + core/state/state_object.go | 16 ++- core/state/statedb.go | 85 +++++++++++++-- core/state_prefetcher.go | 11 +- core/state_processor.go | 16 ++- core/stateless.go | 73 +++++++++++++ core/stateless/database.go | 60 +++++++++++ core/stateless/encoding.go | 129 ++++++++++++++++++++++ core/stateless/gen_encoding_json.go | 74 +++++++++++++ core/stateless/witness.go | 159 ++++++++++++++++++++++++++++ core/types.go | 7 +- core/vm/evm.go | 12 +++ core/vm/instructions.go | 16 ++- core/vm/interface.go | 3 + tests/block_test.go | 52 ++++----- tests/block_test_util.go | 5 +- trie/secure_trie.go | 5 + trie/trie.go | 12 +++ trie/verkle.go | 5 + 23 files changed, 738 insertions(+), 81 deletions(-) create mode 100644 core/stateless.go create mode 100644 core/stateless/database.go create mode 100644 core/stateless/encoding.go create mode 100644 core/stateless/gen_encoding_json.go create mode 100644 core/stateless/witness.go diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go index 0275c019bc..d5cd8d8e3d 100644 --- a/cmd/evm/blockrunner.go +++ b/cmd/evm/blockrunner.go @@ -86,7 +86,7 @@ func blockTestCmd(ctx *cli.Context) error { continue } test := tests[name] - if err := test.Run(false, rawdb.HashScheme, tracer, func(res error, chain *core.BlockChain) { + if err := test.Run(false, rawdb.HashScheme, false, tracer, func(res error, chain *core.BlockChain) { if ctx.Bool(DumpFlag.Name) { if state, _ := chain.State(); state != nil { fmt.Println(string(state.Dump(nil))) diff --git a/core/block_validator.go b/core/block_validator.go index 3d49f4e6a3..75f7f8a94b 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -20,8 +20,10 @@ import ( "errors" "fmt" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" @@ -34,14 +36,12 @@ import ( type BlockValidator struct { config *params.ChainConfig // Chain configuration options bc *BlockChain // Canonical block chain - engine consensus.Engine // Consensus engine used for validating } // NewBlockValidator returns a new block validator which is safe for re-use -func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine) *BlockValidator { +func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain) *BlockValidator { validator := &BlockValidator{ config: config, - engine: engine, bc: blockchain, } return validator @@ -59,7 +59,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { // Header validity is known at this point. Here we verify that uncles, transactions // and withdrawals given in the block body match the header. header := block.Header() - if err := v.engine.VerifyUncles(v.bc, block); err != nil { + if err := v.bc.engine.VerifyUncles(v.bc, block); err != nil { return err } if hash := types.CalcUncleHash(block.Uncles()); hash != header.UncleHash { @@ -121,7 +121,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { // ValidateState validates the various changes that happen after a state transition, // such as amount of used gas, the receipt roots and the state root itself. -func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateDB, receipts types.Receipts, usedGas uint64) error { +func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateDB, receipts types.Receipts, usedGas uint64, stateless bool) error { header := block.Header() if block.GasUsed() != usedGas { return fmt.Errorf("invalid gas used (remote: %d local: %d)", block.GasUsed(), usedGas) @@ -132,6 +132,11 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD if rbloom != header.Bloom { return fmt.Errorf("invalid bloom (remote: %x local: %x)", header.Bloom, rbloom) } + // In stateless mode, return early because the receipt and state root are not + // provided through the witness, rather the cross validator needs to return it. + if stateless { + return nil + } // The receipt Trie's root (R = (Tr [[H1, R1], ... [Hn, Rn]])) receiptSha := types.DeriveSha(receipts, trie.NewStackTrie(nil)) if receiptSha != header.ReceiptHash { @@ -145,6 +150,28 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD return nil } +// ValidateWitness cross validates a block execution with stateless remote clients. +// +// Normally we'd distribute the block witness to remote cross validators, wait +// for them to respond and then merge the results. For now, however, it's only +// Geth, so do an internal stateless run. +func (v *BlockValidator) ValidateWitness(witness *stateless.Witness, receiptRoot common.Hash, stateRoot common.Hash) error { + // Run the cross client stateless execution + // TODO(karalabe): Self-stateless for now, swap with other clients + crossReceiptRoot, crossStateRoot, err := ExecuteStateless(v.config, witness) + if err != nil { + return fmt.Errorf("stateless execution failed: %v", err) + } + // Stateless cross execution suceeeded, validate the withheld computed fields + if crossReceiptRoot != receiptRoot { + return fmt.Errorf("cross validator receipt root mismatch (cross: %x local: %x)", crossReceiptRoot, receiptRoot) + } + if crossStateRoot != stateRoot { + return fmt.Errorf("cross validator state root mismatch (cross: %x local: %x)", crossStateRoot, stateRoot) + } + return nil +} + // CalcGasLimit computes the gas limit of the next block after parent. It aims // to keep the baseline gas close to the provided target, and increase it towards // the target if the baseline gas is lower. diff --git a/core/blockchain.go b/core/blockchain.go index ac4eb1c47e..05ebfd18b8 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -302,18 +303,18 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis vmConfig: vmConfig, logger: vmConfig.Tracer, } - bc.flushInterval.Store(int64(cacheConfig.TrieTimeLimit)) - bc.forker = NewForkChoice(bc, shouldPreserve) - bc.stateCache = state.NewDatabaseWithNodeDB(bc.db, bc.triedb) - bc.validator = NewBlockValidator(chainConfig, bc, engine) - bc.prefetcher = newStatePrefetcher(chainConfig, bc, engine) - bc.processor = NewStateProcessor(chainConfig, bc, engine) - var err error bc.hc, err = NewHeaderChain(db, chainConfig, engine, bc.insertStopped) if err != nil { return nil, err } + bc.flushInterval.Store(int64(cacheConfig.TrieTimeLimit)) + bc.forker = NewForkChoice(bc, shouldPreserve) + bc.stateCache = state.NewDatabaseWithNodeDB(bc.db, bc.triedb) + bc.validator = NewBlockValidator(chainConfig, bc) + bc.prefetcher = newStatePrefetcher(chainConfig, bc.hc) + bc.processor = NewStateProcessor(chainConfig, bc.hc) + bc.genesisBlock = bc.GetBlockByNumber(0) if bc.genesisBlock == nil { return nil, ErrNoGenesis @@ -1809,7 +1810,14 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) // while processing transactions. Before Byzantium the prefetcher is mostly // useless due to the intermediate root hashing after each transaction. if bc.chainConfig.IsByzantium(block.Number()) { - statedb.StartPrefetcher("chain", !bc.vmConfig.EnableWitnessCollection) + var witness *stateless.Witness + if bc.vmConfig.EnableWitnessCollection { + witness, err = stateless.NewWitness(bc, block) + if err != nil { + return it.index, err + } + } + statedb.StartPrefetcher("chain", witness) } activeState = statedb @@ -1924,11 +1932,18 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s ptime := time.Since(pstart) vstart := time.Now() - if err := bc.validator.ValidateState(block, statedb, receipts, usedGas); err != nil { + if err := bc.validator.ValidateState(block, statedb, receipts, usedGas, false); err != nil { bc.reportBlock(block, receipts, err) return nil, err } vtime := time.Since(vstart) + + if witness := statedb.Witness(); witness != nil { + if err = bc.validator.ValidateWitness(witness, block.ReceiptHash(), block.Root()); err != nil { + bc.reportBlock(block, receipts, err) + return nil, fmt.Errorf("cross verification failed: %v", err) + } + } proctime := time.Since(start) // processing + validation // Update the metrics touched during block processing and validation diff --git a/core/blockchain_test.go b/core/blockchain_test.go index e4bc3e09a6..4f28c6f5e6 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -168,8 +168,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { blockchain.reportBlock(block, receipts, err) return err } - err = blockchain.validator.ValidateState(block, statedb, receipts, usedGas) - if err != nil { + if err = blockchain.validator.ValidateState(block, statedb, receipts, usedGas, false); err != nil { blockchain.reportBlock(block, receipts, err) return err } diff --git a/core/state/database.go b/core/state/database.go index d71f8f34b6..d54417d2f9 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -125,6 +125,10 @@ type Trie interface { // be created with new root and updated trie database for following usage Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) + // Witness returns a set containing all trie nodes that have been accessed. + // The returned map could be nil if the witness is empty. + Witness() map[string]struct{} + // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. And error will be returned // if fails to create node iterator. diff --git a/core/state/state_object.go b/core/state/state_object.go index 5c1dab53dc..880b715b4b 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -323,11 +323,17 @@ func (s *stateObject) finalise() { // // It assumes all the dirty storage slots have been finalized before. func (s *stateObject) updateTrie() (Trie, error) { - // Short circuit if nothing changed, don't bother with hashing anything + // Short circuit if nothing was accessed, don't trigger a prefetcher warning if len(s.uncommittedStorage) == 0 { - return s.trie, nil + // Nothing was written, so we could stop early. Unless we have both reads + // and witness collection enabled, in which case we need to fetch the trie. + if s.db.witness == nil || len(s.originStorage) == 0 { + return s.trie, nil + } } - // Retrieve a pretecher populated trie, or fall back to the database + // Retrieve a pretecher populated trie, or fall back to the database. This will + // block until all prefetch tasks are done, which are needed for witnesses even + // for unmodified state objects. tr := s.getPrefetchedTrie() if tr != nil { // Prefetcher returned a live trie, swap it out for the current one @@ -341,6 +347,10 @@ func (s *stateObject) updateTrie() (Trie, error) { return nil, err } } + // Short circuit if nothing changed, don't bother with hashing anything + if len(s.uncommittedStorage) == 0 { + return s.trie, nil + } // Perform trie updates before deletions. This prevents resolution of unnecessary trie nodes // in circumstances similar to the following: // diff --git a/core/state/statedb.go b/core/state/statedb.go index 4f84d93d63..ac82a8e3e3 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -105,7 +106,7 @@ type StateDB struct { // resurrection. The account value is tracked as the original value // before the transition. This map is populated at the transaction // boundaries. - stateObjectsDestruct map[common.Address]*types.StateAccount + stateObjectsDestruct map[common.Address]*stateObject // This map tracks the account mutations that occurred during the // transition. Uncommitted mutations belonging to the same account @@ -146,6 +147,9 @@ type StateDB struct { validRevisions []revision nextRevisionId int + // State witness if cross validation is needed + witness *stateless.Witness + // Measurements gathered during execution for debugging purposes AccountReads time.Duration AccountHashes time.Duration @@ -177,7 +181,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) originalRoot: root, snaps: snaps, stateObjects: make(map[common.Address]*stateObject), - stateObjectsDestruct: make(map[common.Address]*types.StateAccount), + stateObjectsDestruct: make(map[common.Address]*stateObject), mutations: make(map[common.Address]*mutation), logs: make(map[common.Hash][]*types.Log), preimages: make(map[common.Hash][]byte), @@ -200,14 +204,19 @@ func (s *StateDB) SetLogger(l *tracing.Hooks) { // StartPrefetcher initializes a new trie prefetcher to pull in nodes from the // state trie concurrently while the state is mutated so that when we reach the // commit phase, most of the needed data is already hot. -func (s *StateDB) StartPrefetcher(namespace string, noreads bool) { +func (s *StateDB) StartPrefetcher(namespace string, witness *stateless.Witness) { + // Terminate any previously running prefetcher if s.prefetcher != nil { s.prefetcher.terminate(false) s.prefetcher.report() s.prefetcher = nil } + // Enable witness collection if requested + s.witness = witness + + // If snapshots are enabled, start prefethers explicitly if s.snap != nil { - s.prefetcher = newTriePrefetcher(s.db, s.originalRoot, namespace, noreads) + s.prefetcher = newTriePrefetcher(s.db, s.originalRoot, namespace, witness == nil) // With the switch to the Proof-of-Stake consensus algorithm, block production // rewards are now handled at the consensus layer. Consequently, a block may @@ -582,7 +591,6 @@ func (s *StateDB) getStateObject(addr common.Address) *stateObject { start := time.Now() acc, err := s.snap.Account(crypto.HashData(s.hasher, addr.Bytes())) s.SnapshotAccountReads += time.Since(start) - if err == nil { if acc == nil { return nil @@ -683,7 +691,7 @@ func (s *StateDB) Copy() *StateDB { hasher: crypto.NewKeccakState(), originalRoot: s.originalRoot, stateObjects: make(map[common.Address]*stateObject, len(s.stateObjects)), - stateObjectsDestruct: maps.Clone(s.stateObjectsDestruct), + stateObjectsDestruct: make(map[common.Address]*stateObject, len(s.stateObjectsDestruct)), mutations: make(map[common.Address]*mutation, len(s.mutations)), dbErr: s.dbErr, refund: s.refund, @@ -703,10 +711,17 @@ func (s *StateDB) Copy() *StateDB { snaps: s.snaps, snap: s.snap, } + if s.witness != nil { + state.witness = s.witness.Copy() + } // Deep copy cached state objects. for addr, obj := range s.stateObjects { state.stateObjects[addr] = obj.deepCopy(state) } + // Deep copy destructed state objects. + for addr, obj := range s.stateObjectsDestruct { + state.stateObjectsDestruct[addr] = obj.deepCopy(state) + } // Deep copy the object state markers. for addr, op := range s.mutations { state.mutations[addr] = op.copy() @@ -788,7 +803,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // set indefinitely). Note only the first occurred self-destruct // event is tracked. if _, ok := s.stateObjectsDestruct[obj.address]; !ok { - s.stateObjectsDestruct[obj.address] = obj.origin + s.stateObjectsDestruct[obj.address] = obj } } else { obj.finalise() @@ -846,9 +861,46 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { obj := s.stateObjects[addr] // closure for the task runner below workers.Go(func() error { obj.updateRoot() + + // If witness building is enabled and the state object has a trie, + // gather the witnesses for its specific storage trie + if s.witness != nil && obj.trie != nil { + s.witness.AddState(obj.trie.Witness()) + } return nil }) } + // If witness building is enabled, gather all the read-only accesses + if s.witness != nil { + // Pull in anything that has been accessed before destruction + for _, obj := range s.stateObjectsDestruct { + // Skip any objects that haven't touched their storage + if len(obj.originStorage) == 0 { + continue + } + if trie := obj.getPrefetchedTrie(); trie != nil { + s.witness.AddState(trie.Witness()) + } else if obj.trie != nil { + s.witness.AddState(obj.trie.Witness()) + } + } + // Pull in only-read and non-destructed trie witnesses + for _, obj := range s.stateObjects { + // Skip any objects that have been updated + if _, ok := s.mutations[obj.address]; ok { + continue + } + // Skip any objects that haven't touched their storage + if len(obj.originStorage) == 0 { + continue + } + if trie := obj.getPrefetchedTrie(); trie != nil { + s.witness.AddState(trie.Witness()) + } else if obj.trie != nil { + s.witness.AddState(obj.trie.Witness()) + } + } + } workers.Wait() s.StorageUpdates += time.Since(start) @@ -904,7 +956,13 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // Track the amount of time wasted on hashing the account trie defer func(start time.Time) { s.AccountHashes += time.Since(start) }(time.Now()) - return s.trie.Hash() + hash := s.trie.Hash() + + // If witness building is enabled, gather the account trie witness + if s.witness != nil { + s.witness.AddState(s.trie.Witness()) + } + return hash } // SetTxContext sets the current transaction hash and index which are @@ -1060,7 +1118,9 @@ func (s *StateDB) handleDestruction() (map[common.Hash]*accountDelete, []*trieno buf = crypto.NewKeccakState() deletes = make(map[common.Hash]*accountDelete) ) - for addr, prev := range s.stateObjectsDestruct { + for addr, prevObj := range s.stateObjectsDestruct { + prev := prevObj.origin + // The account was non-existent, and it's marked as destructed in the scope // of block. It can be either case (a) or (b) and will be interpreted as // null->null state transition. @@ -1239,7 +1299,7 @@ func (s *StateDB) commit(deleteEmptyObjects bool) (*stateUpdate, error) { // Clear all internal flags and update state root at the end. s.mutations = make(map[common.Address]*mutation) - s.stateObjectsDestruct = make(map[common.Address]*types.StateAccount) + s.stateObjectsDestruct = make(map[common.Address]*stateObject) origin := s.originalRoot s.originalRoot = root @@ -1412,3 +1472,8 @@ func (s *StateDB) markUpdate(addr common.Address) { func (s *StateDB) PointCache() *utils.PointCache { return s.db.PointCache() } + +// Witness retrieves the current state witness being collected. +func (s *StateDB) Witness() *stateless.Witness { + return s.witness +} diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index ff867309de..31405fa078 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -19,7 +19,6 @@ package core import ( "sync/atomic" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" @@ -31,16 +30,14 @@ import ( // data from disk before the main block processor start executing. type statePrefetcher struct { config *params.ChainConfig // Chain configuration options - bc *BlockChain // Canonical block chain - engine consensus.Engine // Consensus engine used for block rewards + chain *HeaderChain // Canonical block chain } // newStatePrefetcher initialises a new statePrefetcher. -func newStatePrefetcher(config *params.ChainConfig, bc *BlockChain, engine consensus.Engine) *statePrefetcher { +func newStatePrefetcher(config *params.ChainConfig, chain *HeaderChain) *statePrefetcher { return &statePrefetcher{ config: config, - bc: bc, - engine: engine, + chain: chain, } } @@ -51,7 +48,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c var ( header = block.Header() gaspool = new(GasPool).AddGas(block.GasLimit()) - blockContext = NewEVMBlockContext(header, p.bc, nil) + blockContext = NewEVMBlockContext(header, p.chain, nil) evm = vm.NewEVM(blockContext, vm.TxContext{}, statedb, p.config, cfg) signer = types.MakeSigner(p.config, header.Number, header.Time) ) diff --git a/core/state_processor.go b/core/state_processor.go index 7166ed8bd8..c21f644f98 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -22,7 +22,6 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -37,16 +36,14 @@ import ( // StateProcessor implements Processor. type StateProcessor struct { config *params.ChainConfig // Chain configuration options - bc *BlockChain // Canonical block chain - engine consensus.Engine // Consensus engine used for block rewards + chain *HeaderChain // Canonical header chain } // NewStateProcessor initialises a new StateProcessor. -func NewStateProcessor(config *params.ChainConfig, bc *BlockChain, engine consensus.Engine) *StateProcessor { +func NewStateProcessor(config *params.ChainConfig, chain *HeaderChain) *StateProcessor { return &StateProcessor{ config: config, - bc: bc, - engine: engine, + chain: chain, } } @@ -73,10 +70,11 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg misc.ApplyDAOHardFork(statedb) } var ( - context = NewEVMBlockContext(header, p.bc, nil) - vmenv = vm.NewEVM(context, vm.TxContext{}, statedb, p.config, cfg) + context vm.BlockContext signer = types.MakeSigner(p.config, header.Number, header.Time) ) + context = NewEVMBlockContext(header, p.chain, nil) + vmenv := vm.NewEVM(context, vm.TxContext{}, statedb, p.config, cfg) if beaconRoot := block.BeaconRoot(); beaconRoot != nil { ProcessBeaconBlockRoot(*beaconRoot, vmenv, statedb) } @@ -101,7 +99,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, nil, 0, errors.New("withdrawals before shanghai") } // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) - p.engine.Finalize(p.bc, header, statedb, block.Body()) + p.chain.engine.Finalize(p.chain, header, statedb, block.Body()) return receipts, allLogs, *usedGas, nil } diff --git a/core/stateless.go b/core/stateless.go new file mode 100644 index 0000000000..4c7e6f3102 --- /dev/null +++ b/core/stateless.go @@ -0,0 +1,73 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package core + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/consensus/beacon" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/stateless" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/triedb" +) + +// ExecuteStateless runs a stateless execution based on a witness, verifies +// everything it can locally and returns the two computed fields that need the +// other side to explicitly check. +// +// This method is a bit of a sore thumb here, but: +// - It cannot be placed in core/stateless, because state.New prodces a circular dep +// - It cannot be placed outside of core, because it needs to construct a dud headerchain +// +// TODO(karalabe): Would be nice to resolve both issues above somehow and move it. +func ExecuteStateless(config *params.ChainConfig, witness *stateless.Witness) (common.Hash, common.Hash, error) { + // Create and populate the state database to serve as the stateless backend + memdb := witness.MakeHashDB() + + db, err := state.New(witness.Root(), state.NewDatabaseWithConfig(memdb, triedb.HashDefaults), nil) + if err != nil { + return common.Hash{}, common.Hash{}, err + } + // Create a blockchain that is idle, but can be used to access headers through + chain := &HeaderChain{ + config: config, + chainDb: memdb, + headerCache: lru.NewCache[common.Hash, *types.Header](256), + engine: beacon.New(ethash.NewFaker()), + } + processor := NewStateProcessor(config, chain) + validator := NewBlockValidator(config, nil) // No chain, we only validate the state, not the block + + // Run the stateless blocks processing and self-validate certain fields + receipts, _, usedGas, err := processor.Process(witness.Block, db, vm.Config{}) + if err != nil { + return common.Hash{}, common.Hash{}, err + } + if err = validator.ValidateState(witness.Block, db, receipts, usedGas, true); err != nil { + return common.Hash{}, common.Hash{}, err + } + // Almost everything validated, but receipt and state root needs to be returned + receiptRoot := types.DeriveSha(receipts, trie.NewStackTrie(nil)) + stateRoot := db.IntermediateRoot(config.IsEIP158(witness.Block.Number())) + + return receiptRoot, stateRoot, nil +} diff --git a/core/stateless/database.go b/core/stateless/database.go new file mode 100644 index 0000000000..135da62193 --- /dev/null +++ b/core/stateless/database.go @@ -0,0 +1,60 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stateless + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" +) + +// MakeHashDB imports tries, codes and block hashes from a witness into a new +// hash-based memory db. We could eventually rewrite this into a pathdb, but +// simple is better for now. +func (w *Witness) MakeHashDB() ethdb.Database { + var ( + memdb = rawdb.NewMemoryDatabase() + hasher = crypto.NewKeccakState() + hash = make([]byte, 32) + ) + // Inject all the "block hashes" (i.e. headers) into the ephemeral database + for _, header := range w.Headers { + rawdb.WriteHeader(memdb, header) + } + // Inject all the bytecodes into the ephemeral database + for code := range w.Codes { + blob := []byte(code) + + hasher.Reset() + hasher.Write(blob) + hasher.Read(hash) + + rawdb.WriteCode(memdb, common.BytesToHash(hash), blob) + } + // Inject all the MPT trie nodes into the ephemeral database + for node := range w.State { + blob := []byte(node) + + hasher.Reset() + hasher.Write(blob) + hasher.Read(hash) + + rawdb.WriteLegacyTrieNode(memdb, common.BytesToHash(hash), blob) + } + return memdb +} diff --git a/core/stateless/encoding.go b/core/stateless/encoding.go new file mode 100644 index 0000000000..2b7245d377 --- /dev/null +++ b/core/stateless/encoding.go @@ -0,0 +1,129 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stateless + +import ( + "bytes" + "errors" + "fmt" + "io" + "slices" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +//go:generate go run github.com/fjl/gencodec -type extWitness -field-override extWitnessMarshalling -out gen_encoding_json.go + +// toExtWitness converts our internal witness representation to the consensus one. +func (w *Witness) toExtWitness() *extWitness { + ext := &extWitness{ + Block: w.Block, + Headers: w.Headers, + } + ext.Codes = make([][]byte, 0, len(w.Codes)) + for code := range w.Codes { + ext.Codes = append(ext.Codes, []byte(code)) + } + slices.SortFunc(ext.Codes, bytes.Compare) + + ext.State = make([][]byte, 0, len(w.State)) + for node := range w.State { + ext.State = append(ext.State, []byte(node)) + } + slices.SortFunc(ext.State, bytes.Compare) + return ext +} + +// fromExtWitness converts the consensus witness format into our internal one. +func (w *Witness) fromExtWitness(ext *extWitness) error { + w.Block, w.Headers = ext.Block, ext.Headers + + w.Codes = make(map[string]struct{}, len(ext.Codes)) + for _, code := range ext.Codes { + w.Codes[string(code)] = struct{}{} + } + w.State = make(map[string]struct{}, len(ext.State)) + for _, node := range ext.State { + w.State[string(node)] = struct{}{} + } + return w.sanitize() +} + +// MarshalJSON marshals a witness as JSON. +func (w *Witness) MarshalJSON() ([]byte, error) { + return w.toExtWitness().MarshalJSON() +} + +// EncodeRLP serializes a witness as RLP. +func (w *Witness) EncodeRLP(wr io.Writer) error { + return rlp.Encode(wr, w.toExtWitness()) +} + +// UnmarshalJSON unmarshals from JSON. +func (w *Witness) UnmarshalJSON(input []byte) error { + var ext extWitness + if err := ext.UnmarshalJSON(input); err != nil { + return err + } + return w.fromExtWitness(&ext) +} + +// DecodeRLP decodes a witness from RLP. +func (w *Witness) DecodeRLP(s *rlp.Stream) error { + var ext extWitness + if err := s.Decode(&ext); err != nil { + return err + } + return w.fromExtWitness(&ext) +} + +// sanitize checks for some mandatory fields in the witness after decoding so +// the rest of the code can assume invariants and doesn't have to deal with +// corrupted data. +func (w *Witness) sanitize() error { + // Verify that the "parent" header (i.e. index 0) is available, and is the + // true parent of the block-to-be executed, since we use that to link the + // current block to the pre-state. + if len(w.Headers) == 0 { + return errors.New("parent header (for pre-root hash) missing") + } + for i, header := range w.Headers { + if header == nil { + return fmt.Errorf("witness header nil at position %d", i) + } + } + if w.Headers[0].Hash() != w.Block.ParentHash() { + return fmt.Errorf("parent hash different: witness %v, block parent %v", w.Headers[0].Hash(), w.Block.ParentHash()) + } + return nil +} + +// extWitness is a witness RLP encoding for transferring across clients. +type extWitness struct { + Block *types.Block `json:"block" gencodec:"required"` + Headers []*types.Header `json:"headers" gencodec:"required"` + Codes [][]byte `json:"codes"` + State [][]byte `json:"state"` +} + +// extWitnessMarshalling defines the hex marshalling types for a witness. +type extWitnessMarshalling struct { + Codes []hexutil.Bytes + State []hexutil.Bytes +} diff --git a/core/stateless/gen_encoding_json.go b/core/stateless/gen_encoding_json.go new file mode 100644 index 0000000000..1d0497976e --- /dev/null +++ b/core/stateless/gen_encoding_json.go @@ -0,0 +1,74 @@ +// Code generated by github.com/fjl/gencodec. DO NOT EDIT. + +package stateless + +import ( + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" +) + +var _ = (*extWitnessMarshalling)(nil) + +// MarshalJSON marshals as JSON. +func (e extWitness) MarshalJSON() ([]byte, error) { + type extWitness struct { + Block *types.Block `json:"block" gencodec:"required"` + Headers []*types.Header `json:"headers" gencodec:"required"` + Codes []hexutil.Bytes `json:"codes"` + State []hexutil.Bytes `json:"state"` + } + var enc extWitness + enc.Block = e.Block + enc.Headers = e.Headers + if e.Codes != nil { + enc.Codes = make([]hexutil.Bytes, len(e.Codes)) + for k, v := range e.Codes { + enc.Codes[k] = v + } + } + if e.State != nil { + enc.State = make([]hexutil.Bytes, len(e.State)) + for k, v := range e.State { + enc.State[k] = v + } + } + return json.Marshal(&enc) +} + +// UnmarshalJSON unmarshals from JSON. +func (e *extWitness) UnmarshalJSON(input []byte) error { + type extWitness struct { + Block *types.Block `json:"block" gencodec:"required"` + Headers []*types.Header `json:"headers" gencodec:"required"` + Codes []hexutil.Bytes `json:"codes"` + State []hexutil.Bytes `json:"state"` + } + var dec extWitness + if err := json.Unmarshal(input, &dec); err != nil { + return err + } + if dec.Block == nil { + return errors.New("missing required field 'block' for extWitness") + } + e.Block = dec.Block + if dec.Headers == nil { + return errors.New("missing required field 'headers' for extWitness") + } + e.Headers = dec.Headers + if dec.Codes != nil { + e.Codes = make([][]byte, len(dec.Codes)) + for k, v := range dec.Codes { + e.Codes[k] = v + } + } + if dec.State != nil { + e.State = make([][]byte, len(dec.State)) + for k, v := range dec.State { + e.State[k] = v + } + } + return nil +} diff --git a/core/stateless/witness.go b/core/stateless/witness.go new file mode 100644 index 0000000000..7622c5eb61 --- /dev/null +++ b/core/stateless/witness.go @@ -0,0 +1,159 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package stateless + +import ( + "bytes" + "errors" + "fmt" + "maps" + "slices" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// HeaderReader is an interface to pull in headers in place of block hashes for +// the witness. +type HeaderReader interface { + // GetHeader retrieves a block header from the database by hash and number, + GetHeader(hash common.Hash, number uint64) *types.Header +} + +// Witness encompasses a block, state and any other chain data required to apply +// a set of transactions and derive a post state/receipt root. +type Witness struct { + Block *types.Block // Current block with rootHash and receiptHash zeroed out + Headers []*types.Header // Past headers in reverse order (0=parent, 1=parent's-parent, etc). First *must* be set. + Codes map[string]struct{} // Set of bytecodes ran or accessed + State map[string]struct{} // Set of MPT state trie nodes (account and storage together) + + chain HeaderReader // Chain reader to convert block hash ops to header proofs + lock sync.Mutex // Lock to allow concurrent state insertions +} + +// NewWitness creates an empty witness ready for population. +func NewWitness(chain HeaderReader, block *types.Block) (*Witness, error) { + // Zero out the result fields to avoid accidentally sending them to the verifier + header := block.Header() + header.Root = common.Hash{} + header.ReceiptHash = common.Hash{} + + // Retrieve the parent header, which will *always* be included to act as a + // trustless pre-root hash container + parent := chain.GetHeader(block.ParentHash(), block.NumberU64()-1) + if parent == nil { + return nil, errors.New("failed to retrieve parent header") + } + // Create the wtness with a reconstructed gutted out block + return &Witness{ + Block: types.NewBlockWithHeader(header).WithBody(*block.Body()), + Codes: make(map[string]struct{}), + State: make(map[string]struct{}), + Headers: []*types.Header{parent}, + chain: chain, + }, nil +} + +// AddBlockHash adds a "blockhash" to the witness with the designated offset from +// chain head. Under the hood, this method actually pulls in enough headers from +// the chain to cover the block being added. +func (w *Witness) AddBlockHash(number uint64) { + // Keep pulling in headers until this hash is populated + for int(w.Block.NumberU64()-number) > len(w.Headers) { + tail := w.Block.Header() + if len(w.Headers) > 0 { + tail = w.Headers[len(w.Headers)-1] + } + w.Headers = append(w.Headers, w.chain.GetHeader(tail.ParentHash, tail.Number.Uint64()-1)) + } +} + +// AddCode adds a bytecode blob to the witness. +func (w *Witness) AddCode(code []byte) { + if len(code) == 0 { + return + } + w.Codes[string(code)] = struct{}{} +} + +// AddState inserts a batch of MPT trie nodes into the witness. +func (w *Witness) AddState(nodes map[string]struct{}) { + if len(nodes) == 0 { + return + } + w.lock.Lock() + defer w.lock.Unlock() + + for node := range nodes { + w.State[node] = struct{}{} + } +} + +// Copy deep-copies the witness object. Witness.Block isn't deep-copied as it +// is never mutated by Witness +func (w *Witness) Copy() *Witness { + return &Witness{ + Block: w.Block, + Headers: slices.Clone(w.Headers), + Codes: maps.Clone(w.Codes), + State: maps.Clone(w.State), + } +} + +// String prints a human-readable summary containing the total size of the +// witness and the sizes of the underlying components +func (w *Witness) String() string { + blob, _ := rlp.EncodeToBytes(w) + bytesTotal := len(blob) + + blob, _ = rlp.EncodeToBytes(w.Block) + bytesBlock := len(blob) + + bytesHeaders := 0 + for _, header := range w.Headers { + blob, _ = rlp.EncodeToBytes(header) + bytesHeaders += len(blob) + } + bytesCodes := 0 + for code := range w.Codes { + bytesCodes += len(code) + } + bytesState := 0 + for node := range w.State { + bytesState += len(node) + } + buf := new(bytes.Buffer) + + fmt.Fprintf(buf, "Witness #%d: %v\n", w.Block.Number(), common.StorageSize(bytesTotal)) + fmt.Fprintf(buf, " block (%4d txs): %10v\n", len(w.Block.Transactions()), common.StorageSize(bytesBlock)) + fmt.Fprintf(buf, "%4d headers: %10v\n", len(w.Headers), common.StorageSize(bytesHeaders)) + fmt.Fprintf(buf, "%4d trie nodes: %10v\n", len(w.State), common.StorageSize(bytesState)) + fmt.Fprintf(buf, "%4d codes: %10v\n", len(w.Codes), common.StorageSize(bytesCodes)) + + return buf.String() +} + +// Root returns the pre-state root from the first header. +// +// Note, this method will panic in case of a bad witness (but RLP decoding will +// sanitize it and fail before that). +func (w *Witness) Root() common.Hash { + return w.Headers[0].Root +} diff --git a/core/types.go b/core/types.go index 36eb0d1ded..dc13de52ce 100644 --- a/core/types.go +++ b/core/types.go @@ -19,7 +19,9 @@ package core import ( "sync/atomic" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" ) @@ -33,7 +35,10 @@ type Validator interface { // ValidateState validates the given statedb and optionally the receipts and // gas used. - ValidateState(block *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64) error + ValidateState(block *types.Block, state *state.StateDB, receipts types.Receipts, usedGas uint64, stateless bool) error + + // ValidateWitness cross validates a block execution with stateless remote clients. + ValidateWitness(witness *stateless.Witness, receiptRoot common.Hash, stateRoot common.Hash) error } // Prefetcher is an interface for pre-caching transaction signatures and state. diff --git a/core/vm/evm.go b/core/vm/evm.go index 26af0ea041..1944189b5d 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -231,6 +231,9 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. code := evm.StateDB.GetCode(addr) + if witness := evm.StateDB.Witness(); witness != nil { + witness.AddCode(code) + } if len(code) == 0 { ret, err = nil, nil // gas is unchanged } else { @@ -298,6 +301,9 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. contract := NewContract(caller, AccountRef(caller.Address()), value, gas) + if witness := evm.StateDB.Witness(); witness != nil { + witness.AddCode(evm.StateDB.GetCode(addrCopy)) + } contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy)) ret, err = evm.interpreter.Run(contract, input, false) gas = contract.Gas @@ -345,6 +351,9 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by addrCopy := addr // Initialise a new contract and make initialise the delegate values contract := NewContract(caller, AccountRef(caller.Address()), nil, gas).AsDelegate() + if witness := evm.StateDB.Witness(); witness != nil { + witness.AddCode(evm.StateDB.GetCode(addrCopy)) + } contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy)) ret, err = evm.interpreter.Run(contract, input, false) gas = contract.Gas @@ -400,6 +409,9 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. contract := NewContract(caller, AccountRef(addrCopy), new(uint256.Int), gas) + if witness := evm.StateDB.Witness(); witness != nil { + witness.AddCode(evm.StateDB.GetCode(addrCopy)) + } contract.SetCallCode(&addrCopy, evm.StateDB.GetCodeHash(addrCopy), evm.StateDB.GetCode(addrCopy)) // When an error was returned by the EVM or when setting the creation code // above we revert to the snapshot and consume any gas remaining. Additionally diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 10cdd72e0c..9ec4544643 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -340,6 +340,10 @@ func opReturnDataCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeConte func opExtCodeSize(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { slot := scope.Stack.peek() + address := slot.Bytes20() + if witness := interpreter.evm.StateDB.Witness(); witness != nil { + witness.AddCode(interpreter.evm.StateDB.GetCode(address)) + } slot.SetUint64(uint64(interpreter.evm.StateDB.GetCodeSize(slot.Bytes20()))) return nil, nil } @@ -378,7 +382,11 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) uint64CodeOffset = math.MaxUint64 } addr := common.Address(a.Bytes20()) - codeCopy := getData(interpreter.evm.StateDB.GetCode(addr), uint64CodeOffset, length.Uint64()) + code := interpreter.evm.StateDB.GetCode(addr) + if witness := interpreter.evm.StateDB.Witness(); witness != nil { + witness.AddCode(code) + } + codeCopy := getData(code, uint64CodeOffset, length.Uint64()) scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy) return nil, nil @@ -443,7 +451,11 @@ func opBlockhash(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ( lower = upper - 256 } if num64 >= lower && num64 < upper { - num.SetBytes(interpreter.evm.Context.GetHash(num64).Bytes()) + res := interpreter.evm.Context.GetHash(num64) + if witness := interpreter.evm.StateDB.Witness(); witness != nil { + witness.AddBlockHash(num64) + } + num.SetBytes(res[:]) } else { num.Clear() } diff --git a/core/vm/interface.go b/core/vm/interface.go index 8b2c58898e..5f42643565 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -20,6 +20,7 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" @@ -87,6 +88,8 @@ type StateDB interface { AddLog(*types.Log) AddPreimage(common.Hash, []byte) + + Witness() *stateless.Witness } // CallContext provides a basic interface for the EVM calling conventions. The EVM diff --git a/tests/block_test.go b/tests/block_test.go index 1ba84f5f24..52184eb274 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -26,11 +26,11 @@ import ( func TestBlockchain(t *testing.T) { bt := new(testMatcher) - // General state tests are 'exported' as blockchain tests, but we can run them natively. - // For speedier CI-runs, the line below can be uncommented, so those are skipped. - // For now, in hardfork-times (Berlin), we run the tests both as StateTests and - // as blockchain tests, since the latter also covers things like receipt root - bt.skipLoad(`^GeneralStateTests/`) + + // We are running most of GeneralStatetests to tests witness support, even + // though they are ran as state tests too. Still, the performance tests are + // less about state andmore about EVM number crunching, so skip those. + bt.skipLoad(`^GeneralStateTests/VMTests/vmPerformance`) // Skip random failures due to selfish mining test bt.skipLoad(`.*bcForgedTest/bcForkUncle\.json`) @@ -70,33 +70,25 @@ func TestExecutionSpecBlocktests(t *testing.T) { } func execBlockTest(t *testing.T, bt *testMatcher, test *BlockTest) { - // If -short flag is used, we don't execute all four permutations, only one. - executionMask := 0xf + // Define all the different flag combinations we should run the tests with, + // picking only one for short tests. + // + // Note, witness building and self-testing is always enabled as it's a very + // good test to ensure that we don't break it. + var ( + snapshotConf = []bool{false, true} + dbschemeConf = []string{rawdb.HashScheme, rawdb.PathScheme} + ) if testing.Short() { - executionMask = (1 << (rand.Int63() & 4)) + snapshotConf = []bool{snapshotConf[rand.Int()%2]} + dbschemeConf = []string{dbschemeConf[rand.Int()%2]} } - if executionMask&0x1 != 0 { - if err := bt.checkFailure(t, test.Run(false, rawdb.HashScheme, nil, nil)); err != nil { - t.Errorf("test in hash mode without snapshotter failed: %v", err) - return - } - } - if executionMask&0x2 != 0 { - if err := bt.checkFailure(t, test.Run(true, rawdb.HashScheme, nil, nil)); err != nil { - t.Errorf("test in hash mode with snapshotter failed: %v", err) - return - } - } - if executionMask&0x4 != 0 { - if err := bt.checkFailure(t, test.Run(false, rawdb.PathScheme, nil, nil)); err != nil { - t.Errorf("test in path mode without snapshotter failed: %v", err) - return - } - } - if executionMask&0x8 != 0 { - if err := bt.checkFailure(t, test.Run(true, rawdb.PathScheme, nil, nil)); err != nil { - t.Errorf("test in path mode with snapshotter failed: %v", err) - return + for _, snapshot := range snapshotConf { + for _, dbscheme := range dbschemeConf { + if err := bt.checkFailure(t, test.Run(snapshot, dbscheme, true, nil, nil)); err != nil { + t.Errorf("test with config {snapshotter:%v, scheme:%v} failed: %v", snapshot, dbscheme, err) + return + } } } } diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 04a04fdc28..62aa582c82 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -110,7 +110,7 @@ type btHeaderMarshaling struct { ExcessBlobGas *math.HexOrDecimal64 } -func (t *BlockTest) Run(snapshotter bool, scheme string, tracer *tracing.Hooks, postCheck func(error, *core.BlockChain)) (result error) { +func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *tracing.Hooks, postCheck func(error, *core.BlockChain)) (result error) { config, ok := Forks[t.json.Network] if !ok { return UnsupportedForkError{t.json.Network} @@ -151,7 +151,8 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, tracer *tracing.Hooks, cache.SnapshotWait = true } chain, err := core.NewBlockChain(db, cache, gspec, nil, engine, vm.Config{ - Tracer: tracer, + Tracer: tracer, + EnableWitnessCollection: witness, }, nil, nil) if err != nil { return err diff --git a/trie/secure_trie.go b/trie/secure_trie.go index cfa7f0bddb..3572117e03 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -214,6 +214,11 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte { return t.db.Preimage(common.BytesToHash(shaKey)) } +// Witness returns a set containing all trie nodes that have been accessed. +func (t *StateTrie) Witness() map[string]struct{} { + return t.trie.Witness() +} + // Commit collects all dirty nodes in the trie and replaces them with the // corresponding node hash. All collected nodes (including dirty leaves if // collectLeaf is true) will be encapsulated into a nodeset for return. diff --git a/trie/trie.go b/trie/trie.go index e1a9201108..f44e10b918 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -661,6 +661,18 @@ func (t *Trie) hashRoot() (node, node) { return hashed, cached } +// Witness returns a set containing all trie nodes that have been accessed. +func (t *Trie) Witness() map[string]struct{} { + if len(t.tracer.accessList) == 0 { + return nil + } + witness := make(map[string]struct{}) + for _, node := range t.tracer.accessList { + witness[string(node)] = struct{}{} + } + return witness +} + // Reset drops the referenced root node and cleans all internal state. func (t *Trie) Reset() { t.root = nil diff --git a/trie/verkle.go b/trie/verkle.go index 1ea23186f9..94a5ca9a2c 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -369,3 +369,8 @@ func (t *VerkleTrie) ToDot() string { func (t *VerkleTrie) nodeResolver(path []byte) ([]byte, error) { return t.reader.node(path, common.Hash{}) } + +// Witness returns a set containing all trie nodes that have been accessed. +func (t *VerkleTrie) Witness() map[string]struct{} { + panic("not implemented") +} From 98b5930d2d780fe13895633ffcc6b84c9beb4450 Mon Sep 17 00:00:00 2001 From: maskpp Date: Tue, 25 Jun 2024 20:19:04 +0800 Subject: [PATCH 06/14] core/txpool/blobpool: avoid use *map as parameter. (#30048) --- core/txpool/blobpool/blobpool.go | 2 +- core/txpool/blobpool/evictheap.go | 12 ++++++------ core/txpool/blobpool/evictheap_test.go | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 69423731ee..2fca5c7b3e 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -407,7 +407,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres if p.head.ExcessBlobGas != nil { blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(*p.head.ExcessBlobGas)) } - p.evict = newPriceHeap(basefee, blobfee, &p.index) + p.evict = newPriceHeap(basefee, blobfee, p.index) // Pool initialized, attach the blob limbo to it to track blobs included // recently but not yet finalized diff --git a/core/txpool/blobpool/evictheap.go b/core/txpool/blobpool/evictheap.go index bc4543a352..701eb405cf 100644 --- a/core/txpool/blobpool/evictheap.go +++ b/core/txpool/blobpool/evictheap.go @@ -35,7 +35,7 @@ import ( // The goal of the heap is to decide which account has the worst bottleneck to // evict transactions from. type evictHeap struct { - metas *map[common.Address][]*blobTxMeta // Pointer to the blob pool's index for price retrievals + metas map[common.Address][]*blobTxMeta // Pointer to the blob pool's index for price retrievals basefeeJumps float64 // Pre-calculated absolute dynamic fee jumps for the base fee blobfeeJumps float64 // Pre-calculated absolute dynamic fee jumps for the blob fee @@ -46,7 +46,7 @@ type evictHeap struct { // newPriceHeap creates a new heap of cheapest accounts in the blob pool to evict // from in case of over saturation. -func newPriceHeap(basefee *uint256.Int, blobfee *uint256.Int, index *map[common.Address][]*blobTxMeta) *evictHeap { +func newPriceHeap(basefee *uint256.Int, blobfee *uint256.Int, index map[common.Address][]*blobTxMeta) *evictHeap { heap := &evictHeap{ metas: index, index: make(map[common.Address]int), @@ -54,8 +54,8 @@ func newPriceHeap(basefee *uint256.Int, blobfee *uint256.Int, index *map[common. // Populate the heap in account sort order. Not really needed in practice, // but it makes the heap initialization deterministic and less annoying to // test in unit tests. - addrs := make([]common.Address, 0, len(*index)) - for addr := range *index { + addrs := make([]common.Address, 0, len(index)) + for addr := range index { addrs = append(addrs, addr) } sort.Slice(addrs, func(i, j int) bool { return bytes.Compare(addrs[i][:], addrs[j][:]) < 0 }) @@ -94,8 +94,8 @@ func (h *evictHeap) Len() int { // Less implements sort.Interface as part of heap.Interface, returning which of // the two requested accounts has a cheaper bottleneck. func (h *evictHeap) Less(i, j int) bool { - txsI := (*(h.metas))[h.addrs[i]] - txsJ := (*(h.metas))[h.addrs[j]] + txsI := h.metas[h.addrs[i]] + txsJ := h.metas[h.addrs[j]] lastI := txsI[len(txsI)-1] lastJ := txsJ[len(txsJ)-1] diff --git a/core/txpool/blobpool/evictheap_test.go b/core/txpool/blobpool/evictheap_test.go index 01b136551c..1cf577cb00 100644 --- a/core/txpool/blobpool/evictheap_test.go +++ b/core/txpool/blobpool/evictheap_test.go @@ -37,17 +37,17 @@ func verifyHeapInternals(t *testing.T, evict *evictHeap) { seen := make(map[common.Address]struct{}) for i, addr := range evict.addrs { seen[addr] = struct{}{} - if _, ok := (*evict.metas)[addr]; !ok { + if _, ok := evict.metas[addr]; !ok { t.Errorf("heap contains unexpected address at slot %d: %v", i, addr) } } - for addr := range *evict.metas { + for addr := range evict.metas { if _, ok := seen[addr]; !ok { t.Errorf("heap is missing required address %v", addr) } } - if len(evict.addrs) != len(*evict.metas) { - t.Errorf("heap size %d mismatches metadata size %d", len(evict.addrs), len(*evict.metas)) + if len(evict.addrs) != len(evict.metas) { + t.Errorf("heap size %d mismatches metadata size %d", len(evict.addrs), len(evict.metas)) } // Ensure that all accounts are present in the heap order index and no extras have := make([]common.Address, len(evict.index)) @@ -159,7 +159,7 @@ func TestPriceHeapSorting(t *testing.T) { }} } // Create a price heap and check the pop order - priceheap := newPriceHeap(uint256.NewInt(tt.basefee), uint256.NewInt(tt.blobfee), &index) + priceheap := newPriceHeap(uint256.NewInt(tt.basefee), uint256.NewInt(tt.blobfee), index) verifyHeapInternals(t, priceheap) for j := 0; j < len(tt.order); j++ { @@ -218,7 +218,7 @@ func benchmarkPriceHeapReinit(b *testing.B, datacap uint64) { }} } // Create a price heap and reinit it over and over - heap := newPriceHeap(uint256.NewInt(rand.Uint64()), uint256.NewInt(rand.Uint64()), &index) + heap := newPriceHeap(uint256.NewInt(rand.Uint64()), uint256.NewInt(rand.Uint64()), index) basefees := make([]*uint256.Int, b.N) blobfees := make([]*uint256.Int, b.N) @@ -278,7 +278,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) { }} } // Create a price heap and overflow it over and over - evict := newPriceHeap(uint256.NewInt(rand.Uint64()), uint256.NewInt(rand.Uint64()), &index) + evict := newPriceHeap(uint256.NewInt(rand.Uint64()), uint256.NewInt(rand.Uint64()), index) var ( addrs = make([]common.Address, b.N) metas = make([]*blobTxMeta, b.N) From 9298d2db884c4e3f9474880e3dcfd080ef9eacfa Mon Sep 17 00:00:00 2001 From: maskpp Date: Tue, 25 Jun 2024 21:45:33 +0800 Subject: [PATCH 07/14] trie/trienode: remove unnecessary check in Summary (#30047) --- trie/trienode/node.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/trie/trienode/node.go b/trie/trienode/node.go index fa7bebe4f9..09f355f3b5 100644 --- a/trie/trienode/node.go +++ b/trie/trienode/node.go @@ -139,16 +139,14 @@ func (set *NodeSet) Size() (int, int) { func (set *NodeSet) Summary() string { var out = new(strings.Builder) fmt.Fprintf(out, "nodeset owner: %v\n", set.Owner) - if set.Nodes != nil { - for path, n := range set.Nodes { - // Deletion - if n.IsDeleted() { - fmt.Fprintf(out, " [-]: %x\n", path) - continue - } - // Insertion or update - fmt.Fprintf(out, " [+/*]: %x -> %v \n", path, n.Hash) + for path, n := range set.Nodes { + // Deletion + if n.IsDeleted() { + fmt.Fprintf(out, " [-]: %x\n", path) + continue } + // Insertion or update + fmt.Fprintf(out, " [+/*]: %x -> %v \n", path, n.Hash) } for _, n := range set.Leaves { fmt.Fprintf(out, "[leaf]: %v\n", n) From 269e80b07ed8b8300372c44ebb5d7ac2328bf84c Mon Sep 17 00:00:00 2001 From: Halimao <1065621723@qq.com> Date: Thu, 27 Jun 2024 17:29:50 +0800 Subject: [PATCH 08/14] eth/tracers,trie: remove unnecessary check (#30071) --- eth/tracers/live/supply.go | 10 ++++------ eth/tracers/native/call_flat.go | 16 +++++++--------- trie/verkle.go | 6 ++---- 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/eth/tracers/live/supply.go b/eth/tracers/live/supply.go index 936ffb9472..96f7059454 100644 --- a/eth/tracers/live/supply.go +++ b/eth/tracers/live/supply.go @@ -204,12 +204,10 @@ func (s *supply) internalTxsHandler(call *supplyTxCallstack) { s.delta.Burn.Misc.Add(s.delta.Burn.Misc, call.burn) } - if len(call.calls) > 0 { - // Recursively handle internal calls - for _, call := range call.calls { - callCopy := call - s.internalTxsHandler(&callCopy) - } + // Recursively handle internal calls + for _, call := range call.calls { + callCopy := call + s.internalTxsHandler(&callCopy) } } diff --git a/eth/tracers/native/call_flat.go b/eth/tracers/native/call_flat.go index ce0fb08114..a47b79f8df 100644 --- a/eth/tracers/native/call_flat.go +++ b/eth/tracers/native/call_flat.go @@ -274,16 +274,14 @@ func flatFromNested(input *callFrame, traceAddress []int, convertErrs bool, ctx } output = append(output, *frame) - if len(input.Calls) > 0 { - for i, childCall := range input.Calls { - childAddr := childTraceAddress(traceAddress, i) - childCallCopy := childCall - flat, err := flatFromNested(&childCallCopy, childAddr, convertErrs, ctx) - if err != nil { - return nil, err - } - output = append(output, flat...) + for i, childCall := range input.Calls { + childAddr := childTraceAddress(traceAddress, i) + childCallCopy := childCall + flat, err := flatFromNested(&childCallCopy, childAddr, convertErrs, ctx) + if err != nil { + return nil, err } + output = append(output, flat...) } return output, nil diff --git a/trie/verkle.go b/trie/verkle.go index 94a5ca9a2c..a457097e95 100644 --- a/trie/verkle.go +++ b/trie/verkle.go @@ -144,10 +144,8 @@ func (t *VerkleTrie) UpdateAccount(addr common.Address, acc *types.StateAccount) // Encode balance in little-endian bytes := acc.Balance.Bytes() - if len(bytes) > 0 { - for i, b := range bytes { - balance[len(bytes)-i-1] = b - } + for i, b := range bytes { + balance[len(bytes)-i-1] = b } values[utils.BalanceLeafKey] = balance[:] From 045b9718d512d24af71331cbe4d8e3e1a7afba12 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 27 Jun 2024 20:30:39 +0800 Subject: [PATCH 09/14] trie: relocate state execution logic into pathdb package (#29861) --- trie/secure_trie.go | 29 ++++- trie/trie_reader.go | 21 ---- trie/triestate/state.go | 212 +-------------------------------- triedb/database.go | 9 +- triedb/database/database.go | 16 +-- triedb/pathdb/database.go | 4 +- triedb/pathdb/database_test.go | 64 ++++++---- triedb/pathdb/disklayer.go | 4 +- triedb/pathdb/execute.go | 186 +++++++++++++++++++++++++++++ triedb/pathdb/testutils.go | 159 ------------------------- 10 files changed, 259 insertions(+), 445 deletions(-) create mode 100644 triedb/pathdb/execute.go delete mode 100644 triedb/pathdb/testutils.go diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 3572117e03..6eb6defa45 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -24,6 +24,16 @@ import ( "github.com/ethereum/go-ethereum/triedb/database" ) +// preimageStore wraps the methods of a backing store for reading and writing +// trie node preimages. +type preimageStore interface { + // Preimage retrieves the preimage of the specified hash. + Preimage(hash common.Hash) []byte + + // InsertPreimage commits a set of preimages along with their hashes. + InsertPreimage(preimages map[common.Hash][]byte) +} + // SecureTrie is the old name of StateTrie. // Deprecated: use StateTrie. type SecureTrie = StateTrie @@ -52,6 +62,7 @@ func NewSecure(stateRoot common.Hash, owner common.Hash, root common.Hash, db da type StateTrie struct { trie Trie db database.Database + preimages preimageStore hashKeyBuf [common.HashLength]byte secKeyCache map[string][]byte secKeyCacheOwner *StateTrie // Pointer to self, replace the key cache on mismatch @@ -70,7 +81,14 @@ func NewStateTrie(id *ID, db database.Database) (*StateTrie, error) { if err != nil { return nil, err } - return &StateTrie{trie: *trie, db: db}, nil + tr := &StateTrie{trie: *trie, db: db} + + // link the preimage store if it's supported + preimages, ok := db.(preimageStore) + if ok { + tr.preimages = preimages + } + return tr, nil } // MustGet returns the value for key stored in the trie. @@ -211,7 +229,10 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte { if key, ok := t.getSecKeyCache()[string(shaKey)]; ok { return key } - return t.db.Preimage(common.BytesToHash(shaKey)) + if t.preimages == nil { + return nil + } + return t.preimages.Preimage(common.BytesToHash(shaKey)) } // Witness returns a set containing all trie nodes that have been accessed. @@ -233,7 +254,9 @@ func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { for hk, key := range t.secKeyCache { preimages[common.BytesToHash([]byte(hk))] = key } - t.db.InsertPreimage(preimages) + if t.preimages != nil { + t.preimages.InsertPreimage(preimages) + } t.secKeyCache = make(map[string][]byte) } // Commit the trie and return its modified nodeset. diff --git a/trie/trie_reader.go b/trie/trie_reader.go index 42bc4316fe..adbf43d287 100644 --- a/trie/trie_reader.go +++ b/trie/trie_reader.go @@ -20,7 +20,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/trie/triestate" "github.com/ethereum/go-ethereum/triedb/database" ) @@ -72,23 +71,3 @@ func (r *trieReader) node(path []byte, hash common.Hash) ([]byte, error) { } return blob, nil } - -// MerkleLoader implements triestate.TrieLoader for constructing tries. -type MerkleLoader struct { - db database.Database -} - -// NewMerkleLoader creates the merkle trie loader. -func NewMerkleLoader(db database.Database) *MerkleLoader { - return &MerkleLoader{db: db} -} - -// OpenTrie opens the main account trie. -func (l *MerkleLoader) OpenTrie(root common.Hash) (triestate.Trie, error) { - return New(TrieID(root), l.db) -} - -// OpenStorageTrie opens the storage trie of an account. -func (l *MerkleLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) { - return New(StorageTrieID(stateRoot, addrHash, root), l.db) -} diff --git a/trie/triestate/state.go b/trie/triestate/state.go index 7508da5d60..62a9043873 100644 --- a/trie/triestate/state.go +++ b/trie/triestate/state.go @@ -16,43 +16,7 @@ package triestate -import ( - "errors" - "fmt" - "sync" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/rlp" - "github.com/ethereum/go-ethereum/trie/trienode" -) - -// Trie is an Ethereum state trie, can be implemented by Ethereum Merkle Patricia -// tree or Verkle tree. -type Trie interface { - // Get returns the value for key stored in the trie. - Get(key []byte) ([]byte, error) - - // Update associates key with value in the trie. - Update(key, value []byte) error - - // Delete removes any existing value for key from the trie. - Delete(key []byte) error - - // Commit the trie and returns a set of dirty nodes generated along with - // the new root hash. - Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) -} - -// TrieLoader wraps functions to load tries. -type TrieLoader interface { - // OpenTrie opens the main account trie. - OpenTrie(root common.Hash) (Trie, error) - - // OpenStorageTrie opens the storage trie of an account. - OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error) -} +import "github.com/ethereum/go-ethereum/common" // Set represents a collection of mutated states during a state transition. // The value refers to the original content of state before the transition @@ -87,177 +51,3 @@ func (s *Set) Size() common.StorageSize { } return s.size } - -// context wraps all fields for executing state diffs. -type context struct { - prevRoot common.Hash - postRoot common.Hash - accounts map[common.Address][]byte - storages map[common.Address]map[common.Hash][]byte - accountTrie Trie - nodes *trienode.MergedNodeSet -} - -// Apply traverses the provided state diffs, apply them in the associated -// post-state and return the generated dirty trie nodes. The state can be -// loaded via the provided trie loader. -func Apply(prevRoot common.Hash, postRoot common.Hash, accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte, loader TrieLoader) (map[common.Hash]map[string]*trienode.Node, error) { - tr, err := loader.OpenTrie(postRoot) - if err != nil { - return nil, err - } - ctx := &context{ - prevRoot: prevRoot, - postRoot: postRoot, - accounts: accounts, - storages: storages, - accountTrie: tr, - nodes: trienode.NewMergedNodeSet(), - } - for addr, account := range accounts { - var err error - if len(account) == 0 { - err = deleteAccount(ctx, loader, addr) - } else { - err = updateAccount(ctx, loader, addr) - } - if err != nil { - return nil, fmt.Errorf("failed to revert state, err: %w", err) - } - } - root, result := tr.Commit(false) - if root != prevRoot { - return nil, fmt.Errorf("failed to revert state, want %#x, got %#x", prevRoot, root) - } - if err := ctx.nodes.Merge(result); err != nil { - return nil, err - } - return ctx.nodes.Flatten(), nil -} - -// updateAccount the account was present in prev-state, and may or may not -// existent in post-state. Apply the reverse diff and verify if the storage -// root matches the one in prev-state account. -func updateAccount(ctx *context, loader TrieLoader, addr common.Address) error { - // The account was present in prev-state, decode it from the - // 'slim-rlp' format bytes. - h := newHasher() - defer h.release() - - addrHash := h.hash(addr.Bytes()) - prev, err := types.FullAccount(ctx.accounts[addr]) - if err != nil { - return err - } - // The account may or may not existent in post-state, try to - // load it and decode if it's found. - blob, err := ctx.accountTrie.Get(addrHash.Bytes()) - if err != nil { - return err - } - post := types.NewEmptyStateAccount() - if len(blob) != 0 { - if err := rlp.DecodeBytes(blob, &post); err != nil { - return err - } - } - // Apply all storage changes into the post-state storage trie. - st, err := loader.OpenStorageTrie(ctx.postRoot, addrHash, post.Root) - if err != nil { - return err - } - for key, val := range ctx.storages[addr] { - var err error - if len(val) == 0 { - err = st.Delete(key.Bytes()) - } else { - err = st.Update(key.Bytes(), val) - } - if err != nil { - return err - } - } - root, result := st.Commit(false) - if root != prev.Root { - return errors.New("failed to reset storage trie") - } - // The returned set can be nil if storage trie is not changed - // at all. - if result != nil { - if err := ctx.nodes.Merge(result); err != nil { - return err - } - } - // Write the prev-state account into the main trie - full, err := rlp.EncodeToBytes(prev) - if err != nil { - return err - } - return ctx.accountTrie.Update(addrHash.Bytes(), full) -} - -// deleteAccount the account was not present in prev-state, and is expected -// to be existent in post-state. Apply the reverse diff and verify if the -// account and storage is wiped out correctly. -func deleteAccount(ctx *context, loader TrieLoader, addr common.Address) error { - // The account must be existent in post-state, load the account. - h := newHasher() - defer h.release() - - addrHash := h.hash(addr.Bytes()) - blob, err := ctx.accountTrie.Get(addrHash.Bytes()) - if err != nil { - return err - } - if len(blob) == 0 { - return fmt.Errorf("account is non-existent %#x", addrHash) - } - var post types.StateAccount - if err := rlp.DecodeBytes(blob, &post); err != nil { - return err - } - st, err := loader.OpenStorageTrie(ctx.postRoot, addrHash, post.Root) - if err != nil { - return err - } - for key, val := range ctx.storages[addr] { - if len(val) != 0 { - return errors.New("expect storage deletion") - } - if err := st.Delete(key.Bytes()); err != nil { - return err - } - } - root, result := st.Commit(false) - if root != types.EmptyRootHash { - return errors.New("failed to clear storage trie") - } - // The returned set can be nil if storage trie is not changed - // at all. - if result != nil { - if err := ctx.nodes.Merge(result); err != nil { - return err - } - } - // Delete the post-state account from the main trie. - return ctx.accountTrie.Delete(addrHash.Bytes()) -} - -// hasher is used to compute the sha256 hash of the provided data. -type hasher struct{ sha crypto.KeccakState } - -var hasherPool = sync.Pool{ - New: func() interface{} { return &hasher{sha: crypto.NewKeccakState()} }, -} - -func newHasher() *hasher { - return hasherPool.Get().(*hasher) -} - -func (h *hasher) hash(data []byte) common.Hash { - return crypto.HashData(h.sha, data) -} - -func (h *hasher) release() { - hasherPool.Put(h) -} diff --git a/triedb/database.go b/triedb/database.go index ef757e7f5b..91386a9dbc 100644 --- a/triedb/database.go +++ b/triedb/database.go @@ -264,14 +264,7 @@ func (db *Database) Recover(target common.Hash) error { if !ok { return errors.New("not supported") } - var loader triestate.TrieLoader - if db.config.IsVerkle { - // TODO define verkle loader - log.Crit("Verkle loader is not defined") - } else { - loader = trie.NewMerkleLoader(db) - } - return pdb.Recover(target, loader) + return pdb.Recover(target) } // Recoverable returns the indicator if the specified state is enabled to be diff --git a/triedb/database/database.go b/triedb/database/database.go index f11c7e9bbd..9bd5da08d1 100644 --- a/triedb/database/database.go +++ b/triedb/database/database.go @@ -16,9 +16,7 @@ package database -import ( - "github.com/ethereum/go-ethereum/common" -) +import "github.com/ethereum/go-ethereum/common" // Reader wraps the Node method of a backing trie reader. type Reader interface { @@ -31,20 +29,8 @@ type Reader interface { Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) } -// PreimageStore wraps the methods of a backing store for reading and writing -// trie node preimages. -type PreimageStore interface { - // Preimage retrieves the preimage of the specified hash. - Preimage(hash common.Hash) []byte - - // InsertPreimage commits a set of preimages along with their hashes. - InsertPreimage(preimages map[common.Hash][]byte) -} - // Database wraps the methods of a backing trie store. type Database interface { - PreimageStore - // Reader returns a node reader associated with the specific state. // An error will be returned if the specified state is not available. Reader(stateRoot common.Hash) (Reader, error) diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index bd6aeaa6ab..450c3a8f4f 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -345,7 +345,7 @@ func (db *Database) Enable(root common.Hash) error { // Recover rollbacks the database to a specified historical point. // The state is supported as the rollback destination only if it's // canonical state and the corresponding trie histories are existent. -func (db *Database) Recover(root common.Hash, loader triestate.TrieLoader) error { +func (db *Database) Recover(root common.Hash) error { db.lock.Lock() defer db.lock.Unlock() @@ -371,7 +371,7 @@ func (db *Database) Recover(root common.Hash, loader triestate.TrieLoader) error if err != nil { return err } - dl, err = dl.revert(h, loader) + dl, err = dl.revert(h) if err != nil { return err } diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index 04c8af415f..8870bfe95b 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -29,24 +29,31 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/internal/testrand" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "github.com/ethereum/go-ethereum/trie/triestate" "github.com/holiman/uint256" ) -func updateTrie(addrHash common.Hash, root common.Hash, dirties, cleans map[common.Hash][]byte) (common.Hash, *trienode.NodeSet) { - h, err := newTestHasher(addrHash, root, cleans) +func updateTrie(db *Database, stateRoot common.Hash, addrHash common.Hash, root common.Hash, dirties map[common.Hash][]byte) (common.Hash, *trienode.NodeSet) { + var id *trie.ID + if addrHash == (common.Hash{}) { + id = trie.StateTrieID(stateRoot) + } else { + id = trie.StorageTrieID(stateRoot, addrHash, root) + } + tr, err := trie.New(id, db) if err != nil { - panic(fmt.Errorf("failed to create hasher, err: %w", err)) + panic(fmt.Errorf("failed to load trie, err: %w", err)) } for key, val := range dirties { if len(val) == 0 { - h.Delete(key.Bytes()) + tr.Delete(key.Bytes()) } else { - h.Update(key.Bytes(), val) + tr.Update(key.Bytes(), val) } } - return h.Commit(false) + return tr.Commit(false) } func generateAccount(storageRoot common.Hash) types.StateAccount { @@ -66,6 +73,7 @@ const ( ) type genctx struct { + stateRoot common.Hash accounts map[common.Hash][]byte storages map[common.Hash]map[common.Hash][]byte accountOrigin map[common.Address][]byte @@ -73,8 +81,9 @@ type genctx struct { nodes *trienode.MergedNodeSet } -func newCtx() *genctx { +func newCtx(stateRoot common.Hash) *genctx { return &genctx{ + stateRoot: stateRoot, accounts: make(map[common.Hash][]byte), storages: make(map[common.Hash]map[common.Hash][]byte), accountOrigin: make(map[common.Address][]byte), @@ -151,7 +160,7 @@ func (t *tester) generateStorage(ctx *genctx, addr common.Address) common.Hash { storage[hash] = v origin[hash] = nil } - root, set := updateTrie(addrHash, types.EmptyRootHash, storage, nil) + root, set := updateTrie(t.db, ctx.stateRoot, addrHash, types.EmptyRootHash, storage) ctx.storages[addrHash] = storage ctx.storageOrigin[addr] = origin @@ -180,7 +189,7 @@ func (t *tester) mutateStorage(ctx *genctx, addr common.Address, root common.Has storage[hash] = v origin[hash] = nil } - root, set := updateTrie(crypto.Keccak256Hash(addr.Bytes()), root, storage, t.storages[addrHash]) + root, set := updateTrie(t.db, ctx.stateRoot, crypto.Keccak256Hash(addr.Bytes()), root, storage) ctx.storages[addrHash] = storage ctx.storageOrigin[addr] = origin @@ -198,7 +207,7 @@ func (t *tester) clearStorage(ctx *genctx, addr common.Address, root common.Hash origin[hash] = val storage[hash] = nil } - root, set := updateTrie(addrHash, root, storage, t.storages[addrHash]) + root, set := updateTrie(t.db, ctx.stateRoot, addrHash, root, storage) if root != types.EmptyRootHash { panic("failed to clear storage trie") } @@ -210,7 +219,7 @@ func (t *tester) clearStorage(ctx *genctx, addr common.Address, root common.Hash func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNodeSet, *triestate.Set) { var ( - ctx = newCtx() + ctx = newCtx(parent) dirties = make(map[common.Hash]struct{}) ) for i := 0; i < 20; i++ { @@ -271,7 +280,7 @@ func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNode ctx.accountOrigin[addr] = account } } - root, set := updateTrie(common.Hash{}, parent, ctx.accounts, t.accounts) + root, set := updateTrie(t.db, parent, common.Hash{}, parent, ctx.accounts) ctx.nodes.Merge(set) // Save state snapshot before commit @@ -297,6 +306,9 @@ func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNode t.storages[addrHash][sHash] = slot } } + if len(t.storages[addrHash]) == 0 { + delete(t.storages, addrHash) + } } return root, ctx.nodes, triestate.New(ctx.accountOrigin, ctx.storageOrigin) } @@ -310,25 +322,31 @@ func (t *tester) lastHash() common.Hash { } func (t *tester) verifyState(root common.Hash) error { - reader, err := t.db.Reader(root) + tr, err := trie.New(trie.StateTrieID(root), t.db) if err != nil { return err } - _, err = reader.Node(common.Hash{}, nil, root) - if err != nil { - return errors.New("root node is not available") - } for addrHash, account := range t.snapAccounts[root] { - path := crypto.Keccak256(addrHash.Bytes()) - blob, err := reader.Node(common.Hash{}, path, crypto.Keccak256Hash(account)) + blob, err := tr.Get(addrHash.Bytes()) if err != nil || !bytes.Equal(blob, account) { return fmt.Errorf("account is mismatched: %w", err) } } for addrHash, slots := range t.snapStorages[root] { + blob := t.snapAccounts[root][addrHash] + if len(blob) == 0 { + return fmt.Errorf("account %x is missing", addrHash) + } + account := new(types.StateAccount) + if err := rlp.DecodeBytes(blob, account); err != nil { + return err + } + storageIt, err := trie.New(trie.StorageTrieID(root, addrHash, account.Root), t.db) + if err != nil { + return err + } for hash, slot := range slots { - path := crypto.Keccak256(hash.Bytes()) - blob, err := reader.Node(addrHash, path, crypto.Keccak256Hash(slot)) + blob, err := storageIt.Get(hash.Bytes()) if err != nil || !bytes.Equal(blob, slot) { return fmt.Errorf("slot is mismatched: %w", err) } @@ -395,13 +413,11 @@ func TestDatabaseRollback(t *testing.T) { } // Revert database from top to bottom for i := tester.bottomIndex(); i >= 0; i-- { - root := tester.roots[i] parent := types.EmptyRootHash if i > 0 { parent = tester.roots[i-1] } - loader := newHashLoader(tester.snapAccounts[root], tester.snapStorages[root]) - if err := tester.db.Recover(parent, loader); err != nil { + if err := tester.db.Recover(parent); err != nil { t.Fatalf("Failed to revert db, err: %v", err) } if i > 0 { diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index 964ad2ef77..e538a79280 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -219,7 +219,7 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { } // revert applies the given state history and return a reverted disk layer. -func (dl *diskLayer) revert(h *history, loader triestate.TrieLoader) (*diskLayer, error) { +func (dl *diskLayer) revert(h *history) (*diskLayer, error) { if h.meta.root != dl.rootHash() { return nil, errUnexpectedHistory } @@ -229,7 +229,7 @@ func (dl *diskLayer) revert(h *history, loader triestate.TrieLoader) (*diskLayer // Apply the reverse state changes upon the current state. This must // be done before holding the lock in order to access state in "this" // layer. - nodes, err := triestate.Apply(h.meta.parent, h.meta.root, h.accounts, h.storages, loader) + nodes, err := apply(dl.db, h.meta.parent, h.meta.root, h.accounts, h.storages) if err != nil { return nil, err } diff --git a/triedb/pathdb/execute.go b/triedb/pathdb/execute.go new file mode 100644 index 0000000000..9074e4debf --- /dev/null +++ b/triedb/pathdb/execute.go @@ -0,0 +1,186 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package pathdb + +import ( + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/triedb/database" +) + +// context wraps all fields for executing state diffs. +type context struct { + prevRoot common.Hash + postRoot common.Hash + accounts map[common.Address][]byte + storages map[common.Address]map[common.Hash][]byte + nodes *trienode.MergedNodeSet + + // TODO (rjl493456442) abstract out the state hasher + // for supporting verkle tree. + accountTrie *trie.Trie +} + +// apply processes the given state diffs, updates the corresponding post-state +// and returns the trie nodes that have been modified. +func apply(db database.Database, prevRoot common.Hash, postRoot common.Hash, accounts map[common.Address][]byte, storages map[common.Address]map[common.Hash][]byte) (map[common.Hash]map[string]*trienode.Node, error) { + tr, err := trie.New(trie.TrieID(postRoot), db) + if err != nil { + return nil, err + } + ctx := &context{ + prevRoot: prevRoot, + postRoot: postRoot, + accounts: accounts, + storages: storages, + accountTrie: tr, + nodes: trienode.NewMergedNodeSet(), + } + for addr, account := range accounts { + var err error + if len(account) == 0 { + err = deleteAccount(ctx, db, addr) + } else { + err = updateAccount(ctx, db, addr) + } + if err != nil { + return nil, fmt.Errorf("failed to revert state, err: %w", err) + } + } + root, result := tr.Commit(false) + if root != prevRoot { + return nil, fmt.Errorf("failed to revert state, want %#x, got %#x", prevRoot, root) + } + if err := ctx.nodes.Merge(result); err != nil { + return nil, err + } + return ctx.nodes.Flatten(), nil +} + +// updateAccount the account was present in prev-state, and may or may not +// existent in post-state. Apply the reverse diff and verify if the storage +// root matches the one in prev-state account. +func updateAccount(ctx *context, db database.Database, addr common.Address) error { + // The account was present in prev-state, decode it from the + // 'slim-rlp' format bytes. + h := newHasher() + defer h.release() + + addrHash := h.hash(addr.Bytes()) + prev, err := types.FullAccount(ctx.accounts[addr]) + if err != nil { + return err + } + // The account may or may not existent in post-state, try to + // load it and decode if it's found. + blob, err := ctx.accountTrie.Get(addrHash.Bytes()) + if err != nil { + return err + } + post := types.NewEmptyStateAccount() + if len(blob) != 0 { + if err := rlp.DecodeBytes(blob, &post); err != nil { + return err + } + } + // Apply all storage changes into the post-state storage trie. + st, err := trie.New(trie.StorageTrieID(ctx.postRoot, addrHash, post.Root), db) + if err != nil { + return err + } + for key, val := range ctx.storages[addr] { + var err error + if len(val) == 0 { + err = st.Delete(key.Bytes()) + } else { + err = st.Update(key.Bytes(), val) + } + if err != nil { + return err + } + } + root, result := st.Commit(false) + if root != prev.Root { + return errors.New("failed to reset storage trie") + } + // The returned set can be nil if storage trie is not changed + // at all. + if result != nil { + if err := ctx.nodes.Merge(result); err != nil { + return err + } + } + // Write the prev-state account into the main trie + full, err := rlp.EncodeToBytes(prev) + if err != nil { + return err + } + return ctx.accountTrie.Update(addrHash.Bytes(), full) +} + +// deleteAccount the account was not present in prev-state, and is expected +// to be existent in post-state. Apply the reverse diff and verify if the +// account and storage is wiped out correctly. +func deleteAccount(ctx *context, db database.Database, addr common.Address) error { + // The account must be existent in post-state, load the account. + h := newHasher() + defer h.release() + + addrHash := h.hash(addr.Bytes()) + blob, err := ctx.accountTrie.Get(addrHash.Bytes()) + if err != nil { + return err + } + if len(blob) == 0 { + return fmt.Errorf("account is non-existent %#x", addrHash) + } + var post types.StateAccount + if err := rlp.DecodeBytes(blob, &post); err != nil { + return err + } + st, err := trie.New(trie.StorageTrieID(ctx.postRoot, addrHash, post.Root), db) + if err != nil { + return err + } + for key, val := range ctx.storages[addr] { + if len(val) != 0 { + return errors.New("expect storage deletion") + } + if err := st.Delete(key.Bytes()); err != nil { + return err + } + } + root, result := st.Commit(false) + if root != types.EmptyRootHash { + return errors.New("failed to clear storage trie") + } + // The returned set can be nil if storage trie is not changed + // at all. + if result != nil { + if err := ctx.nodes.Merge(result); err != nil { + return err + } + } + // Delete the post-state account from the main trie. + return ctx.accountTrie.Delete(addrHash.Bytes()) +} diff --git a/triedb/pathdb/testutils.go b/triedb/pathdb/testutils.go deleted file mode 100644 index af832bc59c..0000000000 --- a/triedb/pathdb/testutils.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package pathdb - -import ( - "bytes" - "fmt" - "slices" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/trie/trienode" - "github.com/ethereum/go-ethereum/trie/triestate" -) - -// testHasher is a test utility for computing root hash of a batch of state -// elements. The hash algorithm is to sort all the elements in lexicographical -// order, concat the key and value in turn, and perform hash calculation on -// the concatenated bytes. Except the root hash, a nodeset will be returned -// once Commit is called, which contains all the changes made to hasher. -type testHasher struct { - owner common.Hash // owner identifier - root common.Hash // original root - dirties map[common.Hash][]byte // dirty states - cleans map[common.Hash][]byte // clean states -} - -// newTestHasher constructs a hasher object with provided states. -func newTestHasher(owner common.Hash, root common.Hash, cleans map[common.Hash][]byte) (*testHasher, error) { - if cleans == nil { - cleans = make(map[common.Hash][]byte) - } - if got, _ := hash(cleans); got != root { - return nil, fmt.Errorf("state root mismatched, want: %x, got: %x", root, got) - } - return &testHasher{ - owner: owner, - root: root, - dirties: make(map[common.Hash][]byte), - cleans: cleans, - }, nil -} - -// Get returns the value for key stored in the trie. -func (h *testHasher) Get(key []byte) ([]byte, error) { - hash := common.BytesToHash(key) - val, ok := h.dirties[hash] - if ok { - return val, nil - } - return h.cleans[hash], nil -} - -// Update associates key with value in the trie. -func (h *testHasher) Update(key, value []byte) error { - h.dirties[common.BytesToHash(key)] = common.CopyBytes(value) - return nil -} - -// Delete removes any existing value for key from the trie. -func (h *testHasher) Delete(key []byte) error { - h.dirties[common.BytesToHash(key)] = nil - return nil -} - -// Commit computes the new hash of the states and returns the set with all -// state changes. -func (h *testHasher) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { - var ( - nodes = make(map[common.Hash][]byte) - set = trienode.NewNodeSet(h.owner) - ) - for hash, val := range h.cleans { - nodes[hash] = val - } - for hash, val := range h.dirties { - nodes[hash] = val - if bytes.Equal(val, h.cleans[hash]) { - continue - } - // Utilize the hash of the state key as the node path to mitigate - // potential collisions within the path. - path := crypto.Keccak256(hash.Bytes()) - if len(val) == 0 { - set.AddNode(path, trienode.NewDeleted()) - } else { - set.AddNode(path, trienode.New(crypto.Keccak256Hash(val), val)) - } - } - root, blob := hash(nodes) - - // Include the dirty root node as well. - if root != types.EmptyRootHash && root != h.root { - set.AddNode(nil, trienode.New(root, blob)) - } - if root == types.EmptyRootHash && h.root != types.EmptyRootHash { - set.AddNode(nil, trienode.NewDeleted()) - } - return root, set -} - -// hash performs the hash computation upon the provided states. -func hash(states map[common.Hash][]byte) (common.Hash, []byte) { - var hs []common.Hash - for hash := range states { - hs = append(hs, hash) - } - slices.SortFunc(hs, common.Hash.Cmp) - - var input []byte - for _, hash := range hs { - if len(states[hash]) == 0 { - continue - } - input = append(input, hash.Bytes()...) - input = append(input, states[hash]...) - } - if len(input) == 0 { - return types.EmptyRootHash, nil - } - return crypto.Keccak256Hash(input), input -} - -type hashLoader struct { - accounts map[common.Hash][]byte - storages map[common.Hash]map[common.Hash][]byte -} - -func newHashLoader(accounts map[common.Hash][]byte, storages map[common.Hash]map[common.Hash][]byte) *hashLoader { - return &hashLoader{ - accounts: accounts, - storages: storages, - } -} - -// OpenTrie opens the main account trie. -func (l *hashLoader) OpenTrie(root common.Hash) (triestate.Trie, error) { - return newTestHasher(common.Hash{}, root, l.accounts) -} - -// OpenStorageTrie opens the storage trie of an account. -func (l *hashLoader) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (triestate.Trie, error) { - return newTestHasher(addrHash, root, l.storages[addrHash]) -} From 19c3c1e20560ff85205dbad9c5c0c698d1854bf6 Mon Sep 17 00:00:00 2001 From: lilasxie Date: Fri, 28 Jun 2024 21:15:54 +0800 Subject: [PATCH 10/14] triedb/pathdb: fix flaky test in pathdb (#29901) --- triedb/pathdb/database_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index 8870bfe95b..f667944784 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -121,7 +121,7 @@ func newTester(t *testing.T, historyLimit uint64) *tester { snapStorages: make(map[common.Hash]map[common.Hash]map[common.Hash][]byte), } ) - for i := 0; i < 8; i++ { + for i := 0; i < 12; i++ { var parent = types.EmptyRootHash if len(obj.roots) != 0 { parent = obj.roots[len(obj.roots)-1] From 36d67be41be59fc92118a6a8cdebdf52e1037eb3 Mon Sep 17 00:00:00 2001 From: maskpp Date: Fri, 28 Jun 2024 21:51:27 +0800 Subject: [PATCH 11/14] core/txpool/blobpool: improve newPriceHeap function (#30050) Co-authored-by: Felix Lange --- core/txpool/blobpool/evictheap.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/core/txpool/blobpool/evictheap.go b/core/txpool/blobpool/evictheap.go index 701eb405cf..5e285e6c53 100644 --- a/core/txpool/blobpool/evictheap.go +++ b/core/txpool/blobpool/evictheap.go @@ -17,13 +17,13 @@ package blobpool import ( - "bytes" "container/heap" "math" - "sort" + "slices" "github.com/ethereum/go-ethereum/common" "github.com/holiman/uint256" + "golang.org/x/exp/maps" ) // evictHeap is a helper data structure to keep track of the cheapest bottleneck @@ -49,20 +49,15 @@ type evictHeap struct { func newPriceHeap(basefee *uint256.Int, blobfee *uint256.Int, index map[common.Address][]*blobTxMeta) *evictHeap { heap := &evictHeap{ metas: index, - index: make(map[common.Address]int), + index: make(map[common.Address]int, len(index)), } // Populate the heap in account sort order. Not really needed in practice, // but it makes the heap initialization deterministic and less annoying to // test in unit tests. - addrs := make([]common.Address, 0, len(index)) - for addr := range index { - addrs = append(addrs, addr) - } - sort.Slice(addrs, func(i, j int) bool { return bytes.Compare(addrs[i][:], addrs[j][:]) < 0 }) - - for _, addr := range addrs { - heap.index[addr] = len(heap.addrs) - heap.addrs = append(heap.addrs, addr) + heap.addrs = maps.Keys(index) + slices.SortFunc(heap.addrs, common.Address.Cmp) + for i, addr := range heap.addrs { + heap.index[addr] = i } heap.reinit(basefee, blobfee, true) return heap From 4939c253411c3a02fff84df6f9506f998b9762de Mon Sep 17 00:00:00 2001 From: maskpp Date: Sat, 29 Jun 2024 00:05:57 +0800 Subject: [PATCH 12/14] cmd/evm/internal/t8ntool: log writeTraceResult error message (#30038) --- cmd/evm/internal/t8ntool/execution.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 3c09229e1c..9537d6d974 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -306,7 +306,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, if tracer.Hooks.OnTxEnd != nil { tracer.Hooks.OnTxEnd(receipt, nil) } - writeTraceResult(tracer, traceOutput) + if err = writeTraceResult(tracer, traceOutput); err != nil { + log.Warn("Error writing tracer output", "err", err) + } } } From 06f1d077d382b457ba28497d1a7bcfb3d0ea5645 Mon Sep 17 00:00:00 2001 From: gitglorythegreat Date: Sat, 29 Jun 2024 00:08:31 +0800 Subject: [PATCH 13/14] all: replace division with right shift if possible (#29911) --- cmd/evm/internal/t8ntool/execution.go | 4 ++-- consensus/ethash/consensus.go | 10 ++-------- core/types/transaction_signing.go | 4 ++-- core/vm/contracts.go | 13 +++++-------- 4 files changed, 11 insertions(+), 20 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 9537d6d974..a4c5f6efcb 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -325,7 +325,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, var ( blockReward = big.NewInt(miningReward) minerReward = new(big.Int).Set(blockReward) - perOmmer = new(big.Int).Div(blockReward, big.NewInt(32)) + perOmmer = new(big.Int).Rsh(blockReward, 5) ) for _, ommer := range pre.Env.Ommers { // Add 1/32th for each ommer included @@ -334,7 +334,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, reward := big.NewInt(8) reward.Sub(reward, new(big.Int).SetUint64(ommer.Delta)) reward.Mul(reward, blockReward) - reward.Div(reward, big.NewInt(8)) + reward.Rsh(reward, 3) statedb.AddBalance(ommer.Address, uint256.MustFromBig(reward), tracing.BalanceIncreaseRewardMineUncle) } statedb.AddBalance(pre.Env.Coinbase, uint256.MustFromBig(minerReward), tracing.BalanceIncreaseRewardMineBlock) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index b5e2754c2d..0bd1a56bce 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -562,12 +562,6 @@ func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) { return hash } -// Some weird constants to avoid constant memory allocs for them. -var ( - u256_8 = uint256.NewInt(8) - u256_32 = uint256.NewInt(32) -) - // accumulateRewards credits the coinbase of the given block with the mining // reward. The total reward consists of the static block reward and rewards for // included uncles. The coinbase of each uncle block is also rewarded. @@ -589,10 +583,10 @@ func accumulateRewards(config *params.ChainConfig, stateDB *state.StateDB, heade r.AddUint64(uNum, 8) r.Sub(r, hNum) r.Mul(r, blockReward) - r.Div(r, u256_8) + r.Rsh(r, 3) stateDB.AddBalance(uncle.Coinbase, r, tracing.BalanceIncreaseRewardMineUncle) - r.Div(blockReward, u256_32) + r.Rsh(blockReward, 5) reward.Add(reward, r) } stateDB.AddBalance(header.Coinbase, reward, tracing.BalanceIncreaseRewardMineBlock) diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 2ae38661f3..339fee6f97 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -572,6 +572,6 @@ func deriveChainId(v *big.Int) *big.Int { } return new(big.Int).SetUint64((v - 35) / 2) } - v = new(big.Int).Sub(v, big.NewInt(35)) - return v.Div(v, big.NewInt(2)) + v.Sub(v, big.NewInt(35)) + return v.Rsh(v, 1) } diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 8e0f846775..dd71a9729f 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -296,10 +296,7 @@ type bigModExp struct { var ( big1 = big.NewInt(1) big3 = big.NewInt(3) - big4 = big.NewInt(4) big7 = big.NewInt(7) - big8 = big.NewInt(8) - big16 = big.NewInt(16) big20 = big.NewInt(20) big32 = big.NewInt(32) big64 = big.NewInt(64) @@ -325,13 +322,13 @@ func modexpMultComplexity(x *big.Int) *big.Int { case x.Cmp(big1024) <= 0: // (x ** 2 // 4 ) + ( 96 * x - 3072) x = new(big.Int).Add( - new(big.Int).Div(new(big.Int).Mul(x, x), big4), + new(big.Int).Rsh(new(big.Int).Mul(x, x), 2), new(big.Int).Sub(new(big.Int).Mul(big96, x), big3072), ) default: // (x ** 2 // 16) + (480 * x - 199680) x = new(big.Int).Add( - new(big.Int).Div(new(big.Int).Mul(x, x), big16), + new(big.Int).Rsh(new(big.Int).Mul(x, x), 4), new(big.Int).Sub(new(big.Int).Mul(big480, x), big199680), ) } @@ -369,7 +366,7 @@ func (c *bigModExp) RequiredGas(input []byte) uint64 { adjExpLen := new(big.Int) if expLen.Cmp(big32) > 0 { adjExpLen.Sub(expLen, big32) - adjExpLen.Mul(big8, adjExpLen) + adjExpLen.Lsh(adjExpLen, 3) } adjExpLen.Add(adjExpLen, big.NewInt(int64(msb))) // Calculate the gas cost of the operation @@ -383,8 +380,8 @@ func (c *bigModExp) RequiredGas(input []byte) uint64 { // ceiling(x/8)^2 // //where is x is max(length_of_MODULUS, length_of_BASE) - gas = gas.Add(gas, big7) - gas = gas.Div(gas, big8) + gas.Add(gas, big7) + gas.Rsh(gas, 3) gas.Mul(gas, gas) gas.Mul(gas, math.BigMax(adjExpLen, big1)) From 7cfff30ba3a67de767a9b2a7405b91f120873d10 Mon Sep 17 00:00:00 2001 From: Ceyhun Onur Date: Fri, 28 Jun 2024 21:37:58 +0300 Subject: [PATCH 14/14] rpc: truncate call error data logs (#30028) Co-authored-by: Felix Lange --- rpc/handler.go | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/rpc/handler.go b/rpc/handler.go index 7b8f64aa7b..f23b544b58 100644 --- a/rpc/handler.go +++ b/rpc/handler.go @@ -17,8 +17,11 @@ package rpc import ( + "bytes" "context" "encoding/json" + "errors" + "fmt" "reflect" "strconv" "strings" @@ -468,16 +471,16 @@ func (h *handler) handleCallMsg(ctx *callProc, msg *jsonrpcMessage) *jsonrpcMess case msg.isCall(): resp := h.handleCall(ctx, msg) - var ctx []interface{} - ctx = append(ctx, "reqid", idForLog{msg.ID}, "duration", time.Since(start)) + var logctx []any + logctx = append(logctx, "reqid", idForLog{msg.ID}, "duration", time.Since(start)) if resp.Error != nil { - ctx = append(ctx, "err", resp.Error.Message) + logctx = append(logctx, "err", resp.Error.Message) if resp.Error.Data != nil { - ctx = append(ctx, "errdata", resp.Error.Data) + logctx = append(logctx, "errdata", formatErrorData(resp.Error.Data)) } - h.log.Warn("Served "+msg.Method, ctx...) + h.log.Warn("Served "+msg.Method, logctx...) } else { - h.log.Debug("Served "+msg.Method, ctx...) + h.log.Debug("Served "+msg.Method, logctx...) } return resp @@ -591,3 +594,33 @@ func (id idForLog) String() string { } return string(id.RawMessage) } + +var errTruncatedOutput = errors.New("truncated output") + +type limitedBuffer struct { + output []byte + limit int +} + +func (buf *limitedBuffer) Write(data []byte) (int, error) { + avail := max(buf.limit, len(buf.output)) + if len(data) < avail { + buf.output = append(buf.output, data...) + return len(data), nil + } + buf.output = append(buf.output, data[:avail]...) + return avail, errTruncatedOutput +} + +func formatErrorData(v any) string { + buf := limitedBuffer{limit: 1024} + err := json.NewEncoder(&buf).Encode(v) + switch { + case err == nil: + return string(bytes.TrimRight(buf.output, "\n")) + case errors.Is(err, errTruncatedOutput): + return fmt.Sprintf("%s... (truncated)", buf.output) + default: + return fmt.Sprintf("bad error data (err=%v)", err) + } +}