From 8c1a36dad355dd5b67b24d6ce7cb86c1a4cb7340 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 28 Nov 2024 11:21:31 +0800 Subject: [PATCH 01/17] core/state/snapshot: handle legacy journal (#30802) This workaround is meant to minimize the possibility for snapshot generation once the geth node upgrades to new version (specifically #30752 ) In #30752, the journal format in state snapshot is modified by removing the destruct set. Therefore, the existing old format (version = 0) will be discarded and all in-memory layers will be lost. Unfortunately, the lost in-memory layers can't be recovered by some other approaches, and the entire state snapshot will be regenerated (it will last about 2.5 hours). This pull request introduces a workaround to adopt the legacy journal if the destruct set contained is empty. Since self-destruction has been deprecated following the cancun fork, the destruct set is expected to be nil for layers above the fork block. However, an exception occurs during contract deployment: pre-funded accounts may self-destruct, causing accounts with non-zero balances to be removed from the state. For example, https://etherscan.io/tx/0xa087333d83f0cd63b96bdafb686462e1622ce25f40bd499e03efb1051f31fe49). For nodes with a fully synced state, the legacy journal is likely compatible with the updated definition, eliminating the need for regeneration. Unfortunately, nodes performing a full sync of historical chain segments or encountering pre-funded account deletions may face incompatibilities, leading to automatic snapshot regeneration. --- core/state/snapshot/journal.go | 47 +++++++++++++++++++++++++++++---- core/state/snapshot/snapshot.go | 2 +- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index 61552c73fe..e4b396b990 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -33,9 +33,11 @@ import ( "github.com/ethereum/go-ethereum/triedb" ) -// 0: initial version -// 1: destruct flag in diff layer is removed -const journalVersion uint64 = 1 +const ( + journalV0 uint64 = 0 // initial version + journalV1 uint64 = 1 // current version, with destruct flag (in diff layers) removed + journalCurrentVersion = journalV1 +) // journalGenerator is a disk layer entry containing the generator progress marker. type journalGenerator struct { @@ -50,6 +52,11 @@ type journalGenerator struct { Storage uint64 } +// journalDestruct is an account deletion entry in a diffLayer's disk journal. +type journalDestruct struct { + Hash common.Hash +} + // journalAccount is an account entry in a diffLayer's disk journal. type journalAccount struct { Hash common.Hash @@ -285,8 +292,8 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { log.Warn("Failed to resolve the journal version", "error", err) return errors.New("failed to resolve journal version") } - if version != journalVersion { - log.Warn("Discarded the snapshot journal with wrong version", "required", journalVersion, "got", version) + if version != journalV0 && version != journalCurrentVersion { + log.Warn("Discarded journal with wrong version", "required", journalCurrentVersion, "got", version) return errors.New("wrong journal version") } // Secondly, resolve the disk layer root, ensure it's continuous @@ -316,6 +323,36 @@ func iterateJournal(db ethdb.KeyValueReader, callback journalCallback) error { } return fmt.Errorf("load diff root: %v", err) } + // If a legacy journal is detected, decode the destruct set from the stream. + // The destruct set has been deprecated. If the journal contains non-empty + // destruct set, then it is deemed incompatible. + // + // Since self-destruction has been deprecated following the cancun fork, + // the destruct set is expected to be nil for layers above the fork block. + // However, an exception occurs during contract deployment: pre-funded accounts + // may self-destruct, causing accounts with non-zero balances to be removed + // from the state. For example, + // https://etherscan.io/tx/0xa087333d83f0cd63b96bdafb686462e1622ce25f40bd499e03efb1051f31fe49). + // + // For nodes with a fully synced state, the legacy journal is likely compatible + // with the updated definition, eliminating the need for regeneration. Unfortunately, + // nodes performing a full sync of historical chain segments or encountering + // pre-funded account deletions may face incompatibilities, leading to automatic + // snapshot regeneration. + // + // This approach minimizes snapshot regeneration for Geth nodes upgrading from a + // legacy version that are already synced. The workaround can be safely removed + // after the next hard fork. + if version == journalV0 { + var destructs []journalDestruct + if err := r.Decode(&destructs); err != nil { + return fmt.Errorf("load diff destructs: %v", err) + } + if len(destructs) > 0 { + log.Warn("Incompatible legacy journal detected", "version", journalV0) + return fmt.Errorf("incompatible legacy journal detected") + } + } if err := r.Decode(&accounts); err != nil { return fmt.Errorf("load diff accounts: %v", err) } diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 7466f12351..f0f6296433 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -664,7 +664,7 @@ func (t *Tree) Journal(root common.Hash) (common.Hash, error) { // Firstly write out the metadata of journal journal := new(bytes.Buffer) - if err := rlp.Encode(journal, journalVersion); err != nil { + if err := rlp.Encode(journal, journalCurrentVersion); err != nil { return common.Hash{}, err } diskroot := t.diskRoot() From 2406305175593bb8547f5711912dc41c7a26f0f7 Mon Sep 17 00:00:00 2001 From: Ng Wei Han <47109095+weiihann@users.noreply.github.com> Date: Thu, 28 Nov 2024 16:17:58 +0700 Subject: [PATCH 02/17] trie: combine validation loops in VerifyRangeProof (#30823) Small optimization. It's guaranteed that `len(keys)` == `len(values)`, so we can combine the checks in a single loop rather than 2 separate loops. --- trie/proof.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/trie/proof.go b/trie/proof.go index 40836fba33..2e527348bf 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -486,13 +486,11 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, valu return false, fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)) } // Ensure the received batch is monotonic increasing and contains no deletions - for i := 0; i < len(keys)-1; i++ { - if bytes.Compare(keys[i], keys[i+1]) >= 0 { + for i := 0; i < len(keys); i++ { + if i < len(keys)-1 && bytes.Compare(keys[i], keys[i+1]) >= 0 { return false, errors.New("range is not monotonically increasing") } - } - for _, value := range values { - if len(value) == 0 { + if len(values[i]) == 0 { return false, errors.New("range contains deletion") } } From db8eed860d85b78b9b85b767ede98e47a4b11ffc Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 28 Nov 2024 11:48:50 +0100 Subject: [PATCH 03/17] all: exclude empty outputs in requests commitment (#30670) Implements changes from these spec PRs: - https://github.com/ethereum/EIPs/pull/8989 - https://github.com/ethereum/execution-apis/pull/599 --- beacon/engine/types.go | 21 +-------- cmd/evm/internal/t8ntool/execution.go | 16 +++---- core/chain_makers.go | 15 +++---- core/genesis.go | 4 +- core/state_processor.go | 43 ++++++++++--------- core/types/hashes.go | 3 ++ eth/tracers/internal/tracetest/supply_test.go | 2 +- ...h_getBlockReceipts-block-with-blob-tx.json | 2 +- ...eceipts-block-with-contract-create-tx.json | 2 +- ...ockReceipts-block-with-dynamic-fee-tx.json | 2 +- ...ts-block-with-legacy-contract-call-tx.json | 4 +- ...eceipts-block-with-legacy-transfer-tx.json | 2 +- .../eth_getBlockReceipts-tag-latest.json | 2 +- .../eth_getTransactionReceipt-blob-tx.json | 2 +- ...TransactionReceipt-create-contract-tx.json | 2 +- ...eipt-create-contract-with-access-list.json | 2 +- ...ansactionReceipt-dynamic-tx-with-logs.json | 2 +- ...TransactionReceipt-normal-transfer-tx.json | 2 +- .../eth_getTransactionReceipt-with-logs.json | 4 +- miner/worker.go | 13 +++--- 20 files changed, 62 insertions(+), 83 deletions(-) diff --git a/beacon/engine/types.go b/beacon/engine/types.go index 9f41aa04ca..984090ef89 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -265,15 +265,7 @@ func ExecutableDataToBlockNoHash(data ExecutableData, versionedHashes []common.H var requestsHash *common.Hash if requests != nil { - // Put back request type byte. - typedRequests := make([][]byte, len(requests)) - for i, reqdata := range requests { - typedReqdata := make([]byte, len(reqdata)+1) - typedReqdata[0] = byte(i) - copy(typedReqdata[1:], reqdata) - typedRequests[i] = typedReqdata - } - h := types.CalcRequestsHash(typedRequests) + h := types.CalcRequestsHash(requests) requestsHash = &h } @@ -343,20 +335,11 @@ func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types. } } - // Remove type byte in requests. - var plainRequests [][]byte - if requests != nil { - plainRequests = make([][]byte, len(requests)) - for i, reqdata := range requests { - plainRequests[i] = reqdata[1:] - } - } - return &ExecutionPayloadEnvelope{ ExecutionPayload: data, BlockValue: fees, BlobsBundle: &bundle, - Requests: plainRequests, + Requests: requests, Override: false, } } diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index c0623a69bf..6034b0d33a 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -366,21 +366,19 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, // Gather the execution-layer triggered requests. var requests [][]byte if chainConfig.IsPrague(vmContext.BlockNumber, vmContext.Time) { - // EIP-6110 deposits + requests = [][]byte{} + // EIP-6110 var allLogs []*types.Log for _, receipt := range receipts { allLogs = append(allLogs, receipt.Logs...) } - depositRequests, err := core.ParseDepositLogs(allLogs, chainConfig) - if err != nil { + if err := core.ParseDepositLogs(&requests, allLogs, chainConfig); err != nil { return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not parse requests logs: %v", err)) } - requests = append(requests, depositRequests) - - // EIP-7002 withdrawals - requests = append(requests, core.ProcessWithdrawalQueue(evm)) - // EIP-7251 consolidations - requests = append(requests, core.ProcessConsolidationQueue(evm)) + // EIP-7002 + core.ProcessWithdrawalQueue(&requests, evm) + // EIP-7251 + core.ProcessConsolidationQueue(&requests, evm) } // Commit block diff --git a/core/chain_makers.go b/core/chain_makers.go index e679a9e557..97c785d52b 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -349,25 +349,22 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse var requests [][]byte if config.IsPrague(b.header.Number, b.header.Time) { + requests = [][]byte{} // EIP-6110 deposits var blockLogs []*types.Log for _, r := range b.receipts { blockLogs = append(blockLogs, r.Logs...) } - depositRequests, err := ParseDepositLogs(blockLogs, config) - if err != nil { + if err := ParseDepositLogs(&requests, blockLogs, config); err != nil { panic(fmt.Sprintf("failed to parse deposit log: %v", err)) } - requests = append(requests, depositRequests) // create EVM for system calls blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase) evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{}) - // EIP-7002 withdrawals - withdrawalRequests := ProcessWithdrawalQueue(evm) - requests = append(requests, withdrawalRequests) - // EIP-7251 consolidations - consolidationRequests := ProcessConsolidationQueue(evm) - requests = append(requests, consolidationRequests) + // EIP-7002 + ProcessWithdrawalQueue(&requests, evm) + // EIP-7251 + ProcessConsolidationQueue(&requests, evm) } if requests != nil { reqHash := types.CalcRequestsHash(requests) diff --git a/core/genesis.go b/core/genesis.go index eff92084eb..85ef049ba6 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -472,9 +472,7 @@ func (g *Genesis) toBlockWithRoot(root common.Hash) *types.Block { } } if conf.IsPrague(num, g.Timestamp) { - emptyRequests := [][]byte{{0x00}, {0x01}, {0x02}} - rhash := types.CalcRequestsHash(emptyRequests) - head.RequestsHash = &rhash + head.RequestsHash = &types.EmptyRequestsHash } } return types.NewBlock(head, &types.Body{Withdrawals: withdrawals}, nil, trie.NewStackTrie(nil)) diff --git a/core/state_processor.go b/core/state_processor.go index 1703377111..9472ab638b 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -106,18 +106,15 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg // Read requests if Prague is enabled. var requests [][]byte if p.config.IsPrague(block.Number(), block.Time()) { - // EIP-6110 deposits - depositRequests, err := ParseDepositLogs(allLogs, p.config) - if err != nil { + requests = [][]byte{} + // EIP-6110 + if err := ParseDepositLogs(&requests, allLogs, p.config); err != nil { return nil, err } - requests = append(requests, depositRequests) - // EIP-7002 withdrawals - withdrawalRequests := ProcessWithdrawalQueue(evm) - requests = append(requests, withdrawalRequests) - // EIP-7251 consolidations - consolidationRequests := ProcessConsolidationQueue(evm) - requests = append(requests, consolidationRequests) + // EIP-7002 + ProcessWithdrawalQueue(&requests, evm) + // EIP-7251 + ProcessConsolidationQueue(&requests, evm) } // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) @@ -271,17 +268,17 @@ func ProcessParentBlockHash(prevHash common.Hash, evm *vm.EVM) { // ProcessWithdrawalQueue calls the EIP-7002 withdrawal queue contract. // It returns the opaque request data returned by the contract. -func ProcessWithdrawalQueue(evm *vm.EVM) []byte { - return processRequestsSystemCall(evm, 0x01, params.WithdrawalQueueAddress) +func ProcessWithdrawalQueue(requests *[][]byte, evm *vm.EVM) { + processRequestsSystemCall(requests, evm, 0x01, params.WithdrawalQueueAddress) } // ProcessConsolidationQueue calls the EIP-7251 consolidation queue contract. // It returns the opaque request data returned by the contract. -func ProcessConsolidationQueue(evm *vm.EVM) []byte { - return processRequestsSystemCall(evm, 0x02, params.ConsolidationQueueAddress) +func ProcessConsolidationQueue(requests *[][]byte, evm *vm.EVM) { + processRequestsSystemCall(requests, evm, 0x02, params.ConsolidationQueueAddress) } -func processRequestsSystemCall(evm *vm.EVM, requestType byte, addr common.Address) []byte { +func processRequestsSystemCall(requests *[][]byte, evm *vm.EVM, requestType byte, addr common.Address) { if tracer := evm.Config.Tracer; tracer != nil { if tracer.OnSystemCallStart != nil { tracer.OnSystemCallStart() @@ -302,26 +299,32 @@ func processRequestsSystemCall(evm *vm.EVM, requestType byte, addr common.Addres evm.StateDB.AddAddressToAccessList(addr) ret, _, _ := evm.Call(vm.AccountRef(msg.From), *msg.To, msg.Data, 30_000_000, common.U2560) evm.StateDB.Finalise(true) + if len(ret) == 0 { + return // skip empty output + } - // Create withdrawals requestsData with prefix 0x01 + // Append prefixed requestsData to the requests list. requestsData := make([]byte, len(ret)+1) requestsData[0] = requestType copy(requestsData[1:], ret) - return requestsData + *requests = append(*requests, requestsData) } // ParseDepositLogs extracts the EIP-6110 deposit values from logs emitted by // BeaconDepositContract. -func ParseDepositLogs(logs []*types.Log, config *params.ChainConfig) ([]byte, error) { +func ParseDepositLogs(requests *[][]byte, logs []*types.Log, config *params.ChainConfig) error { deposits := make([]byte, 1) // note: first byte is 0x00 (== deposit request type) for _, log := range logs { if log.Address == config.DepositContractAddress { request, err := types.DepositLogToRequest(log.Data) if err != nil { - return nil, fmt.Errorf("unable to parse deposit data: %v", err) + return fmt.Errorf("unable to parse deposit data: %v", err) } deposits = append(deposits, request...) } } - return deposits, nil + if len(deposits) > 1 { + *requests = append(*requests, deposits) + } + return nil } diff --git a/core/types/hashes.go b/core/types/hashes.go index 43e9130fd1..55506d63d0 100644 --- a/core/types/hashes.go +++ b/core/types/hashes.go @@ -41,6 +41,9 @@ var ( // EmptyWithdrawalsHash is the known hash of the empty withdrawal set. EmptyWithdrawalsHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + // EmptyRequestsHash is the known hash of an empty request set, sha256(""). + EmptyRequestsHash = common.HexToHash("e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + // EmptyVerkleHash is the known hash of an empty verkle trie. EmptyVerkleHash = common.Hash{} ) diff --git a/eth/tracers/internal/tracetest/supply_test.go b/eth/tracers/internal/tracetest/supply_test.go index 2391add91b..6f06b7c0d5 100644 --- a/eth/tracers/internal/tracetest/supply_test.go +++ b/eth/tracers/internal/tracetest/supply_test.go @@ -86,7 +86,7 @@ func TestSupplyOmittedFields(t *testing.T) { expected := supplyInfo{ Number: 0, - Hash: common.HexToHash("0xc02ee8ee5b54a40e43f0fa827d431e1bd4f217e941790dda10b2521d1925a20b"), + Hash: common.HexToHash("0x3055fc27d6b4a08eb07033a0d1ee755a4b2988086f28a6189eac1b507525eeb1"), ParentHash: common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000"), } actual := out[expected.Number] diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json index d315353ec6..df2d9349fb 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-blob-tx.json @@ -2,7 +2,7 @@ { "blobGasPrice": "0x1", "blobGasUsed": "0x20000", - "blockHash": "0x11e6318d77a45c01f89f76b56d36c6936c5250f4e2bd238cb7b09df73cf0cb7d", + "blockHash": "0x17124e31fb075a301b1d7d4135683b0a09fe4e6d453c54e2e734d5ee00744a49", "blockNumber": "0x6", "contractAddress": null, "cumulativeGasUsed": "0x5208", diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-contract-create-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-contract-create-tx.json index f2e5ced2be..453e0abe8a 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-contract-create-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-contract-create-tx.json @@ -1,6 +1,6 @@ [ { - "blockHash": "0x5526cd89bc188f20fd5e9bb50d8054dc5a51a81a74ed07eacf36a4a8b10de4b1", + "blockHash": "0xb3e447c77374fd285964cba692e96b1673a88a959726826b5b6e2dca15472b0a", "blockNumber": "0x2", "contractAddress": "0xae9bea628c4ce503dcfd7e305cab4e29e7476592", "cumulativeGasUsed": "0xcf50", diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json index 71afd85e54..b01400e605 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-dynamic-fee-tx.json @@ -1,6 +1,6 @@ [ { - "blockHash": "0x3e946aa9e252873af511b257d9d89a1bcafa54ce7c6a6442f8407ecdf81e288d", + "blockHash": "0x102e50de30318ee99a03a09db74387e79cad3165bf6840cc84249806a2a302f3", "blockNumber": "0x4", "contractAddress": null, "cumulativeGasUsed": "0x538d", diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-contract-call-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-contract-call-tx.json index f089ac45ae..ae964b3d3b 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-contract-call-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-contract-call-tx.json @@ -1,6 +1,6 @@ [ { - "blockHash": "0xc281d4299fc4e8ce5bba7ecb8deb50f5403d604c806b36aa887dfe2ff84c064f", + "blockHash": "0xcc6225bf39327429a3d869af71182d619a354155187d0b5a8ecd6a9309cffcaa", "blockNumber": "0x3", "contractAddress": null, "cumulativeGasUsed": "0x5e28", @@ -19,7 +19,7 @@ "blockNumber": "0x3", "transactionHash": "0xeaf3921cbf03ba45bad4e6ab807b196ce3b2a0b5bacc355b6272fa96b11b4287", "transactionIndex": "0x0", - "blockHash": "0xc281d4299fc4e8ce5bba7ecb8deb50f5403d604c806b36aa887dfe2ff84c064f", + "blockHash": "0xcc6225bf39327429a3d869af71182d619a354155187d0b5a8ecd6a9309cffcaa", "logIndex": "0x0", "removed": false } diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-transfer-tx.json b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-transfer-tx.json index 8b69dddd66..7af5016079 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-transfer-tx.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-block-with-legacy-transfer-tx.json @@ -1,6 +1,6 @@ [ { - "blockHash": "0xda50d57d8802553b00bb8e4d777bd5c4114086941119ca04edb15429f4818ed9", + "blockHash": "0xe9bd1d8c303b1af5c704b9d78e62c54a34af47e0db04ac1389a5ef74a619b9da", "blockNumber": "0x1", "contractAddress": null, "cumulativeGasUsed": "0x5208", diff --git a/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json b/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json index d315353ec6..df2d9349fb 100644 --- a/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json +++ b/internal/ethapi/testdata/eth_getBlockReceipts-tag-latest.json @@ -2,7 +2,7 @@ { "blobGasPrice": "0x1", "blobGasUsed": "0x20000", - "blockHash": "0x11e6318d77a45c01f89f76b56d36c6936c5250f4e2bd238cb7b09df73cf0cb7d", + "blockHash": "0x17124e31fb075a301b1d7d4135683b0a09fe4e6d453c54e2e734d5ee00744a49", "blockNumber": "0x6", "contractAddress": null, "cumulativeGasUsed": "0x5208", diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json b/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json index 5debbd4447..8e0669d10a 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-blob-tx.json @@ -1,7 +1,7 @@ { "blobGasPrice": "0x1", "blobGasUsed": "0x20000", - "blockHash": "0x11e6318d77a45c01f89f76b56d36c6936c5250f4e2bd238cb7b09df73cf0cb7d", + "blockHash": "0x17124e31fb075a301b1d7d4135683b0a09fe4e6d453c54e2e734d5ee00744a49", "blockNumber": "0x6", "contractAddress": null, "cumulativeGasUsed": "0x5208", diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-tx.json b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-tx.json index 8cf2ead10f..a9391cb578 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-tx.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-tx.json @@ -1,5 +1,5 @@ { - "blockHash": "0x5526cd89bc188f20fd5e9bb50d8054dc5a51a81a74ed07eacf36a4a8b10de4b1", + "blockHash": "0xb3e447c77374fd285964cba692e96b1673a88a959726826b5b6e2dca15472b0a", "blockNumber": "0x2", "contractAddress": "0xae9bea628c4ce503dcfd7e305cab4e29e7476592", "cumulativeGasUsed": "0xcf50", diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json index 34c318faca..49c06aad62 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-create-contract-with-access-list.json @@ -1,5 +1,5 @@ { - "blockHash": "0xa04ad6be58c45fe483991b89416572bc50426b0de44b769757e95c704250f874", + "blockHash": "0x53bffe54375c0a31fe7bc0db7455db7d48278234c2400efa4d40d1c57cbe868d", "blockNumber": "0x5", "contractAddress": "0xfdaa97661a584d977b4d3abb5370766ff5b86a18", "cumulativeGasUsed": "0xe01c", diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json b/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json index 9f023ed6e3..13bd7bd12c 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-dynamic-tx-with-logs.json @@ -1,5 +1,5 @@ { - "blockHash": "0x3e946aa9e252873af511b257d9d89a1bcafa54ce7c6a6442f8407ecdf81e288d", + "blockHash": "0x102e50de30318ee99a03a09db74387e79cad3165bf6840cc84249806a2a302f3", "blockNumber": "0x4", "contractAddress": null, "cumulativeGasUsed": "0x538d", diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-normal-transfer-tx.json b/internal/ethapi/testdata/eth_getTransactionReceipt-normal-transfer-tx.json index f180a21977..779223af98 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-normal-transfer-tx.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-normal-transfer-tx.json @@ -1,5 +1,5 @@ { - "blockHash": "0xda50d57d8802553b00bb8e4d777bd5c4114086941119ca04edb15429f4818ed9", + "blockHash": "0xe9bd1d8c303b1af5c704b9d78e62c54a34af47e0db04ac1389a5ef74a619b9da", "blockNumber": "0x1", "contractAddress": null, "cumulativeGasUsed": "0x5208", diff --git a/internal/ethapi/testdata/eth_getTransactionReceipt-with-logs.json b/internal/ethapi/testdata/eth_getTransactionReceipt-with-logs.json index 61aed4b7bd..1a1edb7887 100644 --- a/internal/ethapi/testdata/eth_getTransactionReceipt-with-logs.json +++ b/internal/ethapi/testdata/eth_getTransactionReceipt-with-logs.json @@ -1,5 +1,5 @@ { - "blockHash": "0xc281d4299fc4e8ce5bba7ecb8deb50f5403d604c806b36aa887dfe2ff84c064f", + "blockHash": "0xcc6225bf39327429a3d869af71182d619a354155187d0b5a8ecd6a9309cffcaa", "blockNumber": "0x3", "contractAddress": null, "cumulativeGasUsed": "0x5e28", @@ -18,7 +18,7 @@ "blockNumber": "0x3", "transactionHash": "0xeaf3921cbf03ba45bad4e6ab807b196ce3b2a0b5bacc355b6272fa96b11b4287", "transactionIndex": "0x0", - "blockHash": "0xc281d4299fc4e8ce5bba7ecb8deb50f5403d604c806b36aa887dfe2ff84c064f", + "blockHash": "0xcc6225bf39327429a3d869af71182d619a354155187d0b5a8ecd6a9309cffcaa", "logIndex": "0x0", "removed": false } diff --git a/miner/worker.go b/miner/worker.go index aeb6cfcdc2..b5aa080025 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -121,18 +121,15 @@ func (miner *Miner) generateWork(params *generateParams, witness bool) *newPaylo // Collect consensus-layer requests if Prague is enabled. var requests [][]byte if miner.chainConfig.IsPrague(work.header.Number, work.header.Time) { + requests = [][]byte{} // EIP-6110 deposits - depositRequests, err := core.ParseDepositLogs(allLogs, miner.chainConfig) - if err != nil { + if err := core.ParseDepositLogs(&requests, allLogs, miner.chainConfig); err != nil { return &newPayloadResult{err: err} } - requests = append(requests, depositRequests) - // EIP-7002 withdrawals - withdrawalRequests := core.ProcessWithdrawalQueue(work.evm) - requests = append(requests, withdrawalRequests) + // EIP-7002 + core.ProcessWithdrawalQueue(&requests, work.evm) // EIP-7251 consolidations - consolidationRequests := core.ProcessConsolidationQueue(work.evm) - requests = append(requests, consolidationRequests) + core.ProcessConsolidationQueue(&requests, work.evm) } if requests != nil { reqHash := types.CalcRequestsHash(requests) From 53f66c1b03d5880fa807e088c865b31d144fabd0 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 28 Nov 2024 14:37:36 +0100 Subject: [PATCH 04/17] cmd/bootnode: remove bootnode utility (#30813) Since we don't really support custom networks anymore, we don't need the bootnode utility. In case a discovery-only node is wanted, it can still be run using cmd/devp2p. --- README.md | 14 +-- build/ci.go | 5 -- cmd/bootnode/main.go | 209 ------------------------------------------- 3 files changed, 9 insertions(+), 219 deletions(-) delete mode 100644 cmd/bootnode/main.go diff --git a/README.md b/README.md index 182464c639..27842008aa 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,6 @@ directory. | `clef` | Stand-alone signing tool, which can be used as a backend signer for `geth`. | | `devp2p` | Utilities to interact with nodes on the networking layer, without running a full blockchain. | | `abigen` | Source code generator to convert Ethereum contract definitions into easy-to-use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/developers/dapp-developer/native-bindings) page for details. | -| `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | | `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | | `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://ethereum.org/en/developers/docs/data-structures-and-encoding/rlp)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | @@ -270,8 +269,14 @@ start a bootstrap node that others can use to find each other in your network an the internet. The clean way is to configure and run a dedicated bootnode: ```shell -$ bootnode --genkey=boot.key -$ bootnode --nodekey=boot.key +# Use the devp2p tool to create a node file. +# The devp2p tool is also part of the 'alltools' distribution bundle. +$ devp2p key generate node1.key +# file node1.key is created. +$ devp2p key to-enr -ip 10.2.3.4 -udp 30303 -tcp 30303 node1.key +# Prints the ENR for use in --bootnode flag of other nodes. +# Note this method requires knowing the IP/ports ahead of time. +$ geth --nodekey=node1.key ``` With the bootnode online, it will display an [`enode` URL](https://ethereum.org/en/developers/docs/networking-layer/network-addresses/#enode) @@ -279,8 +284,7 @@ that other nodes can use to connect to it and exchange peer information. Make su replace the displayed IP address information (most probably `[::]`) with your externally accessible IP to get the actual `enode` URL. -*Note: You could also use a full-fledged `geth` node as a bootnode, but it's the less -recommended way.* +*Note: You could previously use the `bootnode` utility to start a stripped down version of geth. This is not possible anymore.* #### Starting up your member nodes diff --git a/build/ci.go b/build/ci.go index e3115af46d..fd51b90d17 100644 --- a/build/ci.go +++ b/build/ci.go @@ -74,7 +74,6 @@ var ( allToolsArchiveFiles = []string{ "COPYING", executablePath("abigen"), - executablePath("bootnode"), executablePath("evm"), executablePath("geth"), executablePath("rlpdump"), @@ -87,10 +86,6 @@ var ( BinaryName: "abigen", Description: "Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages.", }, - { - BinaryName: "bootnode", - Description: "Ethereum bootnode.", - }, { BinaryName: "evm", Description: "Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode.", diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go deleted file mode 100644 index 350b85df1e..0000000000 --- a/cmd/bootnode/main.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -// bootnode runs a bootstrap node for the Ethereum Discovery Protocol. -package main - -import ( - "crypto/ecdsa" - "flag" - "fmt" - "net" - "os" - "time" - - "github.com/ethereum/go-ethereum/cmd/utils" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/discover" - "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/nat" - "github.com/ethereum/go-ethereum/p2p/netutil" -) - -func main() { - var ( - listenAddr = flag.String("addr", ":30301", "listen address") - genKey = flag.String("genkey", "", "generate a node key") - writeAddr = flag.Bool("writeaddress", false, "write out the node's public key and quit") - nodeKeyFile = flag.String("nodekey", "", "private key filename") - nodeKeyHex = flag.String("nodekeyhex", "", "private key as hex (for testing)") - natdesc = flag.String("nat", "none", "port mapping mechanism (any|none|upnp|pmp|pmp:|extip:)") - netrestrict = flag.String("netrestrict", "", "restrict network communication to the given IP networks (CIDR masks)") - runv5 = flag.Bool("v5", false, "run a v5 topic discovery bootnode") - verbosity = flag.Int("verbosity", 3, "log verbosity (0-5)") - vmodule = flag.String("vmodule", "", "log verbosity pattern") - - nodeKey *ecdsa.PrivateKey - err error - ) - flag.Parse() - - glogger := log.NewGlogHandler(log.NewTerminalHandler(os.Stderr, false)) - slogVerbosity := log.FromLegacyLevel(*verbosity) - glogger.Verbosity(slogVerbosity) - glogger.Vmodule(*vmodule) - log.SetDefault(log.NewLogger(glogger)) - - natm, err := nat.Parse(*natdesc) - if err != nil { - utils.Fatalf("-nat: %v", err) - } - switch { - case *genKey != "": - nodeKey, err = crypto.GenerateKey() - if err != nil { - utils.Fatalf("could not generate key: %v", err) - } - if err = crypto.SaveECDSA(*genKey, nodeKey); err != nil { - utils.Fatalf("%v", err) - } - if !*writeAddr { - return - } - case *nodeKeyFile == "" && *nodeKeyHex == "": - utils.Fatalf("Use -nodekey or -nodekeyhex to specify a private key") - case *nodeKeyFile != "" && *nodeKeyHex != "": - utils.Fatalf("Options -nodekey and -nodekeyhex are mutually exclusive") - case *nodeKeyFile != "": - if nodeKey, err = crypto.LoadECDSA(*nodeKeyFile); err != nil { - utils.Fatalf("-nodekey: %v", err) - } - case *nodeKeyHex != "": - if nodeKey, err = crypto.HexToECDSA(*nodeKeyHex); err != nil { - utils.Fatalf("-nodekeyhex: %v", err) - } - } - - if *writeAddr { - fmt.Printf("%x\n", crypto.FromECDSAPub(&nodeKey.PublicKey)[1:]) - os.Exit(0) - } - - var restrictList *netutil.Netlist - if *netrestrict != "" { - restrictList, err = netutil.ParseNetlist(*netrestrict) - if err != nil { - utils.Fatalf("-netrestrict: %v", err) - } - } - - addr, err := net.ResolveUDPAddr("udp", *listenAddr) - if err != nil { - utils.Fatalf("-ResolveUDPAddr: %v", err) - } - conn, err := net.ListenUDP("udp", addr) - if err != nil { - utils.Fatalf("-ListenUDP: %v", err) - } - defer conn.Close() - - db, _ := enode.OpenDB("") - ln := enode.NewLocalNode(db, nodeKey) - - listenerAddr := conn.LocalAddr().(*net.UDPAddr) - if natm != nil && !listenerAddr.IP.IsLoopback() { - natAddr := doPortMapping(natm, ln, listenerAddr) - if natAddr != nil { - listenerAddr = natAddr - } - } - - printNotice(&nodeKey.PublicKey, *listenerAddr) - cfg := discover.Config{ - PrivateKey: nodeKey, - NetRestrict: restrictList, - } - if *runv5 { - if _, err := discover.ListenV5(conn, ln, cfg); err != nil { - utils.Fatalf("%v", err) - } - } else { - if _, err := discover.ListenUDP(conn, ln, cfg); err != nil { - utils.Fatalf("%v", err) - } - } - - select {} -} - -func printNotice(nodeKey *ecdsa.PublicKey, addr net.UDPAddr) { - if addr.IP.IsUnspecified() { - addr.IP = net.IP{127, 0, 0, 1} - } - n := enode.NewV4(nodeKey, addr.IP, 0, addr.Port) - fmt.Println(n.URLv4()) - fmt.Println("Note: you're using cmd/bootnode, a developer tool.") - fmt.Println("We recommend using a regular node as bootstrap node for production deployments.") -} - -func doPortMapping(natm nat.Interface, ln *enode.LocalNode, addr *net.UDPAddr) *net.UDPAddr { - const ( - protocol = "udp" - name = "ethereum discovery" - ) - newLogger := func(external int, internal int) log.Logger { - return log.New("proto", protocol, "extport", external, "intport", internal, "interface", natm) - } - - var ( - intport = addr.Port - extaddr = &net.UDPAddr{IP: addr.IP, Port: addr.Port} - mapTimeout = nat.DefaultMapTimeout - log = newLogger(addr.Port, intport) - ) - addMapping := func() { - // Get the external address. - var err error - extaddr.IP, err = natm.ExternalIP() - if err != nil { - log.Debug("Couldn't get external IP", "err", err) - return - } - // Create the mapping. - p, err := natm.AddMapping(protocol, extaddr.Port, intport, name, mapTimeout) - if err != nil { - log.Debug("Couldn't add port mapping", "err", err) - return - } - if p != uint16(extaddr.Port) { - extaddr.Port = int(p) - log = newLogger(extaddr.Port, intport) - log.Info("NAT mapped alternative port") - } else { - log.Info("NAT mapped port") - } - // Update IP/port information of the local node. - ln.SetStaticIP(extaddr.IP) - ln.SetFallbackUDP(extaddr.Port) - } - - // Perform mapping once, synchronously. - log.Info("Attempting port mapping") - addMapping() - - // Refresh the mapping periodically. - go func() { - refresh := time.NewTimer(mapTimeout) - defer refresh.Stop() - for range refresh.C { - addMapping() - refresh.Reset(mapTimeout) - } - }() - - return extaddr -} From c7a8bcecbed7b371f0d6ec97ab1a1ae33aa3ca85 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 28 Nov 2024 18:43:39 +0100 Subject: [PATCH 05/17] core/types: add length check in CalcRequestsHash (#30829) The existing implementation is correct when building and verifying blocks, since we will only collect non-empty requests into the block requests list. But it isn't correct for cases where a requests list containing empty items is sent by the consensus layer on the engine API. We want to ensure that empty requests do not cause a difference in validation there, so the commitment computation should explicitly skip them. --- core/types/block.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/types/block.go b/core/types/block.go index f20fc7d778..ad6c86398d 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -463,9 +463,11 @@ func CalcRequestsHash(requests [][]byte) common.Hash { h1, h2 := sha256.New(), sha256.New() var buf common.Hash for _, item := range requests { - h1.Reset() - h1.Write(item) - h2.Write(h1.Sum(buf[:0])) + if len(item) > 1 { // skip items with only requestType and no data. + h1.Reset() + h1.Write(item) + h2.Write(h1.Sum(buf[:0])) + } } h2.Sum(buf[:0]) return buf From 05148d972c1b0546c0183e7d302c334f5effbb06 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 29 Nov 2024 19:30:45 +0800 Subject: [PATCH 06/17] triedb/pathdb: track flat state changes in pathdb (snapshot integration pt 2) (#30643) This pull request ports some changes from the main state snapshot integration one, specifically introducing the flat state tracking in pathdb. Note, the tracked flat state changes are only held in memory and won't be persisted in the disk. Meanwhile, the correspoding state retrieval in persistent state is also not supported yet. The states management in disk is more complicated and will be implemented in a separate pull request. Part 1: https://github.com/ethereum/go-ethereum/pull/30752 --- triedb/database.go | 11 + triedb/hashdb/database.go | 6 + triedb/pathdb/buffer.go | 36 ++- triedb/pathdb/database.go | 39 ++- triedb/pathdb/database_test.go | 2 +- triedb/pathdb/difflayer.go | 53 ++++ triedb/pathdb/difflayer_test.go | 8 +- triedb/pathdb/disklayer.go | 100 ++++++- triedb/pathdb/journal.go | 16 +- triedb/pathdb/metrics.go | 23 +- triedb/pathdb/nodes.go | 4 +- triedb/pathdb/reader.go | 53 +++- triedb/pathdb/states.go | 393 ++++++++++++++++++++++++++- triedb/pathdb/states_test.go | 453 ++++++++++++++++++++++++++++++++ triedb/states.go | 2 +- 15 files changed, 1149 insertions(+), 50 deletions(-) create mode 100644 triedb/pathdb/states_test.go diff --git a/triedb/database.go b/triedb/database.go index d3eca57b54..b448d7cd07 100644 --- a/triedb/database.go +++ b/triedb/database.go @@ -60,6 +60,10 @@ type backend interface { // An error will be returned if the specified state is not available. NodeReader(root common.Hash) (database.NodeReader, error) + // StateReader returns a reader for accessing flat states within the specified + // state. An error will be returned if the specified state is not available. + StateReader(root common.Hash) (database.StateReader, error) + // Initialized returns an indicator if the state data is already initialized // according to the state scheme. Initialized(genesisRoot common.Hash) bool @@ -122,6 +126,13 @@ func (db *Database) NodeReader(blockRoot common.Hash) (database.NodeReader, erro return db.backend.NodeReader(blockRoot) } +// StateReader returns a reader that allows access to the state data associated +// with the specified state. An error will be returned if the specified state is +// not available. +func (db *Database) StateReader(blockRoot common.Hash) (database.StateReader, error) { + return db.backend.StateReader(blockRoot) +} + // Update performs a state transition by committing dirty nodes contained in the // given set in order to update state from the specified parent to the specified // root. The held pre-images accumulated up to this point will be flushed in case diff --git a/triedb/hashdb/database.go b/triedb/hashdb/database.go index 5de7805c31..fb718f4e74 100644 --- a/triedb/hashdb/database.go +++ b/triedb/hashdb/database.go @@ -635,3 +635,9 @@ func (reader *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([] blob, _ := reader.db.node(hash) return blob, nil } + +// StateReader returns a reader that allows access to the state data associated +// with the specified state. +func (db *Database) StateReader(root common.Hash) (database.StateReader, error) { + return nil, errors.New("not implemented") +} diff --git a/triedb/pathdb/buffer.go b/triedb/pathdb/buffer.go index 3f1d0e90ee..68e136f193 100644 --- a/triedb/pathdb/buffer.go +++ b/triedb/pathdb/buffer.go @@ -33,40 +33,56 @@ import ( // must be checked before diving into disk (since it basically is not yet written // data). type buffer struct { - layers uint64 // The number of diff layers aggregated inside - limit uint64 // The maximum memory allowance in bytes - nodes *nodeSet // Aggregated trie node set + layers uint64 // The number of diff layers aggregated inside + limit uint64 // The maximum memory allowance in bytes + nodes *nodeSet // Aggregated trie node set + states *stateSet // Aggregated state set } // newBuffer initializes the buffer with the provided states and trie nodes. -func newBuffer(limit int, nodes *nodeSet, layers uint64) *buffer { +func newBuffer(limit int, nodes *nodeSet, states *stateSet, layers uint64) *buffer { // Don't panic for lazy users if any provided set is nil if nodes == nil { nodes = newNodeSet(nil) } + if states == nil { + states = newStates(nil, nil) + } return &buffer{ layers: layers, limit: uint64(limit), nodes: nodes, + states: states, } } +// account retrieves the account blob with account address hash. +func (b *buffer) account(hash common.Hash) ([]byte, bool) { + return b.states.account(hash) +} + +// storage retrieves the storage slot with account address hash and slot key. +func (b *buffer) storage(addrHash common.Hash, storageHash common.Hash) ([]byte, bool) { + return b.states.storage(addrHash, storageHash) +} + // node retrieves the trie node with node path and its trie identifier. func (b *buffer) node(owner common.Hash, path []byte) (*trienode.Node, bool) { return b.nodes.node(owner, path) } // commit merges the provided states and trie nodes into the buffer. -func (b *buffer) commit(nodes *nodeSet) *buffer { +func (b *buffer) commit(nodes *nodeSet, states *stateSet) *buffer { b.layers++ b.nodes.merge(nodes) + b.states.merge(states) return b } -// revert is the reverse operation of commit. It also merges the provided states +// revertTo is the reverse operation of commit. It also merges the provided states // and trie nodes into the buffer. The key difference is that the provided state // set should reverse the changes made by the most recent state transition. -func (b *buffer) revert(db ethdb.KeyValueReader, nodes map[common.Hash]map[string]*trienode.Node) error { +func (b *buffer) revertTo(db ethdb.KeyValueReader, nodes map[common.Hash]map[string]*trienode.Node, accounts map[common.Hash][]byte, storages map[common.Hash]map[common.Hash][]byte) error { // Short circuit if no embedded state transition to revert if b.layers == 0 { return errStateUnrecoverable @@ -78,7 +94,8 @@ func (b *buffer) revert(db ethdb.KeyValueReader, nodes map[common.Hash]map[strin b.reset() return nil } - b.nodes.revert(db, nodes) + b.nodes.revertTo(db, nodes) + b.states.revertTo(accounts, storages) return nil } @@ -86,6 +103,7 @@ func (b *buffer) revert(db ethdb.KeyValueReader, nodes map[common.Hash]map[strin func (b *buffer) reset() { b.layers = 0 b.nodes.reset() + b.states.reset() } // empty returns an indicator if buffer is empty. @@ -101,7 +119,7 @@ func (b *buffer) full() bool { // size returns the approximate memory size of the held content. func (b *buffer) size() uint64 { - return b.nodes.size + return b.states.size + b.nodes.size } // flush persists the in-memory dirty trie node into the disk if the configured diff --git a/triedb/pathdb/database.go b/triedb/pathdb/database.go index 48d46c7b08..cfbdb01c49 100644 --- a/triedb/pathdb/database.go +++ b/triedb/pathdb/database.go @@ -68,6 +68,24 @@ type layer interface { // - no error will be returned if the requested node is not found in database. node(owner common.Hash, path []byte, depth int) ([]byte, common.Hash, *nodeLoc, error) + // account directly retrieves the account RLP associated with a particular + // hash in the slim data format. An error will be returned if the read + // operation exits abnormally. Specifically, if the layer is already stale. + // + // Note: + // - the returned account is not a copy, please don't modify it. + // - no error will be returned if the requested account is not found in database. + account(hash common.Hash, depth int) ([]byte, error) + + // storage directly retrieves the storage data associated with a particular hash, + // within a particular account. An error will be returned if the read operation + // exits abnormally. Specifically, if the layer is already stale. + // + // Note: + // - the returned storage data is not a copy, please don't modify it. + // - no error will be returned if the requested slot is not found in database. + storage(accountHash, storageHash common.Hash, depth int) ([]byte, error) + // rootHash returns the root hash for which this layer was made. rootHash() common.Hash @@ -130,17 +148,18 @@ var Defaults = &Config{ // ReadOnly is the config in order to open database in read only mode. var ReadOnly = &Config{ReadOnly: true} -// Database is a multiple-layered structure for maintaining in-memory trie nodes. -// It consists of one persistent base layer backed by a key-value store, on top -// of which arbitrarily many in-memory diff layers are stacked. The memory diffs -// can form a tree with branching, but the disk layer is singleton and common to -// all. If a reorg goes deeper than the disk layer, a batch of reverse diffs can -// be applied to rollback. The deepest reorg that can be handled depends on the -// amount of state histories tracked in the disk. +// Database is a multiple-layered structure for maintaining in-memory states +// along with its dirty trie nodes. It consists of one persistent base layer +// backed by a key-value store, on top of which arbitrarily many in-memory diff +// layers are stacked. The memory diffs can form a tree with branching, but the +// disk layer is singleton and common to all. If a reorg goes deeper than the +// disk layer, a batch of reverse diffs can be applied to rollback. The deepest +// reorg that can be handled depends on the amount of state histories tracked +// in the disk. // // At most one readable and writable database can be opened at the same time in -// the whole system which ensures that only one database writer can operate disk -// state. Unexpected open operations can cause the system to panic. +// the whole system which ensures that only one database writer can operate the +// persistent state. Unexpected open operations can cause the system to panic. type Database struct { // readOnly is the flag whether the mutation is allowed to be applied. // It will be set automatically when the database is journaled during @@ -358,7 +377,7 @@ func (db *Database) Enable(root common.Hash) error { } // Re-construct a new disk layer backed by persistent state // with **empty clean cache and node buffer**. - db.tree.reset(newDiskLayer(root, 0, db, nil, newBuffer(db.config.WriteBufferSize, nil, 0))) + db.tree.reset(newDiskLayer(root, 0, db, nil, newBuffer(db.config.WriteBufferSize, nil, nil, 0))) // Re-enable the database as the final step. db.waitSync = false diff --git a/triedb/pathdb/database_test.go b/triedb/pathdb/database_test.go index 61e0b0928e..648230df15 100644 --- a/triedb/pathdb/database_test.go +++ b/triedb/pathdb/database_test.go @@ -309,7 +309,7 @@ func (t *tester) generate(parent common.Hash) (common.Hash, *trienode.MergedNode delete(t.storages, addrHash) } } - return root, ctx.nodes, NewStateSetWithOrigin(ctx.accountOrigin, ctx.storageOrigin) + return root, ctx.nodes, NewStateSetWithOrigin(ctx.accounts, ctx.storages, ctx.accountOrigin, ctx.storageOrigin) } // lastHash returns the latest root hash, or empty if nothing is cached. diff --git a/triedb/pathdb/difflayer.go b/triedb/pathdb/difflayer.go index ecc318614f..c06026b6ca 100644 --- a/triedb/pathdb/difflayer.go +++ b/triedb/pathdb/difflayer.go @@ -52,6 +52,7 @@ func newDiffLayer(parent layer, root common.Hash, id uint64, block uint64, nodes states: states, } dirtyNodeWriteMeter.Mark(int64(nodes.size)) + dirtyStateWriteMeter.Mark(int64(states.size)) log.Debug("Created new diff layer", "id", id, "block", block, "nodesize", common.StorageSize(nodes.size), "statesize", common.StorageSize(states.size)) return dl } @@ -96,6 +97,58 @@ func (dl *diffLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co return dl.parent.node(owner, path, depth+1) } +// account directly retrieves the account RLP associated with a particular +// hash in the slim data format. +// +// Note the returned account is not a copy, please don't modify it. +func (dl *diffLayer) account(hash common.Hash, depth int) ([]byte, error) { + // Hold the lock, ensure the parent won't be changed during the + // state accessing. + dl.lock.RLock() + defer dl.lock.RUnlock() + + if blob, found := dl.states.account(hash); found { + dirtyStateHitMeter.Mark(1) + dirtyStateHitDepthHist.Update(int64(depth)) + dirtyStateReadMeter.Mark(int64(len(blob))) + + if len(blob) == 0 { + stateAccountInexMeter.Mark(1) + } else { + stateAccountExistMeter.Mark(1) + } + return blob, nil + } + // Account is unknown to this layer, resolve from parent + return dl.parent.account(hash, depth+1) +} + +// storage directly retrieves the storage data associated with a particular hash, +// within a particular account. +// +// Note the returned storage slot is not a copy, please don't modify it. +func (dl *diffLayer) storage(accountHash, storageHash common.Hash, depth int) ([]byte, error) { + // Hold the lock, ensure the parent won't be changed during the + // state accessing. + dl.lock.RLock() + defer dl.lock.RUnlock() + + if blob, found := dl.states.storage(accountHash, storageHash); found { + dirtyStateHitMeter.Mark(1) + dirtyStateHitDepthHist.Update(int64(depth)) + dirtyStateReadMeter.Mark(int64(len(blob))) + + if len(blob) == 0 { + stateStorageInexMeter.Mark(1) + } else { + stateStorageExistMeter.Mark(1) + } + return blob, nil + } + // storage slot is unknown to this layer, resolve from parent + return dl.parent.storage(accountHash, storageHash, depth+1) +} + // update implements the layer interface, creating a new layer on top of the // existing layer tree with the specified data items. func (dl *diffLayer) update(root common.Hash, id uint64, block uint64, nodes *nodeSet, states *StateSetWithOrigin) *diffLayer { diff --git a/triedb/pathdb/difflayer_test.go b/triedb/pathdb/difflayer_test.go index 61e8b4e064..7176d9964d 100644 --- a/triedb/pathdb/difflayer_test.go +++ b/triedb/pathdb/difflayer_test.go @@ -30,7 +30,7 @@ import ( func emptyLayer() *diskLayer { return &diskLayer{ db: New(rawdb.NewMemoryDatabase(), nil, false), - buffer: newBuffer(defaultBufferSize, nil, 0), + buffer: newBuffer(defaultBufferSize, nil, nil, 0), } } @@ -76,7 +76,7 @@ func benchmarkSearch(b *testing.B, depth int, total int) { nblob = common.CopyBytes(blob) } } - return newDiffLayer(parent, common.Hash{}, 0, 0, newNodeSet(nodes), NewStateSetWithOrigin(nil, nil)) + return newDiffLayer(parent, common.Hash{}, 0, 0, newNodeSet(nodes), NewStateSetWithOrigin(nil, nil, nil, nil)) } var layer layer layer = emptyLayer() @@ -118,7 +118,7 @@ func BenchmarkPersist(b *testing.B) { ) nodes[common.Hash{}][string(path)] = node } - return newDiffLayer(parent, common.Hash{}, 0, 0, newNodeSet(nodes), NewStateSetWithOrigin(nil, nil)) + return newDiffLayer(parent, common.Hash{}, 0, 0, newNodeSet(nodes), NewStateSetWithOrigin(nil, nil, nil, nil)) } for i := 0; i < b.N; i++ { b.StopTimer() @@ -156,7 +156,7 @@ func BenchmarkJournal(b *testing.B) { ) nodes[common.Hash{}][string(path)] = node } - return newDiffLayer(parent, common.Hash{}, 0, 0, newNodeSet(nodes), new(StateSetWithOrigin)) + return newDiffLayer(parent, common.Hash{}, 0, 0, newNodeSet(nodes), NewStateSetWithOrigin(nil, nil, nil, nil)) } var layer layer layer = emptyLayer() diff --git a/triedb/pathdb/disklayer.go b/triedb/pathdb/disklayer.go index edbe630968..003431b19b 100644 --- a/triedb/pathdb/disklayer.go +++ b/triedb/pathdb/disklayer.go @@ -17,6 +17,7 @@ package pathdb import ( + "errors" "fmt" "sync" @@ -33,7 +34,7 @@ type diskLayer struct { id uint64 // Immutable, corresponding state id db *Database // Path-based trie database nodes *fastcache.Cache // GC friendly memory cache of clean nodes - buffer *buffer // Dirty buffer to aggregate writes of nodes + buffer *buffer // Dirty buffer to aggregate writes of nodes and states stale bool // Signals that the layer became stale (state progressed) lock sync.RWMutex // Lock used to protect stale flag } @@ -140,6 +141,75 @@ func (dl *diskLayer) node(owner common.Hash, path []byte, depth int) ([]byte, co return blob, h.hash(blob), &nodeLoc{loc: locDiskLayer, depth: depth}, nil } +// account directly retrieves the account RLP associated with a particular +// hash in the slim data format. +// +// Note the returned account is not a copy, please don't modify it. +func (dl *diskLayer) account(hash common.Hash, depth int) ([]byte, error) { + dl.lock.RLock() + defer dl.lock.RUnlock() + + if dl.stale { + return nil, errSnapshotStale + } + // Try to retrieve the account from the not-yet-written + // node buffer first. Note the buffer is lock free since + // it's impossible to mutate the buffer before tagging the + // layer as stale. + blob, found := dl.buffer.account(hash) + if found { + dirtyStateHitMeter.Mark(1) + dirtyStateReadMeter.Mark(int64(len(blob))) + dirtyStateHitDepthHist.Update(int64(depth)) + + if len(blob) == 0 { + stateAccountInexMeter.Mark(1) + } else { + stateAccountExistMeter.Mark(1) + } + return blob, nil + } + dirtyStateMissMeter.Mark(1) + + // TODO(rjl493456442) support persistent state retrieval + return nil, errors.New("not supported") +} + +// storage directly retrieves the storage data associated with a particular hash, +// within a particular account. +// +// Note the returned account is not a copy, please don't modify it. +func (dl *diskLayer) storage(accountHash, storageHash common.Hash, depth int) ([]byte, error) { + // Hold the lock, ensure the parent won't be changed during the + // state accessing. + dl.lock.RLock() + defer dl.lock.RUnlock() + + if dl.stale { + return nil, errSnapshotStale + } + // Try to retrieve the storage slot from the not-yet-written + // node buffer first. Note the buffer is lock free since + // it's impossible to mutate the buffer before tagging the + // layer as stale. + if blob, found := dl.buffer.storage(accountHash, storageHash); found { + dirtyStateHitMeter.Mark(1) + dirtyStateReadMeter.Mark(int64(len(blob))) + dirtyStateHitDepthHist.Update(int64(depth)) + + if len(blob) == 0 { + stateStorageInexMeter.Mark(1) + } else { + stateStorageExistMeter.Mark(1) + } + return blob, nil + } + dirtyStateMissMeter.Mark(1) + + // TODO(rjl493456442) support persistent state retrieval + return nil, errors.New("not supported") +} + // update implements the layer interface, returning a new diff layer on top // with the given state set. func (dl *diskLayer) update(root common.Hash, id uint64, block uint64, nodes *nodeSet, states *StateSetWithOrigin) *diffLayer { @@ -190,14 +260,14 @@ func (dl *diskLayer) commit(bottom *diffLayer, force bool) (*diskLayer, error) { // In a unique scenario where the ID of the oldest history object (after tail // truncation) surpasses the persisted state ID, we take the necessary action - // of forcibly committing the cached dirty nodes to ensure that the persisted + // of forcibly committing the cached dirty states to ensure that the persisted // state ID remains higher. if !force && rawdb.ReadPersistentStateID(dl.db.diskdb) < oldest { force = true } - // Merge the trie nodes of the bottom-most diff layer into the buffer as the - // combined layer. - combined := dl.buffer.commit(bottom.nodes) + // Merge the trie nodes and flat states of the bottom-most diff layer into the + // buffer as the combined layer. + combined := dl.buffer.commit(bottom.nodes, bottom.states.stateSet) if combined.full() || force { if err := combined.flush(dl.db.diskdb, dl.db.freezer, dl.nodes, bottom.stateID()); err != nil { return nil, err @@ -225,6 +295,24 @@ func (dl *diskLayer) revert(h *history) (*diskLayer, error) { if dl.id == 0 { return nil, fmt.Errorf("%w: zero state id", errStateUnrecoverable) } + var ( + buff = crypto.NewKeccakState() + hashes = make(map[common.Address]common.Hash) + accounts = make(map[common.Hash][]byte) + storages = make(map[common.Hash]map[common.Hash][]byte) + ) + for addr, blob := range h.accounts { + hash := crypto.HashData(buff, addr.Bytes()) + hashes[addr] = hash + accounts[hash] = blob + } + for addr, storage := range h.storages { + hash, ok := hashes[addr] + if !ok { + panic(fmt.Errorf("storage history with no account %x", addr)) + } + storages[hash] = storage + } // Apply the reverse state changes upon the current state. This must // be done before holding the lock in order to access state in "this" // layer. @@ -244,7 +332,7 @@ func (dl *diskLayer) revert(h *history) (*diskLayer, error) { // needs to be reverted is not yet flushed and cached in node // buffer, otherwise, manipulate persistent state directly. if !dl.buffer.empty() { - err := dl.buffer.revert(dl.db.diskdb, nodes) + err := dl.buffer.revertTo(dl.db.diskdb, nodes, accounts, storages) if err != nil { return nil, err } diff --git a/triedb/pathdb/journal.go b/triedb/pathdb/journal.go index 70fa1fb9f8..779a262fdd 100644 --- a/triedb/pathdb/journal.go +++ b/triedb/pathdb/journal.go @@ -45,7 +45,8 @@ var ( // // - Version 0: initial version // - Version 1: storage.Incomplete field is removed -const journalVersion uint64 = 1 +// - Version 2: add post-modification state values +const journalVersion uint64 = 2 // loadJournal tries to parse the layer journal from the disk. func (db *Database) loadJournal(diskRoot common.Hash) (layer, error) { @@ -108,7 +109,7 @@ func (db *Database) loadLayers() layer { log.Info("Failed to load journal, discard it", "err", err) } // Return single layer with persistent state. - return newDiskLayer(root, rawdb.ReadPersistentStateID(db.diskdb), db, nil, newBuffer(db.config.WriteBufferSize, nil, 0)) + return newDiskLayer(root, rawdb.ReadPersistentStateID(db.diskdb), db, nil, newBuffer(db.config.WriteBufferSize, nil, nil, 0)) } // loadDiskLayer reads the binary blob from the layer journal, reconstructing @@ -135,7 +136,12 @@ func (db *Database) loadDiskLayer(r *rlp.Stream) (layer, error) { if err := nodes.decode(r); err != nil { return nil, err } - return newDiskLayer(root, id, db, nil, newBuffer(db.config.WriteBufferSize, &nodes, id-stored)), nil + // Resolve flat state sets in aggregated buffer + var states stateSet + if err := states.decode(r); err != nil { + return nil, err + } + return newDiskLayer(root, id, db, nil, newBuffer(db.config.WriteBufferSize, &nodes, &states, id-stored)), nil } // loadDiffLayer reads the next sections of a layer journal, reconstructing a new @@ -189,6 +195,10 @@ func (dl *diskLayer) journal(w io.Writer) error { if err := dl.buffer.nodes.encode(w); err != nil { return err } + // Step four, write the accumulated flat states into the journal + if err := dl.buffer.states.encode(w); err != nil { + return err + } log.Debug("Journaled pathdb disk layer", "root", dl.root) return nil } diff --git a/triedb/pathdb/metrics.go b/triedb/pathdb/metrics.go index 8897cc4450..1a2559e38b 100644 --- a/triedb/pathdb/metrics.go +++ b/triedb/pathdb/metrics.go @@ -30,10 +30,21 @@ var ( dirtyNodeWriteMeter = metrics.NewRegisteredMeter("pathdb/dirty/node/write", nil) dirtyNodeHitDepthHist = metrics.NewRegisteredHistogram("pathdb/dirty/node/depth", nil, metrics.NewExpDecaySample(1028, 0.015)) - cleanFalseMeter = metrics.NewRegisteredMeter("pathdb/clean/false", nil) - dirtyFalseMeter = metrics.NewRegisteredMeter("pathdb/dirty/false", nil) - diskFalseMeter = metrics.NewRegisteredMeter("pathdb/disk/false", nil) - diffFalseMeter = metrics.NewRegisteredMeter("pathdb/diff/false", nil) + stateAccountInexMeter = metrics.NewRegisteredMeter("pathdb/state/account/inex/total", nil) + stateStorageInexMeter = metrics.NewRegisteredMeter("pathdb/state/storage/inex/total", nil) + stateAccountExistMeter = metrics.NewRegisteredMeter("pathdb/state/account/exist/total", nil) + stateStorageExistMeter = metrics.NewRegisteredMeter("pathdb/state/storage/exist/total", nil) + + dirtyStateHitMeter = metrics.NewRegisteredMeter("pathdb/dirty/state/hit", nil) + dirtyStateMissMeter = metrics.NewRegisteredMeter("pathdb/dirty/state/miss", nil) + dirtyStateReadMeter = metrics.NewRegisteredMeter("pathdb/dirty/state/read", nil) + dirtyStateWriteMeter = metrics.NewRegisteredMeter("pathdb/dirty/state/write", nil) + dirtyStateHitDepthHist = metrics.NewRegisteredHistogram("pathdb/dirty/state/depth", nil, metrics.NewExpDecaySample(1028, 0.015)) + + nodeCleanFalseMeter = metrics.NewRegisteredMeter("pathdb/clean/false", nil) + nodeDirtyFalseMeter = metrics.NewRegisteredMeter("pathdb/dirty/false", nil) + nodeDiskFalseMeter = metrics.NewRegisteredMeter("pathdb/disk/false", nil) + nodeDiffFalseMeter = metrics.NewRegisteredMeter("pathdb/diff/false", nil) commitTimeTimer = metrics.NewRegisteredTimer("pathdb/commit/time", nil) commitNodesMeter = metrics.NewRegisteredMeter("pathdb/commit/nodes", nil) @@ -41,6 +52,10 @@ var ( gcTrieNodeMeter = metrics.NewRegisteredMeter("pathdb/gc/node/count", nil) gcTrieNodeBytesMeter = metrics.NewRegisteredMeter("pathdb/gc/node/bytes", nil) + gcAccountMeter = metrics.NewRegisteredMeter("pathdb/gc/account/count", nil) + gcAccountBytesMeter = metrics.NewRegisteredMeter("pathdb/gc/account/bytes", nil) + gcStorageMeter = metrics.NewRegisteredMeter("pathdb/gc/storage/count", nil) + gcStorageBytesMeter = metrics.NewRegisteredMeter("pathdb/gc/storage/bytes", nil) historyBuildTimeMeter = metrics.NewRegisteredTimer("pathdb/history/time", nil) historyDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil) diff --git a/triedb/pathdb/nodes.go b/triedb/pathdb/nodes.go index ade669512e..dee8c872ac 100644 --- a/triedb/pathdb/nodes.go +++ b/triedb/pathdb/nodes.go @@ -131,9 +131,9 @@ func (s *nodeSet) merge(set *nodeSet) { s.updateSize(delta) } -// revert merges the provided trie nodes into the set. This should reverse the +// revertTo merges the provided trie nodes into the set. This should reverse the // changes made by the most recent state transition. -func (s *nodeSet) revert(db ethdb.KeyValueReader, nodes map[common.Hash]map[string]*trienode.Node) { +func (s *nodeSet) revertTo(db ethdb.KeyValueReader, nodes map[common.Hash]map[string]*trienode.Node) { var delta int64 for owner, subset := range nodes { current, ok := s.nodes[owner] diff --git a/triedb/pathdb/reader.go b/triedb/pathdb/reader.go index 2ca4a0205b..4bba813b14 100644 --- a/triedb/pathdb/reader.go +++ b/triedb/pathdb/reader.go @@ -21,7 +21,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/triedb/database" ) @@ -66,13 +68,13 @@ func (r *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, // is not found. switch loc.loc { case locCleanCache: - cleanFalseMeter.Mark(1) + nodeCleanFalseMeter.Mark(1) case locDirtyCache: - dirtyFalseMeter.Mark(1) + nodeDirtyFalseMeter.Mark(1) case locDiffLayer: - diffFalseMeter.Mark(1) + nodeDiffFalseMeter.Mark(1) case locDiskLayer: - diskFalseMeter.Mark(1) + nodeDiskFalseMeter.Mark(1) } blobHex := "nil" if len(blob) > 0 { @@ -84,6 +86,39 @@ func (r *reader) Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, return blob, nil } +// Account directly retrieves the account associated with a particular hash in +// the slim data format. An error will be returned if the read operation exits +// abnormally. Specifically, if the layer is already stale. +// +// Note: +// - the returned account object is safe to modify +// - no error will be returned if the requested account is not found in database +func (r *reader) Account(hash common.Hash) (*types.SlimAccount, error) { + blob, err := r.layer.account(hash, 0) + if err != nil { + return nil, err + } + if len(blob) == 0 { + return nil, nil + } + account := new(types.SlimAccount) + if err := rlp.DecodeBytes(blob, account); err != nil { + panic(err) + } + return account, nil +} + +// Storage directly retrieves the storage data associated with a particular hash, +// within a particular account. An error will be returned if the read operation +// exits abnormally. Specifically, if the layer is already stale. +// +// Note: +// - the returned storage data is not a copy, please don't modify it +// - no error will be returned if the requested slot is not found in database +func (r *reader) Storage(accountHash, storageHash common.Hash) ([]byte, error) { + return r.layer.storage(accountHash, storageHash, 0) +} + // NodeReader retrieves a layer belonging to the given state root. func (db *Database) NodeReader(root common.Hash) (database.NodeReader, error) { layer := db.tree.get(root) @@ -92,3 +127,13 @@ func (db *Database) NodeReader(root common.Hash) (database.NodeReader, error) { } return &reader{layer: layer, noHashCheck: db.isVerkle}, nil } + +// StateReader returns a reader that allows access to the state data associated +// with the specified state. +func (db *Database) StateReader(root common.Hash) (database.StateReader, error) { + layer := db.tree.get(root) + if layer == nil { + return nil, fmt.Errorf("state %#x is not available", root) + } + return &reader{layer: layer}, nil +} diff --git a/triedb/pathdb/states.go b/triedb/pathdb/states.go index da8befab95..e1611b3ebc 100644 --- a/triedb/pathdb/states.go +++ b/triedb/pathdb/states.go @@ -19,10 +19,15 @@ package pathdb import ( "fmt" "io" + "slices" + "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/exp/maps" ) // counter helps in tracking items and their corresponding sizes. @@ -43,9 +48,373 @@ func (c *counter) report(count metrics.Meter, size metrics.Meter) { size.Mark(int64(c.size)) } +// stateSet represents a collection of state modifications associated with a +// transition (e.g., a block execution) or multiple aggregated transitions. +// +// A stateSet can only reside within a diffLayer or the buffer of a diskLayer, +// serving as the envelope for the set. Lock protection is not required for +// accessing or mutating the account set and storage set, as the associated +// envelope is always marked as stale before any mutation is applied. Any +// subsequent state access will be denied due to the stale flag. Therefore, +// state access and mutation won't happen at the same time with guarantee. +type stateSet struct { + accountData map[common.Hash][]byte // Keyed accounts for direct retrieval (nil means deleted) + storageData map[common.Hash]map[common.Hash][]byte // Keyed storage slots for direct retrieval. one per account (nil means deleted) + size uint64 // Memory size of the state data (accountData and storageData) + + accountListSorted []common.Hash // List of account for iteration. If it exists, it's sorted, otherwise it's nil + storageListSorted map[common.Hash][]common.Hash // List of storage slots for iterated retrievals, one per account. Any existing lists are sorted if non-nil + + // Lock for guarding the two lists above. These lists might be accessed + // concurrently and lock protection is essential to avoid concurrent + // slice or map read/write. + listLock sync.RWMutex +} + +// newStates constructs the state set with the provided account and storage data. +func newStates(accounts map[common.Hash][]byte, storages map[common.Hash]map[common.Hash][]byte) *stateSet { + // Don't panic for the lazy callers, initialize the nil maps instead. + if accounts == nil { + accounts = make(map[common.Hash][]byte) + } + if storages == nil { + storages = make(map[common.Hash]map[common.Hash][]byte) + } + s := &stateSet{ + accountData: accounts, + storageData: storages, + storageListSorted: make(map[common.Hash][]common.Hash), + } + s.size = s.check() + return s +} + +// account returns the account data associated with the specified address hash. +func (s *stateSet) account(hash common.Hash) ([]byte, bool) { + // If the account is known locally, return it + if data, ok := s.accountData[hash]; ok { + return data, true + } + return nil, false // account is unknown in this set +} + +// storage returns the storage slot associated with the specified address hash +// and storage key hash. +func (s *stateSet) storage(accountHash, storageHash common.Hash) ([]byte, bool) { + // If the account is known locally, try to resolve the slot locally + if storage, ok := s.storageData[accountHash]; ok { + if data, ok := storage[storageHash]; ok { + return data, true + } + } + return nil, false // storage is unknown in this set +} + +// check sanitizes accounts and storage slots to ensure the data validity. +// Additionally, it computes the total memory size occupied by the maps. +func (s *stateSet) check() uint64 { + var size int + for _, blob := range s.accountData { + size += common.HashLength + len(blob) + } + for accountHash, slots := range s.storageData { + if slots == nil { + panic(fmt.Sprintf("storage %#x nil", accountHash)) // nil slots is not permitted + } + for _, blob := range slots { + size += 2*common.HashLength + len(blob) + } + } + return uint64(size) +} + +// accountList returns a sorted list of all accounts in this state set, including +// the deleted ones. +// +// Note, the returned slice is not a copy, so do not modify it. +// +// nolint:unused +func (s *stateSet) accountList() []common.Hash { + // If an old list already exists, return it + s.listLock.RLock() + list := s.accountListSorted + s.listLock.RUnlock() + + if list != nil { + return list + } + // No old sorted account list exists, generate a new one. It's possible that + // multiple threads waiting for the write lock may regenerate the list + // multiple times, which is acceptable. + s.listLock.Lock() + defer s.listLock.Unlock() + + list = maps.Keys(s.accountData) + slices.SortFunc(list, common.Hash.Cmp) + s.accountListSorted = list + return list +} + +// StorageList returns a sorted list of all storage slot hashes in this state set +// for the given account. The returned list will include the hash of deleted +// storage slot. +// +// Note, the returned slice is not a copy, so do not modify it. +// +// nolint:unused +func (s *stateSet) storageList(accountHash common.Hash) []common.Hash { + s.listLock.RLock() + if _, ok := s.storageData[accountHash]; !ok { + // Account not tracked by this layer + s.listLock.RUnlock() + return nil + } + // If an old list already exists, return it + if list, exist := s.storageListSorted[accountHash]; exist { + s.listLock.RUnlock() + return list // the cached list can't be nil + } + s.listLock.RUnlock() + + // No old sorted account list exists, generate a new one. It's possible that + // multiple threads waiting for the write lock may regenerate the list + // multiple times, which is acceptable. + s.listLock.Lock() + defer s.listLock.Unlock() + + list := maps.Keys(s.storageData[accountHash]) + slices.SortFunc(list, common.Hash.Cmp) + s.storageListSorted[accountHash] = list + return list +} + +// clearLists invalidates the cached account list and storage lists. +func (s *stateSet) clearLists() { + s.listLock.Lock() + defer s.listLock.Unlock() + + s.accountListSorted = nil + s.storageListSorted = make(map[common.Hash][]common.Hash) +} + +// merge integrates the accounts and storages from the external set into the +// local set, ensuring the combined set reflects the combined state of both. +// +// The stateSet supplied as parameter set will not be mutated by this operation, +// as it may still be referenced by other layers. +func (s *stateSet) merge(other *stateSet) { + var ( + delta int + accountOverwrites counter + storageOverwrites counter + ) + // Apply the updated account data + for accountHash, data := range other.accountData { + if origin, ok := s.accountData[accountHash]; ok { + delta += len(data) - len(origin) + accountOverwrites.add(common.HashLength + len(origin)) + } else { + delta += common.HashLength + len(data) + } + s.accountData[accountHash] = data + } + // Apply all the updated storage slots (individually) + for accountHash, storage := range other.storageData { + // If storage didn't exist in the set, overwrite blindly + if _, ok := s.storageData[accountHash]; !ok { + // To prevent potential concurrent map read/write issues, allocate a + // new map for the storage instead of claiming it directly from the + // passed external set. Even after merging, the slots belonging to the + // external state set remain accessible, so ownership of the map should + // not be taken, and any mutation on it should be avoided. + slots := make(map[common.Hash][]byte, len(storage)) + for storageHash, data := range storage { + slots[storageHash] = data + delta += 2*common.HashLength + len(data) + } + s.storageData[accountHash] = slots + continue + } + // Storage exists in both local and external set, merge the slots + slots := s.storageData[accountHash] + for storageHash, data := range storage { + if origin, ok := slots[storageHash]; ok { + delta += len(data) - len(origin) + storageOverwrites.add(2*common.HashLength + len(origin)) + } else { + delta += 2*common.HashLength + len(data) + } + slots[storageHash] = data + } + } + accountOverwrites.report(gcAccountMeter, gcAccountBytesMeter) + storageOverwrites.report(gcStorageMeter, gcStorageBytesMeter) + s.clearLists() + s.updateSize(delta) +} + +// revertTo takes the original value of accounts and storages as input and reverts +// the latest state transition applied on the state set. +// +// Notably, this operation may result in the set containing more entries after a +// revert. For example, if account x did not exist and was created during transition +// w, reverting w will retain an x=nil entry in the set. And also if account x along +// with its storage slots was deleted in the transition w, reverting w will retain +// a list of additional storage slots with their original value. +func (s *stateSet) revertTo(accountOrigin map[common.Hash][]byte, storageOrigin map[common.Hash]map[common.Hash][]byte) { + var delta int // size tracking + for addrHash, blob := range accountOrigin { + data, ok := s.accountData[addrHash] + if !ok { + panic(fmt.Sprintf("non-existent account for reverting, %x", addrHash)) + } + if len(data) == 0 && len(blob) == 0 { + panic(fmt.Sprintf("invalid account mutation (null to null), %x", addrHash)) + } + delta += len(blob) - len(data) + s.accountData[addrHash] = blob + } + // Overwrite the storage data with original value blindly + for addrHash, storage := range storageOrigin { + slots := s.storageData[addrHash] + if len(slots) == 0 { + panic(fmt.Sprintf("non-existent storage set for reverting, %x", addrHash)) + } + for storageHash, blob := range storage { + data, ok := slots[storageHash] + if !ok { + panic(fmt.Sprintf("non-existent storage slot for reverting, %x-%x", addrHash, storageHash)) + } + if len(blob) == 0 && len(data) == 0 { + panic(fmt.Sprintf("invalid storage slot mutation (null to null), %x-%x", addrHash, storageHash)) + } + delta += len(blob) - len(data) + slots[storageHash] = blob + } + } + s.clearLists() + s.updateSize(delta) +} + +// updateSize updates the total cache size by the given delta. +func (s *stateSet) updateSize(delta int) { + size := int64(s.size) + int64(delta) + if size >= 0 { + s.size = uint64(size) + return + } + log.Error("Stateset size underflow", "prev", common.StorageSize(s.size), "delta", common.StorageSize(delta)) + s.size = 0 +} + +// encode serializes the content of state set into the provided writer. +func (s *stateSet) encode(w io.Writer) error { + // Encode accounts + type accounts struct { + AddrHashes []common.Hash + Accounts [][]byte + } + var enc accounts + for addrHash, blob := range s.accountData { + enc.AddrHashes = append(enc.AddrHashes, addrHash) + enc.Accounts = append(enc.Accounts, blob) + } + if err := rlp.Encode(w, enc); err != nil { + return err + } + // Encode storages + type Storage struct { + AddrHash common.Hash + Keys []common.Hash + Vals [][]byte + } + storages := make([]Storage, 0, len(s.storageData)) + for addrHash, slots := range s.storageData { + keys := make([]common.Hash, 0, len(slots)) + vals := make([][]byte, 0, len(slots)) + for key, val := range slots { + keys = append(keys, key) + vals = append(vals, val) + } + storages = append(storages, Storage{ + AddrHash: addrHash, + Keys: keys, + Vals: vals, + }) + } + return rlp.Encode(w, storages) +} + +// decode deserializes the content from the rlp stream into the state set. +func (s *stateSet) decode(r *rlp.Stream) error { + type accounts struct { + AddrHashes []common.Hash + Accounts [][]byte + } + var ( + dec accounts + accountSet = make(map[common.Hash][]byte) + ) + if err := r.Decode(&dec); err != nil { + return fmt.Errorf("load diff accounts: %v", err) + } + for i := 0; i < len(dec.AddrHashes); i++ { + accountSet[dec.AddrHashes[i]] = dec.Accounts[i] + } + s.accountData = accountSet + + // Decode storages + type storage struct { + AddrHash common.Hash + Keys []common.Hash + Vals [][]byte + } + var ( + storages []storage + storageSet = make(map[common.Hash]map[common.Hash][]byte) + ) + if err := r.Decode(&storages); err != nil { + return fmt.Errorf("load diff storage: %v", err) + } + for _, entry := range storages { + storageSet[entry.AddrHash] = make(map[common.Hash][]byte, len(entry.Keys)) + for i := 0; i < len(entry.Keys); i++ { + storageSet[entry.AddrHash][entry.Keys[i]] = entry.Vals[i] + } + } + s.storageData = storageSet + s.storageListSorted = make(map[common.Hash][]common.Hash) + + s.size = s.check() + return nil +} + +// reset clears all cached state data, including any optional sorted lists that +// may have been generated. +func (s *stateSet) reset() { + s.accountData = make(map[common.Hash][]byte) + s.storageData = make(map[common.Hash]map[common.Hash][]byte) + s.size = 0 + s.accountListSorted = nil + s.storageListSorted = make(map[common.Hash][]common.Hash) +} + +// dbsize returns the approximate size for db write. +// +// nolint:unused +func (s *stateSet) dbsize() int { + m := len(s.accountData) * len(rawdb.SnapshotAccountPrefix) + for _, slots := range s.storageData { + m += len(slots) * len(rawdb.SnapshotStoragePrefix) + } + return m + int(s.size) +} + // StateSetWithOrigin wraps the state set with additional original values of the // mutated states. type StateSetWithOrigin struct { + *stateSet + // AccountOrigin represents the account data before the state transition, // corresponding to both the accountData and destructSet. It's keyed by the // account address. The nil value means the account was not present before. @@ -62,7 +431,7 @@ type StateSetWithOrigin struct { } // NewStateSetWithOrigin constructs the state set with the provided data. -func NewStateSetWithOrigin(accountOrigin map[common.Address][]byte, storageOrigin map[common.Address]map[common.Hash][]byte) *StateSetWithOrigin { +func NewStateSetWithOrigin(accounts map[common.Hash][]byte, storages map[common.Hash]map[common.Hash][]byte, accountOrigin map[common.Address][]byte, storageOrigin map[common.Address]map[common.Hash][]byte) *StateSetWithOrigin { // Don't panic for the lazy callers, initialize the nil maps instead. if accountOrigin == nil { accountOrigin = make(map[common.Address][]byte) @@ -82,15 +451,21 @@ func NewStateSetWithOrigin(accountOrigin map[common.Address][]byte, storageOrigi size += 2*common.HashLength + len(data) } } + set := newStates(accounts, storages) return &StateSetWithOrigin{ + stateSet: set, accountOrigin: accountOrigin, storageOrigin: storageOrigin, - size: uint64(size), + size: set.size + uint64(size), } } // encode serializes the content of state set into the provided writer. func (s *StateSetWithOrigin) encode(w io.Writer) error { + // Encode state set + if err := s.stateSet.encode(w); err != nil { + return err + } // Encode accounts type Accounts struct { Addresses []common.Address @@ -108,7 +483,7 @@ func (s *StateSetWithOrigin) encode(w io.Writer) error { type Storage struct { Address common.Address Keys []common.Hash - Blobs [][]byte + Vals [][]byte } storages := make([]Storage, 0, len(s.storageOrigin)) for address, slots := range s.storageOrigin { @@ -118,13 +493,19 @@ func (s *StateSetWithOrigin) encode(w io.Writer) error { keys = append(keys, key) vals = append(vals, val) } - storages = append(storages, Storage{Address: address, Keys: keys, Blobs: vals}) + storages = append(storages, Storage{Address: address, Keys: keys, Vals: vals}) } return rlp.Encode(w, storages) } // decode deserializes the content from the rlp stream into the state set. func (s *StateSetWithOrigin) decode(r *rlp.Stream) error { + if s.stateSet == nil { + s.stateSet = &stateSet{} + } + if err := s.stateSet.decode(r); err != nil { + return err + } // Decode account origin type Accounts struct { Addresses []common.Address @@ -146,7 +527,7 @@ func (s *StateSetWithOrigin) decode(r *rlp.Stream) error { type Storage struct { Address common.Address Keys []common.Hash - Blobs [][]byte + Vals [][]byte } var ( storages []Storage @@ -158,7 +539,7 @@ func (s *StateSetWithOrigin) decode(r *rlp.Stream) error { for _, storage := range storages { storageSet[storage.Address] = make(map[common.Hash][]byte) for i := 0; i < len(storage.Keys); i++ { - storageSet[storage.Address][storage.Keys[i]] = storage.Blobs[i] + storageSet[storage.Address][storage.Keys[i]] = storage.Vals[i] } } s.storageOrigin = storageSet diff --git a/triedb/pathdb/states_test.go b/triedb/pathdb/states_test.go new file mode 100644 index 0000000000..4557fa958d --- /dev/null +++ b/triedb/pathdb/states_test.go @@ -0,0 +1,453 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package pathdb + +import ( + "bytes" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" +) + +func TestStatesMerge(t *testing.T) { + a := newStates( + map[common.Hash][]byte{ + common.Hash{0xa}: {0xa0}, + common.Hash{0xb}: {0xb0}, + common.Hash{0xc}: {0xc0}, + }, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0xa}: { + common.Hash{0x1}: {0x10}, + common.Hash{0x2}: {0x20}, + }, + common.Hash{0xb}: { + common.Hash{0x1}: {0x10}, + }, + common.Hash{0xc}: { + common.Hash{0x1}: {0x10}, + }, + }, + ) + b := newStates( + map[common.Hash][]byte{ + common.Hash{0xa}: {0xa1}, + common.Hash{0xb}: {0xb1}, + common.Hash{0xc}: nil, // delete account + }, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0xa}: { + common.Hash{0x1}: {0x11}, + common.Hash{0x2}: nil, // delete slot + common.Hash{0x3}: {0x31}, + }, + common.Hash{0xb}: { + common.Hash{0x1}: {0x11}, + }, + common.Hash{0xc}: { + common.Hash{0x1}: nil, // delete slot + }, + }, + ) + a.merge(b) + + blob, exist := a.account(common.Hash{0xa}) + if !exist || !bytes.Equal(blob, []byte{0xa1}) { + t.Error("Unexpected value for account a") + } + blob, exist = a.account(common.Hash{0xb}) + if !exist || !bytes.Equal(blob, []byte{0xb1}) { + t.Error("Unexpected value for account b") + } + blob, exist = a.account(common.Hash{0xc}) + if !exist || len(blob) != 0 { + t.Error("Unexpected value for account c") + } + // unknown account + blob, exist = a.account(common.Hash{0xd}) + if exist || len(blob) != 0 { + t.Error("Unexpected value for account d") + } + + blob, exist = a.storage(common.Hash{0xa}, common.Hash{0x1}) + if !exist || !bytes.Equal(blob, []byte{0x11}) { + t.Error("Unexpected value for a's storage") + } + blob, exist = a.storage(common.Hash{0xa}, common.Hash{0x2}) + if !exist || len(blob) != 0 { + t.Error("Unexpected value for a's storage") + } + blob, exist = a.storage(common.Hash{0xa}, common.Hash{0x3}) + if !exist || !bytes.Equal(blob, []byte{0x31}) { + t.Error("Unexpected value for a's storage") + } + blob, exist = a.storage(common.Hash{0xb}, common.Hash{0x1}) + if !exist || !bytes.Equal(blob, []byte{0x11}) { + t.Error("Unexpected value for b's storage") + } + blob, exist = a.storage(common.Hash{0xc}, common.Hash{0x1}) + if !exist || len(blob) != 0 { + t.Error("Unexpected value for c's storage") + } + + // unknown storage slots + blob, exist = a.storage(common.Hash{0xd}, common.Hash{0x1}) + if exist || len(blob) != 0 { + t.Error("Unexpected value for d's storage") + } +} + +func TestStatesRevert(t *testing.T) { + a := newStates( + map[common.Hash][]byte{ + common.Hash{0xa}: {0xa0}, + common.Hash{0xb}: {0xb0}, + common.Hash{0xc}: {0xc0}, + }, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0xa}: { + common.Hash{0x1}: {0x10}, + common.Hash{0x2}: {0x20}, + }, + common.Hash{0xb}: { + common.Hash{0x1}: {0x10}, + }, + common.Hash{0xc}: { + common.Hash{0x1}: {0x10}, + }, + }, + ) + b := newStates( + map[common.Hash][]byte{ + common.Hash{0xa}: {0xa1}, + common.Hash{0xb}: {0xb1}, + common.Hash{0xc}: nil, + }, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0xa}: { + common.Hash{0x1}: {0x11}, + common.Hash{0x2}: nil, + common.Hash{0x3}: {0x31}, + }, + common.Hash{0xb}: { + common.Hash{0x1}: {0x11}, + }, + common.Hash{0xc}: { + common.Hash{0x1}: nil, + }, + }, + ) + a.merge(b) + a.revertTo( + map[common.Hash][]byte{ + common.Hash{0xa}: {0xa0}, + common.Hash{0xb}: {0xb0}, + common.Hash{0xc}: {0xc0}, + }, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0xa}: { + common.Hash{0x1}: {0x10}, + common.Hash{0x2}: {0x20}, + common.Hash{0x3}: nil, + }, + common.Hash{0xb}: { + common.Hash{0x1}: {0x10}, + }, + common.Hash{0xc}: { + common.Hash{0x1}: {0x10}, + }, + }, + ) + + blob, exist := a.account(common.Hash{0xa}) + if !exist || !bytes.Equal(blob, []byte{0xa0}) { + t.Error("Unexpected value for account a") + } + blob, exist = a.account(common.Hash{0xb}) + if !exist || !bytes.Equal(blob, []byte{0xb0}) { + t.Error("Unexpected value for account b") + } + blob, exist = a.account(common.Hash{0xc}) + if !exist || !bytes.Equal(blob, []byte{0xc0}) { + t.Error("Unexpected value for account c") + } + // unknown account + blob, exist = a.account(common.Hash{0xd}) + if exist || len(blob) != 0 { + t.Error("Unexpected value for account d") + } + + blob, exist = a.storage(common.Hash{0xa}, common.Hash{0x1}) + if !exist || !bytes.Equal(blob, []byte{0x10}) { + t.Error("Unexpected value for a's storage") + } + blob, exist = a.storage(common.Hash{0xa}, common.Hash{0x2}) + if !exist || !bytes.Equal(blob, []byte{0x20}) { + t.Error("Unexpected value for a's storage") + } + blob, exist = a.storage(common.Hash{0xa}, common.Hash{0x3}) + if !exist || len(blob) != 0 { + t.Error("Unexpected value for a's storage") + } + blob, exist = a.storage(common.Hash{0xb}, common.Hash{0x1}) + if !exist || !bytes.Equal(blob, []byte{0x10}) { + t.Error("Unexpected value for b's storage") + } + blob, exist = a.storage(common.Hash{0xc}, common.Hash{0x1}) + if !exist || !bytes.Equal(blob, []byte{0x10}) { + t.Error("Unexpected value for c's storage") + } + // unknown storage slots + blob, exist = a.storage(common.Hash{0xd}, common.Hash{0x1}) + if exist || len(blob) != 0 { + t.Error("Unexpected value for d's storage") + } +} + +// TestStateRevertAccountNullMarker tests the scenario that account x did not exist +// before and was created during transition w, reverting w will retain an x=nil +// entry in the set. +func TestStateRevertAccountNullMarker(t *testing.T) { + a := newStates(nil, nil) // empty initial state + b := newStates( + map[common.Hash][]byte{ + common.Hash{0xa}: {0xa}, + }, + nil, + ) + a.merge(b) // create account 0xa + a.revertTo( + map[common.Hash][]byte{ + common.Hash{0xa}: nil, + }, + nil, + ) // revert the transition b + + blob, exist := a.account(common.Hash{0xa}) + if !exist { + t.Fatal("null marker is not found") + } + if len(blob) != 0 { + t.Fatalf("Unexpected value for account, %v", blob) + } +} + +// TestStateRevertStorageNullMarker tests the scenario that slot x did not exist +// before and was created during transition w, reverting w will retain an x=nil +// entry in the set. +func TestStateRevertStorageNullMarker(t *testing.T) { + a := newStates(map[common.Hash][]byte{ + common.Hash{0xa}: {0xa}, + }, nil) // initial state with account 0xa + + b := newStates( + nil, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0xa}: { + common.Hash{0x1}: {0x1}, + }, + }, + ) + a.merge(b) // create slot 0x1 + a.revertTo( + nil, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0xa}: { + common.Hash{0x1}: nil, + }, + }, + ) // revert the transition b + + blob, exist := a.storage(common.Hash{0xa}, common.Hash{0x1}) + if !exist { + t.Fatal("null marker is not found") + } + if len(blob) != 0 { + t.Fatalf("Unexpected value for storage slot, %v", blob) + } +} + +func TestStatesEncode(t *testing.T) { + s := newStates( + map[common.Hash][]byte{ + common.Hash{0x1}: {0x1}, + }, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0x1}: { + common.Hash{0x1}: {0x1}, + }, + }, + ) + buf := bytes.NewBuffer(nil) + if err := s.encode(buf); err != nil { + t.Fatalf("Failed to encode states, %v", err) + } + var dec stateSet + if err := dec.decode(rlp.NewStream(buf, 0)); err != nil { + t.Fatalf("Failed to decode states, %v", err) + } + if !reflect.DeepEqual(s.accountData, dec.accountData) { + t.Fatal("Unexpected account data") + } + if !reflect.DeepEqual(s.storageData, dec.storageData) { + t.Fatal("Unexpected storage data") + } +} + +func TestStateWithOriginEncode(t *testing.T) { + s := NewStateSetWithOrigin( + map[common.Hash][]byte{ + common.Hash{0x1}: {0x1}, + }, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0x1}: { + common.Hash{0x1}: {0x1}, + }, + }, + map[common.Address][]byte{ + common.Address{0x1}: {0x1}, + }, + map[common.Address]map[common.Hash][]byte{ + common.Address{0x1}: { + common.Hash{0x1}: {0x1}, + }, + }, + ) + buf := bytes.NewBuffer(nil) + if err := s.encode(buf); err != nil { + t.Fatalf("Failed to encode states, %v", err) + } + var dec StateSetWithOrigin + if err := dec.decode(rlp.NewStream(buf, 0)); err != nil { + t.Fatalf("Failed to decode states, %v", err) + } + if !reflect.DeepEqual(s.accountData, dec.accountData) { + t.Fatal("Unexpected account data") + } + if !reflect.DeepEqual(s.storageData, dec.storageData) { + t.Fatal("Unexpected storage data") + } + if !reflect.DeepEqual(s.accountOrigin, dec.accountOrigin) { + t.Fatal("Unexpected account origin data") + } + if !reflect.DeepEqual(s.storageOrigin, dec.storageOrigin) { + t.Fatal("Unexpected storage origin data") + } +} + +func TestStateSizeTracking(t *testing.T) { + expSizeA := 3*(common.HashLength+1) + /* account data */ + 2*(2*common.HashLength+1) + /* storage data of 0xa */ + 2*common.HashLength + 3 + /* storage data of 0xb */ + 2*common.HashLength + 1 /* storage data of 0xc */ + + a := newStates( + map[common.Hash][]byte{ + common.Hash{0xa}: {0xa0}, // common.HashLength+1 + common.Hash{0xb}: {0xb0}, // common.HashLength+1 + common.Hash{0xc}: {0xc0}, // common.HashLength+1 + }, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0xa}: { + common.Hash{0x1}: {0x10}, // 2*common.HashLength+1 + common.Hash{0x2}: {0x20}, // 2*common.HashLength+1 + }, + common.Hash{0xb}: { + common.Hash{0x1}: {0x10, 0x11, 0x12}, // 2*common.HashLength+3 + }, + common.Hash{0xc}: { + common.Hash{0x1}: {0x10}, // 2*common.HashLength+1 + }, + }, + ) + if a.size != uint64(expSizeA) { + t.Fatalf("Unexpected size, want: %d, got: %d", expSizeA, a.size) + } + + expSizeB := common.HashLength + 2 + common.HashLength + 3 + common.HashLength + /* account data */ + 2*common.HashLength + 3 + 2*common.HashLength + 2 + /* storage data of 0xa */ + 2*common.HashLength + 2 + 2*common.HashLength + 2 + /* storage data of 0xb */ + 3*2*common.HashLength /* storage data of 0xc */ + b := newStates( + map[common.Hash][]byte{ + common.Hash{0xa}: {0xa1, 0xa1}, // common.HashLength+2 + common.Hash{0xb}: {0xb1, 0xb1, 0xb1}, // common.HashLength+3 + common.Hash{0xc}: nil, // common.HashLength, account deletion + }, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0xa}: { + common.Hash{0x1}: {0x11, 0x11, 0x11}, // 2*common.HashLength+3 + common.Hash{0x3}: {0x31, 0x31}, // 2*common.HashLength+2, slot creation + }, + common.Hash{0xb}: { + common.Hash{0x1}: {0x11, 0x11}, // 2*common.HashLength+2 + common.Hash{0x2}: {0x22, 0x22}, // 2*common.HashLength+2, slot creation + }, + // The storage of 0xc is entirely removed + common.Hash{0xc}: { + common.Hash{0x1}: nil, // 2*common.HashLength, slot deletion + common.Hash{0x2}: nil, // 2*common.HashLength, slot deletion + common.Hash{0x3}: nil, // 2*common.HashLength, slot deletion + }, + }, + ) + if b.size != uint64(expSizeB) { + t.Fatalf("Unexpected size, want: %d, got: %d", expSizeB, b.size) + } + + a.merge(b) + mergeSize := expSizeA + 1 /* account a data change */ + 2 /* account b data change */ - 1 /* account c data change */ + mergeSize += 2*common.HashLength + 2 + 2 /* storage a change */ + mergeSize += 2*common.HashLength + 2 - 1 /* storage b change */ + mergeSize += 2*2*common.HashLength - 1 /* storage data removal of 0xc */ + + if a.size != uint64(mergeSize) { + t.Fatalf("Unexpected size, want: %d, got: %d", mergeSize, a.size) + } + + // Revert the set to original status + a.revertTo( + map[common.Hash][]byte{ + common.Hash{0xa}: {0xa0}, + common.Hash{0xb}: {0xb0}, + common.Hash{0xc}: {0xc0}, + }, + map[common.Hash]map[common.Hash][]byte{ + common.Hash{0xa}: { + common.Hash{0x1}: {0x10}, + common.Hash{0x2}: {0x20}, + common.Hash{0x3}: nil, // revert slot creation + }, + common.Hash{0xb}: { + common.Hash{0x1}: {0x10, 0x11, 0x12}, + common.Hash{0x2}: nil, // revert slot creation + }, + common.Hash{0xc}: { + common.Hash{0x1}: {0x10}, + common.Hash{0x2}: {0x20}, // resurrected slot + common.Hash{0x3}: {0x30}, // resurrected slot + }, + }, + ) + revertSize := expSizeA + 2*common.HashLength + 2*common.HashLength // delete-marker of a.3 and b.2 slot + revertSize += 2 * (2*common.HashLength + 1) // resurrected slot, c.2, c.3 + if a.size != uint64(revertSize) { + t.Fatalf("Unexpected size, want: %d, got: %d", revertSize, a.size) + } +} diff --git a/triedb/states.go b/triedb/states.go index 0b03f2b9f3..fa432e0704 100644 --- a/triedb/states.go +++ b/triedb/states.go @@ -45,5 +45,5 @@ func (set *StateSet) internal() *pathdb.StateSetWithOrigin { if set == nil { return nil } - return pathdb.NewStateSetWithOrigin(set.AccountsOrigin, set.StoragesOrigin) + return pathdb.NewStateSetWithOrigin(set.Accounts, set.Storages, set.AccountsOrigin, set.StoragesOrigin) } From 03c37cdb2b78942d2ca16be4016e090cf7f12356 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 29 Nov 2024 22:32:45 +0800 Subject: [PATCH 07/17] core/state: introduce code reader interface (#30816) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR introduces a `ContractCodeReader` interface with functions defined: type ContractCodeReader interface { Code(addr common.Address, codeHash common.Hash) ([]byte, error) CodeSize(addr common.Address, codeHash common.Hash) (int, error) } This interface can be implemented in various ways. Although the codebase currently includes only one implementation, additional implementations could be created for different purposes and scenarios, such as a code reader designed for the Verkle tree approach or one that reads code from the witness. *Notably, this interface modifies the function’s semantics. If the contract code is not found, no error will be returned. An error should only be returned in the event of an unexpected issue, primarily for future implementations.* The original state.Reader interface is extended with ContractCodeReader methods, it gives us more flexibility to manipulate the reader with additional logic on top, e.g. Hooks. type Reader interface { ContractCodeReader StateReader } --------- Co-authored-by: Felix Lange --- core/blockchain_reader.go | 5 +- core/state/database.go | 48 ++------- core/state/iterator.go | 5 +- core/state/reader.go | 182 ++++++++++++++++++++++------------ core/state/state_object.go | 10 +- core/state/statedb.go | 3 +- core/state/sync_test.go | 45 +++++++-- eth/protocols/snap/handler.go | 2 +- 8 files changed, 178 insertions(+), 122 deletions(-) diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 8f1da82a2b..cf72e98cb2 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -344,10 +344,7 @@ func (bc *BlockChain) stateRecoverable(root common.Hash) bool { // ContractCodeWithPrefix retrieves a blob of data associated with a contract // hash either from ephemeral in-memory cache, or from persistent storage. -// -// If the code doesn't exist in the in-memory cache, check the storage with -// new code scheme. -func (bc *BlockChain) ContractCodeWithPrefix(hash common.Hash) ([]byte, error) { +func (bc *BlockChain) ContractCodeWithPrefix(hash common.Hash) []byte { // TODO(rjl493456442) The associated account address is also required // in Verkle scheme. Fix it once snap-sync is supported for Verkle. return bc.statedb.ContractCodeWithPrefix(common.Address{}, hash) diff --git a/core/state/database.go b/core/state/database.go index 0d8acec35a..fff7f1519f 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -17,7 +17,6 @@ package state import ( - "errors" "fmt" "github.com/ethereum/go-ethereum/common" @@ -55,12 +54,6 @@ type Database interface { // OpenStorageTrie opens the storage trie of an account. OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, trie Trie) (Trie, error) - // ContractCode retrieves a particular contract's code. - ContractCode(addr common.Address, codeHash common.Hash) ([]byte, error) - - // ContractCodeSize retrieves a particular contracts code's size. - ContractCodeSize(addr common.Address, codeHash common.Hash) (int, error) - // PointCache returns the cache holding points used in verkle tree key computation PointCache() *utils.PointCache @@ -180,7 +173,7 @@ func NewDatabaseForTesting() *CachingDB { // Reader returns a state reader associated with the specified state root. func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { - var readers []Reader + var readers []StateReader // Set up the state snapshot reader if available. This feature // is optional and may be partially useful if it's not fully @@ -188,7 +181,7 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { if db.snap != nil { snap := db.snap.Snapshot(stateRoot) if snap != nil { - readers = append(readers, newStateReader(snap)) // snap reader is optional + readers = append(readers, newFlatReader(snap)) } } // Set up the trie reader, which is expected to always be available @@ -199,7 +192,11 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { } readers = append(readers, tr) - return newMultiReader(readers...) + combined, err := newMultiStateReader(readers...) + if err != nil { + return nil, err + } + return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), combined), nil } // OpenTrie opens the main account trie at a specific root hash. @@ -229,45 +226,20 @@ func (db *CachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Addre return tr, nil } -// ContractCode retrieves a particular contract's code. -func (db *CachingDB) ContractCode(address common.Address, codeHash common.Hash) ([]byte, error) { - code, _ := db.codeCache.Get(codeHash) - if len(code) > 0 { - return code, nil - } - code = rawdb.ReadCode(db.disk, codeHash) - if len(code) > 0 { - db.codeCache.Add(codeHash, code) - db.codeSizeCache.Add(codeHash, len(code)) - return code, nil - } - return nil, errors.New("not found") -} - // ContractCodeWithPrefix retrieves a particular contract's code. If the // code can't be found in the cache, then check the existence with **new** // db scheme. -func (db *CachingDB) ContractCodeWithPrefix(address common.Address, codeHash common.Hash) ([]byte, error) { +func (db *CachingDB) ContractCodeWithPrefix(address common.Address, codeHash common.Hash) []byte { code, _ := db.codeCache.Get(codeHash) if len(code) > 0 { - return code, nil + return code } code = rawdb.ReadCodeWithPrefix(db.disk, codeHash) if len(code) > 0 { db.codeCache.Add(codeHash, code) db.codeSizeCache.Add(codeHash, len(code)) - return code, nil } - return nil, errors.New("not found") -} - -// ContractCodeSize retrieves a particular contracts code's size. -func (db *CachingDB) ContractCodeSize(addr common.Address, codeHash common.Hash) (int, error) { - if cached, ok := db.codeSizeCache.Get(codeHash); ok { - return cached, nil - } - code, err := db.ContractCode(addr, codeHash) - return len(code), err + return code } // TrieDB retrieves any intermediate trie-node caching layer. diff --git a/core/state/iterator.go b/core/state/iterator.go index 83c552ca1a..5ea52c6183 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -136,10 +136,13 @@ func (it *nodeIterator) step() error { } if !bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) { it.codeHash = common.BytesToHash(account.CodeHash) - it.code, err = it.state.db.ContractCode(address, common.BytesToHash(account.CodeHash)) + it.code, err = it.state.reader.Code(address, common.BytesToHash(account.CodeHash)) if err != nil { return fmt.Errorf("code %x: %v", account.CodeHash, err) } + if len(it.code) == 0 { + return fmt.Errorf("code is not found: %x", account.CodeHash) + } } it.accountHash = it.stateIt.Parent() return nil diff --git a/core/state/reader.go b/core/state/reader.go index 85842adde8..a0f15dfcc8 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -18,11 +18,13 @@ package state import ( "errors" - "maps" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/utils" @@ -30,9 +32,26 @@ import ( "github.com/ethereum/go-ethereum/triedb/database" ) -// Reader defines the interface for accessing accounts and storage slots +// ContractCodeReader defines the interface for accessing contract code. +type ContractCodeReader interface { + // Code retrieves a particular contract's code. + // + // - Returns nil code along with nil error if the requested contract code + // doesn't exist + // - Returns an error only if an unexpected issue occurs + Code(addr common.Address, codeHash common.Hash) ([]byte, error) + + // CodeSize retrieves a particular contracts code's size. + // + // - Returns zero code size along with nil error if the requested contract code + // doesn't exist + // - Returns an error only if an unexpected issue occurs + CodeSize(addr common.Address, codeHash common.Hash) (int, error) +} + +// StateReader defines the interface for accessing accounts and storage slots // associated with a specific state. -type Reader interface { +type StateReader interface { // Account retrieves the account associated with a particular address. // // - Returns a nil account if it does not exist @@ -47,32 +66,84 @@ type Reader interface { // - Returns an error only if an unexpected issue occurs // - The returned storage slot is safe to modify after the call Storage(addr common.Address, slot common.Hash) (common.Hash, error) - - // Copy returns a deep-copied state reader. - Copy() Reader } -// stateReader wraps a database state reader. -type stateReader struct { +// Reader defines the interface for accessing accounts, storage slots and contract +// code associated with a specific state. +type Reader interface { + ContractCodeReader + StateReader +} + +// cachingCodeReader implements ContractCodeReader, accessing contract code either in +// local key-value store or the shared code cache. +type cachingCodeReader struct { + db ethdb.KeyValueReader + + // These caches could be shared by multiple code reader instances, + // they are natively thread-safe. + codeCache *lru.SizeConstrainedCache[common.Hash, []byte] + codeSizeCache *lru.Cache[common.Hash, int] +} + +// newCachingCodeReader constructs the code reader. +func newCachingCodeReader(db ethdb.KeyValueReader, codeCache *lru.SizeConstrainedCache[common.Hash, []byte], codeSizeCache *lru.Cache[common.Hash, int]) *cachingCodeReader { + return &cachingCodeReader{ + db: db, + codeCache: codeCache, + codeSizeCache: codeSizeCache, + } +} + +// Code implements ContractCodeReader, retrieving a particular contract's code. +// If the contract code doesn't exist, no error will be returned. +func (r *cachingCodeReader) Code(addr common.Address, codeHash common.Hash) ([]byte, error) { + code, _ := r.codeCache.Get(codeHash) + if len(code) > 0 { + return code, nil + } + code = rawdb.ReadCode(r.db, codeHash) + if len(code) > 0 { + r.codeCache.Add(codeHash, code) + r.codeSizeCache.Add(codeHash, len(code)) + } + return code, nil +} + +// CodeSize implements ContractCodeReader, retrieving a particular contracts code's size. +// If the contract code doesn't exist, no error will be returned. +func (r *cachingCodeReader) CodeSize(addr common.Address, codeHash common.Hash) (int, error) { + if cached, ok := r.codeSizeCache.Get(codeHash); ok { + return cached, nil + } + code, err := r.Code(addr, codeHash) + if err != nil { + return 0, err + } + return len(code), nil +} + +// flatReader wraps a database state reader. +type flatReader struct { reader database.StateReader buff crypto.KeccakState } -// newStateReader constructs a state reader with on the given state root. -func newStateReader(reader database.StateReader) *stateReader { - return &stateReader{ +// newFlatReader constructs a state reader with on the given state root. +func newFlatReader(reader database.StateReader) *flatReader { + return &flatReader{ reader: reader, buff: crypto.NewKeccakState(), } } -// Account implements Reader, retrieving the account specified by the address. +// Account implements StateReader, retrieving the account specified by the address. // // An error will be returned if the associated snapshot is already stale or // the requested account is not yet covered by the snapshot. // // The returned account might be nil if it's not existent. -func (r *stateReader) Account(addr common.Address) (*types.StateAccount, error) { +func (r *flatReader) Account(addr common.Address) (*types.StateAccount, error) { account, err := r.reader.Account(crypto.HashData(r.buff, addr.Bytes())) if err != nil { return nil, err @@ -95,14 +166,14 @@ func (r *stateReader) Account(addr common.Address) (*types.StateAccount, error) return acct, nil } -// Storage implements Reader, retrieving the storage slot specified by the +// Storage implements StateReader, retrieving the storage slot specified by the // address and slot key. // // An error will be returned if the associated snapshot is already stale or // the requested storage slot is not yet covered by the snapshot. // // The returned storage slot might be empty if it's not existent. -func (r *stateReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { +func (r *flatReader) Storage(addr common.Address, key common.Hash) (common.Hash, error) { addrHash := crypto.HashData(r.buff, addr.Bytes()) slotHash := crypto.HashData(r.buff, key.Bytes()) ret, err := r.reader.Storage(addrHash, slotHash) @@ -123,15 +194,7 @@ func (r *stateReader) Storage(addr common.Address, key common.Hash) (common.Hash return value, nil } -// Copy implements Reader, returning a deep-copied snap reader. -func (r *stateReader) Copy() Reader { - return &stateReader{ - reader: r.reader, - buff: crypto.NewKeccakState(), - } -} - -// trieReader implements the Reader interface, providing functions to access +// trieReader implements the StateReader interface, providing functions to access // state from the referenced trie. type trieReader struct { root common.Hash // State root which uniquely represent a state @@ -167,7 +230,7 @@ func newTrieReader(root common.Hash, db *triedb.Database, cache *utils.PointCach }, nil } -// Account implements Reader, retrieving the account specified by the address. +// Account implements StateReader, retrieving the account specified by the address. // // An error will be returned if the trie state is corrupted. An nil account // will be returned if it's not existent in the trie. @@ -184,7 +247,7 @@ func (r *trieReader) Account(addr common.Address) (*types.StateAccount, error) { return account, nil } -// Storage implements Reader, retrieving the storage slot specified by the +// Storage implements StateReader, retrieving the storage slot specified by the // address and slot key. // // An error will be returned if the trie state is corrupted. An empty storage @@ -227,48 +290,32 @@ func (r *trieReader) Storage(addr common.Address, key common.Hash) (common.Hash, return value, nil } -// Copy implements Reader, returning a deep-copied trie reader. -func (r *trieReader) Copy() Reader { - tries := make(map[common.Address]Trie) - for addr, tr := range r.subTries { - tries[addr] = mustCopyTrie(tr) - } - return &trieReader{ - root: r.root, - db: r.db, - buff: crypto.NewKeccakState(), - mainTrie: mustCopyTrie(r.mainTrie), - subRoots: maps.Clone(r.subRoots), - subTries: tries, - } +// multiStateReader is the aggregation of a list of StateReader interface, +// providing state access by leveraging all readers. The checking priority +// is determined by the position in the reader list. +type multiStateReader struct { + readers []StateReader // List of state readers, sorted by checking priority } -// multiReader is the aggregation of a list of Reader interface, providing state -// access by leveraging all readers. The checking priority is determined by the -// position in the reader list. -type multiReader struct { - readers []Reader // List of readers, sorted by checking priority -} - -// newMultiReader constructs a multiReader instance with the given readers. The -// priority among readers is assumed to be sorted. Note, it must contain at least -// one reader for constructing a multiReader. -func newMultiReader(readers ...Reader) (*multiReader, error) { +// newMultiStateReader constructs a multiStateReader instance with the given +// readers. The priority among readers is assumed to be sorted. Note, it must +// contain at least one reader for constructing a multiStateReader. +func newMultiStateReader(readers ...StateReader) (*multiStateReader, error) { if len(readers) == 0 { return nil, errors.New("empty reader set") } - return &multiReader{ + return &multiStateReader{ readers: readers, }, nil } -// Account implementing Reader interface, retrieving the account associated with -// a particular address. +// Account implementing StateReader interface, retrieving the account associated +// with a particular address. // // - Returns a nil account if it does not exist // - Returns an error only if an unexpected issue occurs // - The returned account is safe to modify after the call -func (r *multiReader) Account(addr common.Address) (*types.StateAccount, error) { +func (r *multiStateReader) Account(addr common.Address) (*types.StateAccount, error) { var errs []error for _, reader := range r.readers { acct, err := reader.Account(addr) @@ -280,13 +327,13 @@ func (r *multiReader) Account(addr common.Address) (*types.StateAccount, error) return nil, errors.Join(errs...) } -// Storage implementing Reader interface, retrieving the storage slot associated -// with a particular account address and slot key. +// Storage implementing StateReader interface, retrieving the storage slot +// associated with a particular account address and slot key. // // - Returns an empty slot if it does not exist // - Returns an error only if an unexpected issue occurs // - The returned storage slot is safe to modify after the call -func (r *multiReader) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { +func (r *multiStateReader) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { var errs []error for _, reader := range r.readers { slot, err := reader.Storage(addr, slot) @@ -298,11 +345,16 @@ func (r *multiReader) Storage(addr common.Address, slot common.Hash) (common.Has return common.Hash{}, errors.Join(errs...) } -// Copy implementing Reader interface, returning a deep-copied state reader. -func (r *multiReader) Copy() Reader { - var readers []Reader - for _, reader := range r.readers { - readers = append(readers, reader.Copy()) - } - return &multiReader{readers: readers} +// reader is the wrapper of ContractCodeReader and StateReader interface. +type reader struct { + ContractCodeReader + StateReader +} + +// newReader constructs a reader with the supplied code reader and state reader. +func newReader(codeReader ContractCodeReader, stateReader StateReader) *reader { + return &reader{ + ContractCodeReader: codeReader, + StateReader: stateReader, + } } diff --git a/core/state/state_object.go b/core/state/state_object.go index b659bf7ff2..2d542e5005 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -510,10 +510,13 @@ func (s *stateObject) Code() []byte { if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return nil } - code, err := s.db.db.ContractCode(s.address, common.BytesToHash(s.CodeHash())) + code, err := s.db.reader.Code(s.address, common.BytesToHash(s.CodeHash())) if err != nil { s.db.setError(fmt.Errorf("can't load code hash %x: %v", s.CodeHash(), err)) } + if len(code) == 0 { + s.db.setError(fmt.Errorf("code is not found %x", s.CodeHash())) + } s.code = code return code } @@ -528,10 +531,13 @@ func (s *stateObject) CodeSize() int { if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return 0 } - size, err := s.db.db.ContractCodeSize(s.address, common.BytesToHash(s.CodeHash())) + size, err := s.db.reader.CodeSize(s.address, common.BytesToHash(s.CodeHash())) if err != nil { s.db.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err)) } + if size == 0 { + s.db.setError(fmt.Errorf("code is not found %x", s.CodeHash())) + } return size } diff --git a/core/state/statedb.go b/core/state/statedb.go index 9cc91c9332..b0603db7f0 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -650,10 +650,11 @@ func (s *StateDB) CreateContract(addr common.Address) { // Snapshots of the copied state cannot be applied to the copy. func (s *StateDB) Copy() *StateDB { // Copy all the basic fields, initialize the memory ones + reader, _ := s.db.Reader(s.originalRoot) // impossible to fail state := &StateDB{ db: s.db, trie: mustCopyTrie(s.trie), - reader: s.reader.Copy(), + reader: reader, originalRoot: s.originalRoot, stateObjects: make(map[common.Address]*stateObject, len(s.stateObjects)), stateObjectsDestruct: make(map[common.Address]*stateObject, len(s.stateObjectsDestruct)), diff --git a/core/state/sync_test.go b/core/state/sync_test.go index b2c75e72fe..efa56f8860 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -210,14 +210,18 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, s if err != nil { t.Fatalf("state is not existent, %#x", srcRoot) } + cReader, err := srcDb.Reader(srcRoot) + if err != nil { + t.Fatalf("state is not existent, %#x", srcRoot) + } for len(nodeElements)+len(codeElements) > 0 { var ( nodeResults = make([]trie.NodeSyncResult, len(nodeElements)) codeResults = make([]trie.CodeSyncResult, len(codeElements)) ) for i, element := range codeElements { - data, err := srcDb.ContractCode(common.Address{}, element.code) - if err != nil { + data, err := cReader.Code(common.Address{}, element.code) + if err != nil || len(data) == 0 { t.Fatalf("failed to retrieve contract bytecode for hash %x", element.code) } codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} @@ -329,6 +333,10 @@ func testIterativeDelayedStateSync(t *testing.T, scheme string) { if err != nil { t.Fatalf("state is not existent, %#x", srcRoot) } + cReader, err := srcDb.Reader(srcRoot) + if err != nil { + t.Fatalf("state is not existent, %#x", srcRoot) + } for len(nodeElements)+len(codeElements) > 0 { // Sync only half of the scheduled nodes var nodeProcessed int @@ -336,8 +344,8 @@ func testIterativeDelayedStateSync(t *testing.T, scheme string) { if len(codeElements) > 0 { codeResults := make([]trie.CodeSyncResult, len(codeElements)/2+1) for i, element := range codeElements[:len(codeResults)] { - data, err := srcDb.ContractCode(common.Address{}, element.code) - if err != nil { + data, err := cReader.Code(common.Address{}, element.code) + if err != nil || len(data) == 0 { t.Fatalf("failed to retrieve contract bytecode for %x", element.code) } codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} @@ -433,13 +441,17 @@ func testIterativeRandomStateSync(t *testing.T, count int, scheme string) { if err != nil { t.Fatalf("state is not existent, %#x", srcRoot) } + cReader, err := srcDb.Reader(srcRoot) + if err != nil { + t.Fatalf("state is not existent, %#x", srcRoot) + } for len(nodeQueue)+len(codeQueue) > 0 { // Fetch all the queued nodes in a random order if len(codeQueue) > 0 { results := make([]trie.CodeSyncResult, 0, len(codeQueue)) for hash := range codeQueue { - data, err := srcDb.ContractCode(common.Address{}, hash) - if err != nil { + data, err := cReader.Code(common.Address{}, hash) + if err != nil || len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) @@ -526,6 +538,10 @@ func testIterativeRandomDelayedStateSync(t *testing.T, scheme string) { if err != nil { t.Fatalf("state is not existent, %#x", srcRoot) } + cReader, err := srcDb.Reader(srcRoot) + if err != nil { + t.Fatalf("state is not existent, %#x", srcRoot) + } for len(nodeQueue)+len(codeQueue) > 0 { // Sync only half of the scheduled nodes, even those in random order if len(codeQueue) > 0 { @@ -533,8 +549,8 @@ func testIterativeRandomDelayedStateSync(t *testing.T, scheme string) { for hash := range codeQueue { delete(codeQueue, hash) - data, err := srcDb.ContractCode(common.Address{}, hash) - if err != nil { + data, err := cReader.Code(common.Address{}, hash) + if err != nil || len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) @@ -631,6 +647,10 @@ func testIncompleteStateSync(t *testing.T, scheme string) { if err != nil { t.Fatalf("state is not available %x", srcRoot) } + cReader, err := srcDb.Reader(srcRoot) + if err != nil { + t.Fatalf("state is not existent, %#x", srcRoot) + } nodeQueue := make(map[string]stateElement) codeQueue := make(map[common.Hash]struct{}) paths, nodes, codes := sched.Missing(1) @@ -649,8 +669,8 @@ func testIncompleteStateSync(t *testing.T, scheme string) { if len(codeQueue) > 0 { results := make([]trie.CodeSyncResult, 0, len(codeQueue)) for hash := range codeQueue { - data, err := srcDb.ContractCode(common.Address{}, hash) - if err != nil { + data, err := cReader.Code(common.Address{}, hash) + if err != nil || len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) @@ -713,6 +733,11 @@ func testIncompleteStateSync(t *testing.T, scheme string) { // Sanity check that removing any node from the database is detected for _, node := range addedCodes { val := rawdb.ReadCode(dstDb, node) + if len(val) == 0 { + t.Logf("no code: %v", node) + } else { + t.Logf("has code: %v", node) + } rawdb.DeleteCode(dstDb, node) if err := checkStateConsistency(dstDb, ndb.Scheme(), srcRoot); err == nil { t.Errorf("trie inconsistency not caught, missing: %x", node) diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index d36f9621b1..8164973c20 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -454,7 +454,7 @@ func ServiceGetByteCodesQuery(chain *core.BlockChain, req *GetByteCodesPacket) [ // Peers should not request the empty code, but if they do, at // least sent them back a correct response without db lookups codes = append(codes, []byte{}) - } else if blob, err := chain.ContractCodeWithPrefix(hash); err == nil { + } else if blob := chain.ContractCodeWithPrefix(hash); len(blob) > 0 { codes = append(codes, blob) bytes += uint64(len(blob)) } From a793bc7f5f0e09c97a410797cfc24015c0083bae Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 29 Nov 2024 22:39:42 +0800 Subject: [PATCH 08/17] core: switch EVM tx context in ApplyMessage (#30809) This change relocates the EVM tx context switching to the ApplyMessage function. With this change, we can remove a lot of EVM.SetTxContext calls before message execution. ### Tracing API changes - This PR replaces the `GasPrice` field of the `VMContext` struct with `BaseFee`. Users may instead take the effective gas price from `tx.EffectiveGasTipValue(env.BaseFee)`. --------- Co-authored-by: Sina Mahmoodi --- cmd/evm/internal/t8ntool/execution.go | 7 ++-- core/block_validator.go | 2 +- core/state_prefetcher.go | 16 +++------ core/state_processor.go | 6 ---- core/state_transition.go | 29 ++++++++-------- core/tracing/hooks.go | 5 ++- core/vm/evm.go | 2 +- eth/gasestimator/gasestimator.go | 8 ++--- eth/state_accessor.go | 2 -- eth/tracers/api.go | 20 ++++------- eth/tracers/api_test.go | 2 -- .../internal/tracetest/calltrace_test.go | 33 ++++++++----------- .../internal/tracetest/flat_calltrace_test.go | 1 - .../internal/tracetest/prestate_test.go | 1 - eth/tracers/js/goja.go | 2 +- eth/tracers/js/tracer_test.go | 4 +-- eth/tracers/tracers_test.go | 8 +---- internal/ethapi/api.go | 11 +++---- internal/ethapi/simulate.go | 1 - tests/state_test_util.go | 2 -- 20 files changed, 57 insertions(+), 105 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 6034b0d33a..7a0de86a11 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -253,16 +253,13 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, statedb.SetTxContext(tx.Hash(), txIndex) var ( - txContext = core.NewEVMTxContext(msg) - snapshot = statedb.Snapshot() - prevGas = gaspool.Gas() + snapshot = statedb.Snapshot() + prevGas = gaspool.Gas() ) if tracer != nil && tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } // (ret []byte, usedGas uint64, failed bool, err error) - - evm.SetTxContext(txContext) msgResult, err := core.ApplyMessage(evm, msg, gaspool) if err != nil { statedb.RevertToSnapshot(snapshot) diff --git a/core/block_validator.go b/core/block_validator.go index 59783a0407..5885df9ee2 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -93,7 +93,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { } // The individual checks for blob validity (version-check + not empty) - // happens in StateTransition. + // happens in state transition. } // Check blob gas usage. diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index 3b987bb289..805df5ef62 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -65,7 +65,10 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c return // Also invalid block, bail out } statedb.SetTxContext(tx.Hash(), i) - if err := precacheTransaction(msg, gaspool, evm); err != nil { + + // We attempt to apply a transaction. The goal is not to execute + // the transaction successfully, rather to warm up touched data slots. + if _, err := ApplyMessage(evm, msg, gaspool); err != nil { return // Ugh, something went horribly wrong, bail out } // If we're pre-byzantium, pre-load trie nodes for the intermediate root @@ -78,14 +81,3 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c statedb.IntermediateRoot(true) } } - -// precacheTransaction attempts to apply a transaction to the given state database -// and uses the input parameters for its environment. The goal is not to execute -// the transaction successfully, rather to warm up touched data slots. -func precacheTransaction(msg *Message, gaspool *GasPool, evm *vm.EVM) error { - // Update the evm with the new transaction context. - evm.SetTxContext(NewEVMTxContext(msg)) - // Add addresses to access list if applicable - _, err := ApplyMessage(evm, msg, gaspool) - return err -} diff --git a/core/state_processor.go b/core/state_processor.go index 9472ab638b..d8637a2a52 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -140,17 +140,11 @@ func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, defer func() { hooks.OnTxEnd(receipt, err) }() } } - - // Create a new context to be used in the EVM environment. - txContext := NewEVMTxContext(msg) - evm.SetTxContext(txContext) - // Apply the transaction to the current state (included in the env). result, err := ApplyMessage(evm, msg, gp) if err != nil { return nil, err } - // Update the state with pending changes. var root []byte if evm.ChainConfig().IsByzantium(blockNumber) { diff --git a/core/state_transition.go b/core/state_transition.go index 4bd3c00167..ea7e3df2ff 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -187,10 +187,11 @@ func TransactionToMessage(tx *types.Transaction, s types.Signer, baseFee *big.In // indicates a core error meaning that the message would always fail for that particular // state and would never be accepted within a block. func ApplyMessage(evm *vm.EVM, msg *Message, gp *GasPool) (*ExecutionResult, error) { - return NewStateTransition(evm, msg, gp).TransitionDb() + evm.SetTxContext(NewEVMTxContext(msg)) + return newStateTransition(evm, msg, gp).execute() } -// StateTransition represents a state transition. +// stateTransition represents a state transition. // // == The State Transitioning Model // @@ -212,7 +213,7 @@ func ApplyMessage(evm *vm.EVM, msg *Message, gp *GasPool) (*ExecutionResult, err // // 5. Run Script section // 6. Derive new state root -type StateTransition struct { +type stateTransition struct { gp *GasPool msg *Message gasRemaining uint64 @@ -221,9 +222,9 @@ type StateTransition struct { evm *vm.EVM } -// NewStateTransition initialises and returns a new state transition object. -func NewStateTransition(evm *vm.EVM, msg *Message, gp *GasPool) *StateTransition { - return &StateTransition{ +// newStateTransition initialises and returns a new state transition object. +func newStateTransition(evm *vm.EVM, msg *Message, gp *GasPool) *stateTransition { + return &stateTransition{ gp: gp, evm: evm, msg: msg, @@ -232,14 +233,14 @@ func NewStateTransition(evm *vm.EVM, msg *Message, gp *GasPool) *StateTransition } // to returns the recipient of the message. -func (st *StateTransition) to() common.Address { +func (st *stateTransition) to() common.Address { if st.msg == nil || st.msg.To == nil /* contract creation */ { return common.Address{} } return *st.msg.To } -func (st *StateTransition) buyGas() error { +func (st *stateTransition) buyGas() error { mgval := new(big.Int).SetUint64(st.msg.GasLimit) mgval.Mul(mgval, st.msg.GasPrice) balanceCheck := new(big.Int).Set(mgval) @@ -283,7 +284,7 @@ func (st *StateTransition) buyGas() error { return nil } -func (st *StateTransition) preCheck() error { +func (st *stateTransition) preCheck() error { // Only check transactions that are not fake msg := st.msg if !msg.SkipNonceChecks { @@ -368,7 +369,7 @@ func (st *StateTransition) preCheck() error { return st.buyGas() } -// TransitionDb will transition the state by applying the current message and +// execute will transition the state by applying the current message and // returning the evm execution result with following fields. // // - used gas: total gas used (including gas being refunded) @@ -378,7 +379,7 @@ func (st *StateTransition) preCheck() error { // // However if any consensus issue encountered, return the error directly with // nil evm execution result. -func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { +func (st *stateTransition) execute() (*ExecutionResult, error) { // First check this message satisfies all consensus rules before // applying the message. The rules include these clauses // @@ -493,7 +494,7 @@ func (st *StateTransition) TransitionDb() (*ExecutionResult, error) { }, nil } -func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { +func (st *stateTransition) refundGas(refundQuotient uint64) uint64 { // Apply refund counter, capped to a refund quotient refund := st.gasUsed() / refundQuotient if refund > st.state.GetRefund() { @@ -523,11 +524,11 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 { } // gasUsed returns the amount of gas used up by the state transition. -func (st *StateTransition) gasUsed() uint64 { +func (st *stateTransition) gasUsed() uint64 { return st.initialGas - st.gasRemaining } // blobGasUsed returns the amount of blob gas used by the message. -func (st *StateTransition) blobGasUsed() uint64 { +func (st *stateTransition) blobGasUsed() uint64 { return uint64(len(st.msg.BlobHashes) * params.BlobTxBlobGasPerBlob) } diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go index 0c5cae354b..5218e80c9b 100644 --- a/core/tracing/hooks.go +++ b/core/tracing/hooks.go @@ -55,9 +55,8 @@ type VMContext struct { BlockNumber *big.Int Time uint64 Random *common.Hash - // Effective tx gas price - GasPrice *big.Int - StateDB StateDB + BaseFee *big.Int + StateDB StateDB } // BlockEvent is emitted upon tracing an incoming block. diff --git a/core/vm/evm.go b/core/vm/evm.go index 34e5fa766b..07e4a272fa 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -605,7 +605,7 @@ func (evm *EVM) GetVMContext() *tracing.VMContext { BlockNumber: evm.Context.BlockNumber, Time: evm.Context.Time, Random: evm.Context.Random, - GasPrice: evm.TxContext.GasPrice, + BaseFee: evm.Context.BaseFee, StateDB: evm.StateDB, } } diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go index 5729f84278..9b24bfbf96 100644 --- a/eth/gasestimator/gasestimator.go +++ b/eth/gasestimator/gasestimator.go @@ -217,21 +217,19 @@ func execute(ctx context.Context, call *core.Message, opts *Options, gasLimit ui func run(ctx context.Context, call *core.Message, opts *Options) (*core.ExecutionResult, error) { // Assemble the call and the call context var ( - msgContext = core.NewEVMTxContext(call) evmContext = core.NewEVMBlockContext(opts.Header, opts.Chain, nil) - dirtyState = opts.State.Copy() ) // Lower the basefee to 0 to avoid breaking EVM // invariants (basefee < feecap). - if msgContext.GasPrice.Sign() == 0 { + if call.GasPrice.Sign() == 0 { evmContext.BaseFee = new(big.Int) } - if msgContext.BlobFeeCap != nil && msgContext.BlobFeeCap.BitLen() == 0 { + if call.BlobGasFeeCap != nil && call.BlobGasFeeCap.BitLen() == 0 { evmContext.BlobBaseFee = new(big.Int) } evm := vm.NewEVM(evmContext, dirtyState, opts.Config, vm.Config{NoBaseFee: true}) - evm.SetTxContext(msgContext) + // Monitor the outer context and interrupt the EVM upon cancellation. To avoid // a dangling goroutine until the outer estimation finishes, create an internal // context for the lifetime of this method call. diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 43432cff31..0749d73791 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -255,8 +255,6 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, } // Assemble the transaction call message and return if the requested offset msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) - txContext := core.NewEVMTxContext(msg) - evm.SetTxContext(txContext) // Not yet the searched for transaction, execute on top of the current state statedb.SetTxContext(tx.Hash(), idx) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index a2c11e0fe2..3416c11a67 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -546,11 +546,7 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config if err := ctx.Err(); err != nil { return nil, err } - var ( - msg, _ = core.TransactionToMessage(tx, signer, block.BaseFee()) - txContext = core.NewEVMTxContext(msg) - ) - evm.SetTxContext(txContext) + msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) statedb.SetTxContext(tx.Hash(), i) if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) @@ -708,7 +704,6 @@ txloop: // Generate the next state snapshot fast without tracing msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) statedb.SetTxContext(tx.Hash(), i) - evm.SetTxContext(core.NewEVMTxContext(msg)) if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { failed = err break txloop @@ -792,12 +787,11 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block for i, tx := range block.Transactions() { // Prepare the transaction for un-traced execution var ( - msg, _ = core.TransactionToMessage(tx, signer, block.BaseFee()) - txContext = core.NewEVMTxContext(msg) - vmConf vm.Config - dump *os.File - writer *bufio.Writer - err error + msg, _ = core.TransactionToMessage(tx, signer, block.BaseFee()) + vmConf vm.Config + dump *os.File + writer *bufio.Writer + err error ) // If the transaction needs tracing, swap out the configs if tx.Hash() == txHash || txHash == (common.Hash{}) { @@ -820,7 +814,6 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } } // Execute the transaction and flush any traces to disk - evm.SetTxContext(txContext) statedb.SetTxContext(tx.Hash(), i) if vmConf.Tracer.OnTxStart != nil { vmConf.Tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) @@ -1016,7 +1009,6 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor } // The actual TxContext will be created as part of ApplyTransactionWithEVM. evm := vm.NewEVM(vmctx, statedb, api.backend.ChainConfig(), vm.Config{Tracer: tracer.Hooks, NoBaseFee: true}) - evm.SetTxContext(vm.TxContext{GasPrice: message.GasPrice, BlobFeeCap: message.BlobGasFeeCap}) // Define a meaningful timeout of a single transaction trace if config.Timeout != nil { diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 2b5cfa35c8..2ae36287fa 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -177,8 +177,6 @@ func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block return tx, context, statedb, release, nil } msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) - txContext := core.NewEVMTxContext(msg) - evm.SetTxContext(txContext) if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index 869558c324..999ab211c0 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -133,7 +133,6 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { t.Fatalf("failed to prepare transaction for tracing: %v", err) } evm := vm.NewEVM(context, logState, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) - evm.SetTxContext(core.NewEVMTxContext(msg)) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { @@ -206,11 +205,6 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { b.Fatalf("failed to parse testcase input: %v", err) } signer := types.MakeSigner(test.Genesis.Config, new(big.Int).SetUint64(uint64(test.Context.Number)), uint64(test.Context.Time)) - origin, _ := signer.Sender(tx) - txContext := vm.TxContext{ - Origin: origin, - GasPrice: tx.GasPrice(), - } context := test.Context.toBlockContext(test.Genesis) msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { @@ -222,19 +216,25 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { b.ReportAllocs() b.ResetTimer() - tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), nil, test.Genesis.Config) - if err != nil { - b.Fatalf("failed to create call tracer: %v", err) - } - evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) - evm.SetTxContext(txContext) + evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{}) for i := 0; i < b.N; i++ { snap := state.StateDB.Snapshot() - st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(tx.Gas())) - if _, err = st.TransitionDb(); err != nil { + tracer, err := tracers.DefaultDirectory.New(tracerName, new(tracers.Context), nil, test.Genesis.Config) + if err != nil { + b.Fatalf("failed to create call tracer: %v", err) + } + evm.Config.Tracer = tracer.Hooks + if tracer.OnTxStart != nil { + tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) + } + _, err = core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + if err != nil { b.Fatalf("failed to execute transaction: %v", err) } + if tracer.OnTxEnd != nil { + tracer.OnTxEnd(&types.Receipt{GasUsed: tx.Gas()}, nil) + } if _, err = tracer.GetResult(); err != nil { b.Fatal(err) } @@ -372,12 +372,7 @@ func TestInternals(t *testing.T) { if err != nil { t.Fatalf("test %v: failed to sign transaction: %v", tc.name, err) } - txContext := vm.TxContext{ - Origin: origin, - GasPrice: tx.GasPrice(), - } evm := vm.NewEVM(context, logState, config, vm.Config{Tracer: tc.tracer.Hooks}) - evm.SetTxContext(txContext) msg, err := core.TransactionToMessage(tx, signer, big.NewInt(0)) if err != nil { t.Fatalf("test %v: failed to create message: %v", tc.name, err) diff --git a/eth/tracers/internal/tracetest/flat_calltrace_test.go b/eth/tracers/internal/tracetest/flat_calltrace_test.go index 1dbdc7caac..553eaf1b57 100644 --- a/eth/tracers/internal/tracetest/flat_calltrace_test.go +++ b/eth/tracers/internal/tracetest/flat_calltrace_test.go @@ -99,7 +99,6 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string return fmt.Errorf("failed to prepare transaction for tracing: %v", err) } evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) - evm.SetTxContext(core.NewEVMTxContext(msg)) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index b8dec51db1..ad3d75d8de 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -107,7 +107,6 @@ func testPrestateDiffTracer(tracerName string, dirPath string, t *testing.T) { t.Fatalf("failed to prepare transaction for tracing: %v", err) } evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) - evm.SetTxContext(core.NewEVMTxContext(msg)) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index 35abd00017..227ea57226 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -260,7 +260,7 @@ func (t *jsTracer) OnTxStart(env *tracing.VMContext, tx *types.Transaction, from t.activePrecompiles = vm.ActivePrecompiles(rules) t.ctx["block"] = t.vm.ToValue(t.env.BlockNumber.Uint64()) t.ctx["gas"] = t.vm.ToValue(tx.Gas()) - gasPriceBig, err := t.toBig(t.vm, env.GasPrice.String()) + gasPriceBig, err := t.toBig(t.vm, tx.EffectiveGasTipValue(env.BaseFee).String()) if err != nil { t.err = err return diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 54f628b5f3..faad1e7194 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -59,7 +59,7 @@ type vmContext struct { } func testCtx() *vmContext { - return &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1)}, txCtx: vm.TxContext{GasPrice: big.NewInt(100000)}} + return &vmContext{blockCtx: vm.BlockContext{BlockNumber: big.NewInt(1), BaseFee: big.NewInt(0)}, txCtx: vm.TxContext{GasPrice: big.NewInt(100000)}} } func runTrace(tracer *tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainConfig, contractCode []byte) (json.RawMessage, error) { @@ -76,7 +76,7 @@ func runTrace(tracer *tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCo contract.Code = contractCode } - tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit}), contract.Caller()) + tracer.OnTxStart(evm.GetVMContext(), types.NewTx(&types.LegacyTx{Gas: gasLimit, GasPrice: vmctx.txCtx.GasPrice}), contract.Caller()) tracer.OnEnter(0, byte(vm.CALL), contract.Caller(), contract.Address(), []byte{}, startGas, value.ToBig()) ret, err := evm.Interpreter().Run(contract, []byte{}, false) tracer.OnExit(0, ret, startGas-contract.Gas, err, true) diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 72cc1bde25..9d8f70386f 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -47,10 +47,6 @@ func BenchmarkTransactionTrace(b *testing.B) { if err != nil { b.Fatal(err) } - txContext := vm.TxContext{ - Origin: from, - GasPrice: tx.GasPrice(), - } context := vm.BlockContext{ CanTransfer: core.CanTransfer, Transfer: core.Transfer, @@ -90,7 +86,6 @@ func BenchmarkTransactionTrace(b *testing.B) { //EnableReturnData: false, }) evm := vm.NewEVM(context, state.StateDB, params.AllEthashProtocolChanges, vm.Config{Tracer: tracer.Hooks()}) - evm.SetTxContext(txContext) msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { b.Fatalf("failed to prepare transaction for tracing: %v", err) @@ -101,8 +96,7 @@ func BenchmarkTransactionTrace(b *testing.B) { for i := 0; i < b.N; i++ { snap := state.StateDB.Snapshot() tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(tx.Gas())) - res, err := st.TransitionDb() + res, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { b.Fatal(err) } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index ded4db50aa..067d07ba7a 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -867,7 +867,6 @@ func applyMessage(ctx context.Context, b Backend, args TransactionArgs, state *s if precompiles != nil { evm.SetPrecompiles(precompiles) } - evm.SetTxContext(core.NewEVMTxContext(msg)) res, err := applyMessageWithEVM(ctx, evm, msg, timeout, gp) // If an internal state error occurred, let that have precedence. Otherwise, // a "trie root missing" type of error will masquerade as e.g. "insufficient gas" @@ -1331,17 +1330,17 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH // Apply the transaction with the access list tracer tracer := logger.NewAccessListTracer(accessList, args.from(), to, precompiles) config := vm.Config{Tracer: tracer.Hooks(), NoBaseFee: true} - vmenv := b.GetEVM(ctx, statedb, header, &config, nil) + evm := b.GetEVM(ctx, statedb, header, &config, nil) + // Lower the basefee to 0 to avoid breaking EVM // invariants (basefee < feecap). if msg.GasPrice.Sign() == 0 { - vmenv.Context.BaseFee = new(big.Int) + evm.Context.BaseFee = new(big.Int) } if msg.BlobGasFeeCap != nil && msg.BlobGasFeeCap.BitLen() == 0 { - vmenv.Context.BlobBaseFee = new(big.Int) + evm.Context.BlobBaseFee = new(big.Int) } - vmenv.SetTxContext(core.NewEVMTxContext(msg)) - res, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.GasLimit)) + res, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) if err != nil { return nil, 0, nil, fmt.Errorf("failed to apply transaction: %v err: %v", args.ToTransaction(types.LegacyTxType).Hash(), err) } diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 2161e1d5f4..f6647c7ba4 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -207,7 +207,6 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, tracer.reset(tx.Hash(), uint(i)) // EoA check is always skipped, even in validation mode. msg := call.ToMessage(header.BaseFee, !sim.validate, true) - evm.SetTxContext(core.NewEVMTxContext(msg)) result, err := applyMessageWithEVM(ctx, evm, msg, timeout, sim.gp) if err != nil { txErr := txValidationError(err) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 446ffb40d5..6884ae7ed5 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -277,7 +277,6 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh } // Prepare the EVM. - txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(block.Header(), nil, &t.json.Env.Coinbase) context.GetHash = vmTestBlockHash context.BaseFee = baseFee @@ -294,7 +293,6 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh context.BlobBaseFee = eip4844.CalcBlobFee(*t.json.Env.ExcessBlobGas) } evm := vm.NewEVM(context, st.StateDB, config, vmconfig) - evm.SetTxContext(txContext) if tracer := vmconfig.Tracer; tracer != nil && tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), nil, msg.From) From ce8cec007c42e54f79e53c3769a7279487968c07 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Fri, 29 Nov 2024 18:42:28 +0100 Subject: [PATCH 09/17] eth/tracers: fix state hooks in API (#30830) When a tx/block was being traced through the API the state hooks weren't being called as they should. This is due to #30745 moving the hooked statedb one level up in the state processor. This PR fixes that. --------- Co-authored-by: Martin HS Co-authored-by: Gary Rong --- eth/tracers/api.go | 4 +- eth/tracers/api_test.go | 89 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 3416c11a67..c15c5c4eb2 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -1007,8 +1007,8 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor return nil, err } } - // The actual TxContext will be created as part of ApplyTransactionWithEVM. - evm := vm.NewEVM(vmctx, statedb, api.backend.ChainConfig(), vm.Config{Tracer: tracer.Hooks, NoBaseFee: true}) + tracingStateDB := state.NewHookedState(statedb, tracer.Hooks) + evm := vm.NewEVM(vmctx, tracingStateDB, api.backend.ChainConfig(), vm.Config{Tracer: tracer.Hooks, NoBaseFee: true}) // Define a meaningful timeout of a single transaction trace if config.Timeout != nil { diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 2ae36287fa..e5940b6828 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -185,6 +186,94 @@ func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction index %d out of range for block %#x", txIndex, block.Hash()) } +type stateTracer struct { + Balance map[common.Address]*hexutil.Big + Nonce map[common.Address]hexutil.Uint64 + Storage map[common.Address]map[common.Hash]common.Hash +} + +func newStateTracer(ctx *Context, cfg json.RawMessage, chainCfg *params.ChainConfig) (*Tracer, error) { + t := &stateTracer{ + Balance: make(map[common.Address]*hexutil.Big), + Nonce: make(map[common.Address]hexutil.Uint64), + Storage: make(map[common.Address]map[common.Hash]common.Hash), + } + return &Tracer{ + GetResult: func() (json.RawMessage, error) { + return json.Marshal(t) + }, + Hooks: &tracing.Hooks{ + OnBalanceChange: func(addr common.Address, prev, new *big.Int, reason tracing.BalanceChangeReason) { + t.Balance[addr] = (*hexutil.Big)(new) + }, + OnNonceChange: func(addr common.Address, prev, new uint64) { + t.Nonce[addr] = hexutil.Uint64(new) + }, + OnStorageChange: func(addr common.Address, slot common.Hash, prev, new common.Hash) { + if t.Storage[addr] == nil { + t.Storage[addr] = make(map[common.Hash]common.Hash) + } + t.Storage[addr][slot] = new + }, + }, + }, nil +} + +func TestStateHooks(t *testing.T) { + t.Parallel() + + // Initialize test accounts + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + from = crypto.PubkeyToAddress(key.PublicKey) + to = common.HexToAddress("0x00000000000000000000000000000000deadbeef") + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + from: {Balance: big.NewInt(params.Ether)}, + to: { + Code: []byte{ + byte(vm.PUSH1), 0x2a, // stack: [42] + byte(vm.PUSH1), 0x0, // stack: [0, 42] + byte(vm.SSTORE), // stack: [] + byte(vm.STOP), + }, + }, + }, + } + genBlocks = 2 + signer = types.HomesteadSigner{} + nonce = uint64(0) + backend = newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { + // Transfer from account[0] to account[1] + // value: 1000 wei + // fee: 0 wei + tx, _ := types.SignTx(types.NewTx(&types.LegacyTx{ + Nonce: nonce, + To: &to, + Value: big.NewInt(1000), + Gas: params.TxGas, + GasPrice: b.BaseFee(), + Data: nil}), + signer, key) + b.AddTx(tx) + nonce++ + }) + ) + defer backend.teardown() + DefaultDirectory.Register("stateTracer", newStateTracer, false) + api := NewAPI(backend) + tracer := "stateTracer" + res, err := api.TraceCall(context.Background(), ethapi.TransactionArgs{From: &from, To: &to, Value: (*hexutil.Big)(big.NewInt(1000))}, rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber), &TraceCallConfig{TraceConfig: TraceConfig{Tracer: &tracer}}) + if err != nil { + t.Fatalf("failed to trace call: %v", err) + } + expected := `{"Balance":{"0x00000000000000000000000000000000deadbeef":"0x3e8","0x71562b71999873db5b286df957af199ec94617f7":"0xde0975924ed6f90"},"Nonce":{"0x71562b71999873db5b286df957af199ec94617f7":"0x3"},"Storage":{"0x00000000000000000000000000000000deadbeef":{"0x0000000000000000000000000000000000000000000000000000000000000000":"0x000000000000000000000000000000000000000000000000000000000000002a"}}}` + if expected != fmt.Sprintf("%s", res) { + t.Fatalf("unexpected trace result: have %s want %s", res, expected) + } +} + func TestTraceCall(t *testing.T) { t.Parallel() From 53472803199e6bfca0e06d9035876b69dc2e91dc Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Mon, 2 Dec 2024 07:18:02 -0700 Subject: [PATCH 10/17] cmd/evm: improve block/state test runner (#30633) * unify `staterunner` and `blockrunner` CLI flags, especially around tracing * added support for struct logger or json logging (although having issue #30658) * new --cross-check flag to validate the stateless witness collection / execution matches stateful * adds support for tracing the stateless execution when a tracer is set (to more easily debug differences) * --human for more readable test summary * directory or file input, so if you pass tests/spec-tests/fixtures/blockchain_tests it will execute all blockchain tests --- cmd/evm/blockrunner.go | 87 ++++---- cmd/evm/compiler.go | 55 ----- cmd/evm/disasm.go | 55 ----- cmd/evm/eest.go | 49 +++++ cmd/evm/eofparse.go | 32 ++- cmd/evm/internal/compiler/compiler.go | 39 ---- cmd/evm/main.go | 298 +++++++++++++------------- cmd/evm/reporter.go | 87 ++++++++ cmd/evm/runner.go | 141 ++++++++---- cmd/evm/staterunner.go | 193 +++++++---------- core/blockchain.go | 2 +- core/stateless.go | 4 +- eth/catalyst/api.go | 3 +- 13 files changed, 539 insertions(+), 506 deletions(-) delete mode 100644 cmd/evm/compiler.go delete mode 100644 cmd/evm/disasm.go create mode 100644 cmd/evm/eest.go delete mode 100644 cmd/evm/internal/compiler/compiler.go create mode 100644 cmd/evm/reporter.go diff --git a/cmd/evm/blockrunner.go b/cmd/evm/blockrunner.go index d5cd8d8e3d..2cb0531e28 100644 --- a/cmd/evm/blockrunner.go +++ b/cmd/evm/blockrunner.go @@ -22,79 +22,84 @@ import ( "fmt" "os" "regexp" - "sort" + "slices" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/tracing" - "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" + "golang.org/x/exp/maps" ) -var RunFlag = &cli.StringFlag{ - Name: "run", - Value: ".*", - Usage: "Run only those tests matching the regular expression.", -} - var blockTestCommand = &cli.Command{ Action: blockTestCmd, Name: "blocktest", Usage: "Executes the given blockchain tests", - ArgsUsage: "", - Flags: []cli.Flag{RunFlag}, + ArgsUsage: "", + Flags: slices.Concat([]cli.Flag{ + DumpFlag, + HumanReadableFlag, + RunFlag, + WitnessCrossCheckFlag, + }, traceFlags), } func blockTestCmd(ctx *cli.Context) error { - if len(ctx.Args().First()) == 0 { - return errors.New("path-to-test argument required") + path := ctx.Args().First() + if len(path) == 0 { + return errors.New("path argument required") } + var ( + collected = collectJSONFiles(path) + results []testResult + ) + for _, fname := range collected { + r, err := runBlockTest(ctx, fname) + if err != nil { + return err + } + results = append(results, r...) + } + report(ctx, results) + return nil +} - var tracer *tracing.Hooks - // Configure the EVM logger - if ctx.Bool(MachineFlag.Name) { - tracer = logger.NewJSONLogger(&logger.Config{ - EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), - DisableStack: ctx.Bool(DisableStackFlag.Name), - DisableStorage: ctx.Bool(DisableStorageFlag.Name), - EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name), - }, os.Stderr) - } - // Load the test content from the input file - src, err := os.ReadFile(ctx.Args().First()) +func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) { + src, err := os.ReadFile(fname) if err != nil { - return err + return nil, err } - var tests map[string]tests.BlockTest + var tests map[string]*tests.BlockTest if err = json.Unmarshal(src, &tests); err != nil { - return err + return nil, err } re, err := regexp.Compile(ctx.String(RunFlag.Name)) if err != nil { - return fmt.Errorf("invalid regex -%s: %v", RunFlag.Name, err) + return nil, fmt.Errorf("invalid regex -%s: %v", RunFlag.Name, err) } + tracer := tracerFromFlags(ctx) - // Run them in order - var keys []string - for key := range tests { - keys = append(keys, key) - } - sort.Strings(keys) + // Pull out keys to sort and ensure tests are run in order. + keys := maps.Keys(tests) + slices.Sort(keys) + + // Run all the tests. + var results []testResult for _, name := range keys { if !re.MatchString(name) { continue } - test := tests[name] - if err := test.Run(false, rawdb.HashScheme, false, tracer, func(res error, chain *core.BlockChain) { + result := &testResult{Name: name, Pass: true} + if err := tests[name].Run(false, rawdb.HashScheme, ctx.Bool(WitnessCrossCheckFlag.Name), tracer, func(res error, chain *core.BlockChain) { if ctx.Bool(DumpFlag.Name) { - if state, _ := chain.State(); state != nil { - fmt.Println(string(state.Dump(nil))) + if s, _ := chain.State(); s != nil { + result.State = dump(s) } } }); err != nil { - return fmt.Errorf("test %v: %w", name, err) + result.Pass, result.Error = false, err.Error() } + results = append(results, *result) } - return nil + return results, nil } diff --git a/cmd/evm/compiler.go b/cmd/evm/compiler.go deleted file mode 100644 index c071834b59..0000000000 --- a/cmd/evm/compiler.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "errors" - "fmt" - "os" - - "github.com/ethereum/go-ethereum/cmd/evm/internal/compiler" - - "github.com/urfave/cli/v2" -) - -var compileCommand = &cli.Command{ - Action: compileCmd, - Name: "compile", - Usage: "Compiles easm source to evm binary", - ArgsUsage: "", -} - -func compileCmd(ctx *cli.Context) error { - debug := ctx.Bool(DebugFlag.Name) - - if len(ctx.Args().First()) == 0 { - return errors.New("filename required") - } - - fn := ctx.Args().First() - src, err := os.ReadFile(fn) - if err != nil { - return err - } - - bin, err := compiler.Compile(fn, src, debug) - if err != nil { - return err - } - fmt.Println(bin) - return nil -} diff --git a/cmd/evm/disasm.go b/cmd/evm/disasm.go deleted file mode 100644 index b1f35cbaf5..0000000000 --- a/cmd/evm/disasm.go +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package main - -import ( - "errors" - "fmt" - "os" - "strings" - - "github.com/ethereum/go-ethereum/core/asm" - "github.com/urfave/cli/v2" -) - -var disasmCommand = &cli.Command{ - Action: disasmCmd, - Name: "disasm", - Usage: "Disassembles evm binary", - ArgsUsage: "", -} - -func disasmCmd(ctx *cli.Context) error { - var in string - switch { - case len(ctx.Args().First()) > 0: - fn := ctx.Args().First() - input, err := os.ReadFile(fn) - if err != nil { - return err - } - in = string(input) - case ctx.IsSet(InputFlag.Name): - in = ctx.String(InputFlag.Name) - default: - return errors.New("missing filename or --input value") - } - - code := strings.TrimSpace(in) - fmt.Printf("%v\n", code) - return asm.PrintDisassembled(code) -} diff --git a/cmd/evm/eest.go b/cmd/evm/eest.go new file mode 100644 index 0000000000..43071a3e5d --- /dev/null +++ b/cmd/evm/eest.go @@ -0,0 +1,49 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import "regexp" + +// testMetadata provides more granular access to the test information encoded +// within its filename by the execution spec test (EEST). +type testMetadata struct { + fork string + module string // which python module gnerated the test, e.g. eip7702 + file string // exact file the test came from, e.g. test_gas.py + function string // func that created the test, e.g. test_valid_mcopy_operations + parameters string // the name of the parameters which were used to fill the test, e.g. zero_inputs +} + +// parseTestMetadata reads a test name and parses out more specific information +// about the test. +func parseTestMetadata(s string) *testMetadata { + var ( + pattern = `tests\/([^\/]+)\/([^\/]+)\/([^:]+)::([^[]+)\[fork_([^-\]]+)-[^-]+-(.+)\]` + re = regexp.MustCompile(pattern) + ) + match := re.FindStringSubmatch(s) + if len(match) == 0 { + return nil + } + return &testMetadata{ + fork: match[5], + module: match[2], + file: match[3], + function: match[4], + parameters: match[6], + } +} diff --git a/cmd/evm/eofparse.go b/cmd/evm/eofparse.go index 2122270942..df8581146a 100644 --- a/cmd/evm/eofparse.go +++ b/cmd/evm/eofparse.go @@ -31,13 +31,41 @@ import ( "github.com/urfave/cli/v2" ) +var jt vm.JumpTable + +const initcode = "INITCODE" + func init() { jt = vm.NewPragueEOFInstructionSetForTesting() } var ( - jt vm.JumpTable - initcode = "INITCODE" + hexFlag = &cli.StringFlag{ + Name: "hex", + Usage: "Single container data parse and validation", + } + refTestFlag = &cli.StringFlag{ + Name: "test", + Usage: "Path to EOF validation reference test.", + } + eofParseCommand = &cli.Command{ + Name: "eofparse", + Aliases: []string{"eof"}, + Usage: "Parses hex eof container and returns validation errors (if any)", + Action: eofParseAction, + Flags: []cli.Flag{ + hexFlag, + refTestFlag, + }, + } + eofDumpCommand = &cli.Command{ + Name: "eofdump", + Usage: "Parses hex eof container and prints out human-readable representation of the container.", + Action: eofDumpAction, + Flags: []cli.Flag{ + hexFlag, + }, + } ) func eofParseAction(ctx *cli.Context) error { diff --git a/cmd/evm/internal/compiler/compiler.go b/cmd/evm/internal/compiler/compiler.go deleted file mode 100644 index 54981b6697..0000000000 --- a/cmd/evm/internal/compiler/compiler.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of go-ethereum. -// -// go-ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// go-ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with go-ethereum. If not, see . - -package compiler - -import ( - "errors" - "fmt" - - "github.com/ethereum/go-ethereum/core/asm" -) - -func Compile(fn string, src []byte, debug bool) (string, error) { - compiler := asm.NewCompiler(debug) - compiler.Feed(asm.Lex(src, debug)) - - bin, compileErrors := compiler.Compile() - if len(compileErrors) > 0 { - // report errors - for _, err := range compileErrors { - fmt.Printf("%s:%v\n", fn, err) - } - return "", errors.New("compiling failed") - } - return bin, nil -} diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 0d4471b8d5..750b269906 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -19,11 +19,14 @@ package main import ( "fmt" - "math/big" + "io/fs" "os" - "slices" + "path/filepath" "github.com/ethereum/go-ethereum/cmd/evm/internal/t8ntool" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/flags" "github.com/urfave/cli/v2" @@ -33,122 +36,100 @@ import ( _ "github.com/ethereum/go-ethereum/eth/tracers/native" ) +// Some other nice-to-haves: +// * accumulate traces into an object to bundle with test +// * write tx identifier for trace before hand (blocktest only) +// * combine blocktest and statetest runner logic using unified test interface + +const traceCategory = "TRACING" + var ( - DebugFlag = &cli.BoolFlag{ - Name: "debug", - Usage: "output full trace logs", - Category: flags.VMCategory, - } - StatDumpFlag = &cli.BoolFlag{ - Name: "statdump", - Usage: "displays stack and heap memory information", - Category: flags.VMCategory, - } - CodeFlag = &cli.StringFlag{ - Name: "code", - Usage: "EVM code", - Category: flags.VMCategory, - } - CodeFileFlag = &cli.StringFlag{ - Name: "codefile", - Usage: "File containing EVM code. If '-' is specified, code is read from stdin ", - Category: flags.VMCategory, - } - GasFlag = &cli.Uint64Flag{ - Name: "gas", - Usage: "gas limit for the evm", - Value: 10000000000, - Category: flags.VMCategory, - } - PriceFlag = &flags.BigFlag{ - Name: "price", - Usage: "price set for the evm", - Value: new(big.Int), - Category: flags.VMCategory, - } - ValueFlag = &flags.BigFlag{ - Name: "value", - Usage: "value set for the evm", - Value: new(big.Int), - Category: flags.VMCategory, - } - DumpFlag = &cli.BoolFlag{ - Name: "dump", - Usage: "dumps the state after the run", - Category: flags.VMCategory, - } - InputFlag = &cli.StringFlag{ - Name: "input", - Usage: "input for the EVM", - Category: flags.VMCategory, - } - InputFileFlag = &cli.StringFlag{ - Name: "inputfile", - Usage: "file containing input for the EVM", - Category: flags.VMCategory, + // Test running flags. + RunFlag = &cli.StringFlag{ + Name: "run", + Value: ".*", + Usage: "Run only those tests matching the regular expression.", } BenchFlag = &cli.BoolFlag{ Name: "bench", Usage: "benchmark the execution", Category: flags.VMCategory, } - CreateFlag = &cli.BoolFlag{ - Name: "create", - Usage: "indicates the action should be create rather than call", - Category: flags.VMCategory, + WitnessCrossCheckFlag = &cli.BoolFlag{ + Name: "cross-check", + Aliases: []string{"xc"}, + Usage: "Cross-check stateful execution against stateless, verifying the witness generation.", } - GenesisFlag = &cli.StringFlag{ - Name: "prestate", - Usage: "JSON file with prestate (genesis) config", - Category: flags.VMCategory, + + // Debugging flags. + DumpFlag = &cli.BoolFlag{ + Name: "dump", + Usage: "dumps the state after the run", + } + HumanReadableFlag = &cli.BoolFlag{ + Name: "human", + Usage: "\"Human-readable\" output", + } + StatDumpFlag = &cli.BoolFlag{ + Name: "statdump", + Usage: "displays stack and heap memory information", + } + + // Tracing flags. + TraceFlag = &cli.BoolFlag{ + Name: "trace", + Usage: "Enable tracing and output trace log.", + Category: traceCategory, + } + TraceFormatFlag = &cli.StringFlag{ + Name: "trace.format", + Usage: "Trace output format to use (struct|json)", + Value: "struct", + Category: traceCategory, + } + TraceDisableMemoryFlag = &cli.BoolFlag{ + Name: "trace.nomemory", + Aliases: []string{"nomemory"}, + Value: true, + Usage: "disable memory output", + Category: traceCategory, + } + TraceDisableStackFlag = &cli.BoolFlag{ + Name: "trace.nostack", + Aliases: []string{"nostack"}, + Usage: "disable stack output", + Category: traceCategory, + } + TraceDisableStorageFlag = &cli.BoolFlag{ + Name: "trace.nostorage", + Aliases: []string{"nostorage"}, + Usage: "disable storage output", + Category: traceCategory, + } + TraceDisableReturnDataFlag = &cli.BoolFlag{ + Name: "trace.noreturndata", + Aliases: []string{"noreturndata"}, + Value: true, + Usage: "enable return data output", + Category: traceCategory, + } + + // Deprecated flags. + DebugFlag = &cli.BoolFlag{ + Name: "debug", + Usage: "output full trace logs (deprecated)", + Hidden: true, + Category: traceCategory, } MachineFlag = &cli.BoolFlag{ Name: "json", - Usage: "output trace logs in machine readable format (json)", - Category: flags.VMCategory, - } - SenderFlag = &cli.StringFlag{ - Name: "sender", - Usage: "The transaction origin", - Category: flags.VMCategory, - } - ReceiverFlag = &cli.StringFlag{ - Name: "receiver", - Usage: "The transaction receiver (execution context)", - Category: flags.VMCategory, - } - DisableMemoryFlag = &cli.BoolFlag{ - Name: "nomemory", - Value: true, - Usage: "disable memory output", - Category: flags.VMCategory, - } - DisableStackFlag = &cli.BoolFlag{ - Name: "nostack", - Usage: "disable stack output", - Category: flags.VMCategory, - } - DisableStorageFlag = &cli.BoolFlag{ - Name: "nostorage", - Usage: "disable storage output", - Category: flags.VMCategory, - } - DisableReturnDataFlag = &cli.BoolFlag{ - Name: "noreturndata", - Value: true, - Usage: "enable return data output", - Category: flags.VMCategory, - } - refTestFlag = &cli.StringFlag{ - Name: "test", - Usage: "Path to EOF validation reference test.", - } - hexFlag = &cli.StringFlag{ - Name: "hex", - Usage: "single container data parse and validation", + Usage: "output trace logs in machine readable format, json (deprecated)", + Hidden: true, + Category: traceCategory, } ) +// Command definitions. var ( stateTransitionCommand = &cli.Command{ Name: "transition", @@ -175,7 +156,6 @@ var ( t8ntool.RewardFlag, }, } - transactionCommand = &cli.Command{ Name: "transaction", Aliases: []string{"t9n"}, @@ -203,62 +183,27 @@ var ( t8ntool.SealCliqueFlag, }, } - eofParseCommand = &cli.Command{ - Name: "eofparse", - Aliases: []string{"eof"}, - Usage: "Parses hex eof container and returns validation errors (if any)", - Action: eofParseAction, - Flags: []cli.Flag{ - hexFlag, - refTestFlag, - }, - } - - eofDumpCommand = &cli.Command{ - Name: "eofdump", - Usage: "Parses hex eof container and prints out human-readable representation of the container.", - Action: eofDumpAction, - Flags: []cli.Flag{ - hexFlag, - }, - } ) -// vmFlags contains flags related to running the EVM. -var vmFlags = []cli.Flag{ - CodeFlag, - CodeFileFlag, - CreateFlag, - GasFlag, - PriceFlag, - ValueFlag, - InputFlag, - InputFileFlag, - GenesisFlag, - SenderFlag, - ReceiverFlag, -} - // traceFlags contains flags that configure tracing output. var traceFlags = []cli.Flag{ - BenchFlag, + TraceFlag, + TraceFormatFlag, + TraceDisableStackFlag, + TraceDisableMemoryFlag, + TraceDisableStorageFlag, + TraceDisableReturnDataFlag, + + // deprecated DebugFlag, - DumpFlag, MachineFlag, - StatDumpFlag, - DisableMemoryFlag, - DisableStackFlag, - DisableStorageFlag, - DisableReturnDataFlag, } var app = flags.NewApp("the evm command line interface") func init() { - app.Flags = slices.Concat(vmFlags, traceFlags, debug.Flags) + app.Flags = debug.Flags app.Commands = []*cli.Command{ - compileCommand, - disasmCommand, runCommand, blockTestCommand, stateTestCommand, @@ -280,11 +225,56 @@ func init() { func main() { if err := app.Run(os.Args); err != nil { - code := 1 - if ec, ok := err.(*t8ntool.NumberedError); ok { - code = ec.ExitCode() - } fmt.Fprintln(os.Stderr, err) - os.Exit(code) + os.Exit(1) } } + +// tracerFromFlags parses the cli flags and returns the specified tracer. +func tracerFromFlags(ctx *cli.Context) *tracing.Hooks { + config := &logger.Config{ + EnableMemory: !ctx.Bool(TraceDisableMemoryFlag.Name), + DisableStack: ctx.Bool(TraceDisableStackFlag.Name), + DisableStorage: ctx.Bool(TraceDisableStorageFlag.Name), + EnableReturnData: !ctx.Bool(TraceDisableReturnDataFlag.Name), + } + switch { + case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "struct": + return logger.NewStructLogger(config).Hooks() + case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "json": + return logger.NewJSONLogger(config, os.Stderr) + case ctx.Bool(MachineFlag.Name): + return logger.NewJSONLogger(config, os.Stderr) + case ctx.Bool(DebugFlag.Name): + return logger.NewStructLogger(config).Hooks() + default: + return nil + } +} + +// collectJSONFiles walks the given path and accumulates all files with json +// extension. +func collectJSONFiles(path string) []string { + var out []string + err := filepath.Walk(path, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && filepath.Ext(info.Name()) == ".json" { + out = append(out, path) + } + return nil + }) + if err != nil { + fmt.Fprintln(os.Stderr, err) + } + return out +} + +// dump returns a state dump for the most current trie. +func dump(s *state.StateDB) *state.Dump { + root := s.IntermediateRoot(false) + cpy, _ := state.New(root, s.Database()) + dump := cpy.RawDump(nil) + return &dump +} diff --git a/cmd/evm/reporter.go b/cmd/evm/reporter.go new file mode 100644 index 0000000000..f6249e1843 --- /dev/null +++ b/cmd/evm/reporter.go @@ -0,0 +1,87 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "encoding/json" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/state" + "github.com/urfave/cli/v2" +) + +const ( + PASS = "\033[32mPASS\033[0m" + FAIL = "\033[31mFAIL\033[0m" +) + +// testResult contains the execution status after running a state test, any +// error that might have occurred and a dump of the final state if requested. +type testResult struct { + Name string `json:"name"` + Pass bool `json:"pass"` + Root *common.Hash `json:"stateRoot,omitempty"` + Fork string `json:"fork"` + Error string `json:"error,omitempty"` + State *state.Dump `json:"state,omitempty"` + Stats *execStats `json:"benchStats,omitempty"` +} + +func (r testResult) String() string { + var status string + if r.Pass { + status = fmt.Sprintf("[%s]", PASS) + } else { + status = fmt.Sprintf("[%s]", FAIL) + } + info := r.Name + m := parseTestMetadata(r.Name) + if m != nil { + info = fmt.Sprintf("%s %s, param=%s", m.module, m.function, m.parameters) + } + var extra string + if !r.Pass { + extra = fmt.Sprintf(", err=%v, fork=%s", r.Error, r.Fork) + } + out := fmt.Sprintf("%s %s%s", status, info, extra) + if r.State != nil { + state, _ := json.MarshalIndent(r.State, "", " ") + out += "\n" + string(state) + } + return out +} + +// report prints the after-test summary. +func report(ctx *cli.Context, results []testResult) { + if ctx.Bool(HumanReadableFlag.Name) { + pass := 0 + for _, r := range results { + if r.Pass { + pass++ + } + } + for _, r := range results { + fmt.Println(r) + } + fmt.Println("--") + fmt.Printf("%d tests passed, %d tests failed.\n", pass, len(results)-pass) + return + } + out, _ := json.MarshalIndent(results, "", " ") + fmt.Println(string(out)) +} diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 2884487faa..afb0bec0ef 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -25,10 +25,10 @@ import ( "os" goruntime "runtime" "slices" + "strings" "testing" "time" - "github.com/ethereum/go-ethereum/cmd/evm/internal/compiler" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" @@ -51,14 +51,82 @@ var runCommand = &cli.Command{ Usage: "Run arbitrary evm binary", ArgsUsage: "", Description: `The run command runs arbitrary EVM code.`, - Flags: slices.Concat(vmFlags, traceFlags), + Flags: slices.Concat([]cli.Flag{ + BenchFlag, + CodeFileFlag, + CreateFlag, + GasFlag, + GenesisFlag, + InputFlag, + InputFileFlag, + PriceFlag, + ReceiverFlag, + SenderFlag, + ValueFlag, + StatDumpFlag, + }, traceFlags), } +var ( + CodeFileFlag = &cli.StringFlag{ + Name: "codefile", + Usage: "File containing EVM code. If '-' is specified, code is read from stdin ", + Category: flags.VMCategory, + } + CreateFlag = &cli.BoolFlag{ + Name: "create", + Usage: "Indicates the action should be create rather than call", + Category: flags.VMCategory, + } + GasFlag = &cli.Uint64Flag{ + Name: "gas", + Usage: "Gas limit for the evm", + Value: 10000000000, + Category: flags.VMCategory, + } + GenesisFlag = &cli.StringFlag{ + Name: "prestate", + Usage: "JSON file with prestate (genesis) config", + Category: flags.VMCategory, + } + InputFlag = &cli.StringFlag{ + Name: "input", + Usage: "Input for the EVM", + Category: flags.VMCategory, + } + InputFileFlag = &cli.StringFlag{ + Name: "inputfile", + Usage: "File containing input for the EVM", + Category: flags.VMCategory, + } + PriceFlag = &flags.BigFlag{ + Name: "price", + Usage: "Price set for the evm", + Value: new(big.Int), + Category: flags.VMCategory, + } + ReceiverFlag = &cli.StringFlag{ + Name: "receiver", + Usage: "The transaction receiver (execution context)", + Category: flags.VMCategory, + } + SenderFlag = &cli.StringFlag{ + Name: "sender", + Usage: "The transaction origin", + Category: flags.VMCategory, + } + ValueFlag = &flags.BigFlag{ + Name: "value", + Usage: "Value set for the evm", + Value: new(big.Int), + Category: flags.VMCategory, + } +) + // readGenesis will read the given JSON format genesis file and return // the initialized Genesis structure func readGenesis(genesisPath string) *core.Genesis { // Make sure we have a valid genesis JSON - //genesisPath := ctx.Args().First() if len(genesisPath) == 0 { utils.Fatalf("Must supply path to genesis JSON file") } @@ -128,10 +196,10 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) ([]byte, exe func runCmd(ctx *cli.Context) error { logconfig := &logger.Config{ - EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), - DisableStack: ctx.Bool(DisableStackFlag.Name), - DisableStorage: ctx.Bool(DisableStorageFlag.Name), - EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name), + EnableMemory: !ctx.Bool(TraceDisableMemoryFlag.Name), + DisableStack: ctx.Bool(TraceDisableStackFlag.Name), + DisableStorage: ctx.Bool(TraceDisableStorageFlag.Name), + EnableReturnData: !ctx.Bool(TraceDisableReturnDataFlag.Name), Debug: ctx.Bool(DebugFlag.Name), } @@ -188,48 +256,35 @@ func runCmd(ctx *cli.Context) error { var code []byte codeFileFlag := ctx.String(CodeFileFlag.Name) - codeFlag := ctx.String(CodeFlag.Name) + hexcode := ctx.Args().First() - // The '--code' or '--codefile' flag overrides code in state - if codeFileFlag != "" || codeFlag != "" { - var hexcode []byte - if codeFileFlag != "" { - var err error - // If - is specified, it means that code comes from stdin - if codeFileFlag == "-" { - //Try reading from stdin - if hexcode, err = io.ReadAll(os.Stdin); err != nil { - fmt.Printf("Could not load code from stdin: %v\n", err) - os.Exit(1) - } - } else { - // Codefile with hex assembly - if hexcode, err = os.ReadFile(codeFileFlag); err != nil { - fmt.Printf("Could not load code from file: %v\n", err) - os.Exit(1) - } - } - } else { - hexcode = []byte(codeFlag) - } - hexcode = bytes.TrimSpace(hexcode) - if len(hexcode)%2 != 0 { - fmt.Printf("Invalid input length for hex data (%d)\n", len(hexcode)) + // The '--codefile' flag overrides code in state + if codeFileFlag == "-" { + // If - is specified, it means that code comes from stdin + // Try reading from stdin + input, err := io.ReadAll(os.Stdin) + if err != nil { + fmt.Printf("Could not load code from stdin: %v\n", err) os.Exit(1) } - code = common.FromHex(string(hexcode)) - } else if fn := ctx.Args().First(); len(fn) > 0 { - // EASM-file to compile - src, err := os.ReadFile(fn) + hexcode = string(input) + } else if codeFileFlag != "" { + // Codefile with hex assembly + input, err := os.ReadFile(codeFileFlag) if err != nil { - return err + fmt.Printf("Could not load code from file: %v\n", err) + os.Exit(1) } - bin, err := compiler.Compile(fn, src, false) - if err != nil { - return err - } - code = common.Hex2Bytes(bin) + hexcode = string(input) } + + hexcode = strings.TrimSpace(hexcode) + if len(hexcode)%2 != 0 { + fmt.Printf("Invalid input length for hex data (%d)\n", len(hexcode)) + os.Exit(1) + } + code = common.FromHex(hexcode) + runtimeConfig := runtime.Config{ Origin: sender, State: prestate, diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index d0a0d3287c..323b7d60ab 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -21,12 +21,12 @@ import ( "encoding/json" "fmt" "os" + "regexp" + "slices" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/tracers/logger" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/tests" "github.com/urfave/cli/v2" @@ -35,157 +35,124 @@ import ( var ( forkFlag = &cli.StringFlag{ Name: "statetest.fork", - Usage: "The hard-fork to run the test against", + Usage: "Only run tests for the specified fork.", Category: flags.VMCategory, } idxFlag = &cli.IntFlag{ Name: "statetest.index", - Usage: "The index of the subtest to run", + Usage: "The index of the subtest to run.", Category: flags.VMCategory, Value: -1, // default to select all subtest indices } - testNameFlag = &cli.StringFlag{ - Name: "statetest.name", - Usage: "The name of the state test to run", - Category: flags.VMCategory, - } ) var stateTestCommand = &cli.Command{ Action: stateTestCmd, Name: "statetest", Usage: "Executes the given state tests. Filenames can be fed via standard input (batch mode) or as an argument (one-off execution).", ArgsUsage: "", - Flags: []cli.Flag{ - forkFlag, - idxFlag, - testNameFlag, - }, -} - -// StatetestResult contains the execution status after running a state test, any -// error that might have occurred and a dump of the final state if requested. -type StatetestResult struct { - Name string `json:"name"` - Pass bool `json:"pass"` - Root *common.Hash `json:"stateRoot,omitempty"` - Fork string `json:"fork"` - Error string `json:"error,omitempty"` - State *state.Dump `json:"state,omitempty"` - BenchStats *execStats `json:"benchStats,omitempty"` + Flags: slices.Concat([]cli.Flag{ + DumpFlag, + HumanReadableFlag, + RunFlag, + }, traceFlags), } func stateTestCmd(ctx *cli.Context) error { - // Configure the EVM logger - config := &logger.Config{ - EnableMemory: !ctx.Bool(DisableMemoryFlag.Name), - DisableStack: ctx.Bool(DisableStackFlag.Name), - DisableStorage: ctx.Bool(DisableStorageFlag.Name), - EnableReturnData: !ctx.Bool(DisableReturnDataFlag.Name), - } - var cfg vm.Config - switch { - case ctx.Bool(MachineFlag.Name): - cfg.Tracer = logger.NewJSONLogger(config, os.Stderr) + path := ctx.Args().First() - case ctx.Bool(DebugFlag.Name): - cfg.Tracer = logger.NewStructLogger(config).Hooks() + // If path is provided, run the tests at that path. + if len(path) != 0 { + var ( + collected = collectJSONFiles(path) + results []testResult + ) + for _, fname := range collected { + r, err := runStateTest(ctx, fname) + if err != nil { + return err + } + results = append(results, r...) + } + report(ctx, results) + return nil } - // Load the test content from the input file - if len(ctx.Args().First()) != 0 { - return runStateTest(ctx, ctx.Args().First(), cfg, ctx.Bool(DumpFlag.Name), ctx.Bool(BenchFlag.Name)) - } - // Read filenames from stdin and execute back-to-back + // Otherwise, read filenames from stdin and execute back-to-back. scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { fname := scanner.Text() if len(fname) == 0 { return nil } - if err := runStateTest(ctx, fname, cfg, ctx.Bool(DumpFlag.Name), ctx.Bool(BenchFlag.Name)); err != nil { + results, err := runStateTest(ctx, fname) + if err != nil { return err } + report(ctx, results) } return nil } -type stateTestCase struct { - name string - test tests.StateTest - st tests.StateSubtest -} - -// collectMatchedSubtests returns test cases which match against provided filtering CLI parameters -func collectMatchedSubtests(ctx *cli.Context, testsByName map[string]tests.StateTest) []stateTestCase { - var res []stateTestCase - subtestName := ctx.String(testNameFlag.Name) - if subtestName != "" { - if subtest, ok := testsByName[subtestName]; ok { - testsByName := make(map[string]tests.StateTest) - testsByName[subtestName] = subtest - } - } - idx := ctx.Int(idxFlag.Name) - fork := ctx.String(forkFlag.Name) - - for key, test := range testsByName { - for _, st := range test.Subtests() { - if idx != -1 && st.Index != idx { - continue - } - if fork != "" && st.Fork != fork { - continue - } - res = append(res, stateTestCase{name: key, st: st, test: test}) - } - } - return res -} - // runStateTest loads the state-test given by fname, and executes the test. -func runStateTest(ctx *cli.Context, fname string, cfg vm.Config, dump bool, bench bool) error { +func runStateTest(ctx *cli.Context, fname string) ([]testResult, error) { src, err := os.ReadFile(fname) if err != nil { - return err + return nil, err } var testsByName map[string]tests.StateTest if err := json.Unmarshal(src, &testsByName); err != nil { - return err + return nil, fmt.Errorf("unable to read test file %s: %w", fname, err) } - matchingTests := collectMatchedSubtests(ctx, testsByName) + cfg := vm.Config{Tracer: tracerFromFlags(ctx)} + re, err := regexp.Compile(ctx.String(RunFlag.Name)) + if err != nil { + return nil, fmt.Errorf("invalid regex -%s: %v", RunFlag.Name, err) + } // Iterate over all the tests, run them and aggregate the results - var results []StatetestResult - for _, test := range matchingTests { - // Run the test and aggregate the result - result := &StatetestResult{Name: test.name, Fork: test.st.Fork, Pass: true} - test.test.Run(test.st, cfg, false, rawdb.HashScheme, func(err error, tstate *tests.StateTestState) { - var root common.Hash - if tstate.StateDB != nil { - root = tstate.StateDB.IntermediateRoot(false) - result.Root = &root - fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) - if dump { // Dump any state to aid debugging - cpy, _ := state.New(root, tstate.StateDB.Database()) - dump := cpy.RawDump(nil) - result.State = &dump - } - } - if err != nil { - // Test failed, mark as so - result.Pass, result.Error = false, err.Error() - } - }) - if bench { - _, stats, _ := timedExec(true, func() ([]byte, uint64, error) { - _, _, gasUsed, _ := test.test.RunNoVerify(test.st, cfg, false, rawdb.HashScheme) - return nil, gasUsed, nil - }) - result.BenchStats = &stats + results := make([]testResult, 0, len(testsByName)) + for key, test := range testsByName { + if !re.MatchString(key) { + continue + } + for i, st := range test.Subtests() { + if idx := ctx.Int(idxFlag.Name); idx != -1 && idx != i { + // If specific index requested, skip all tests that do not match. + continue + } + if fork := ctx.String(forkFlag.Name); fork != "" && st.Fork != fork { + // If specific fork requested, skip all tests that do not match. + continue + } + // Run the test and aggregate the result + result := &testResult{Name: key, Fork: st.Fork, Pass: true} + test.Run(st, cfg, false, rawdb.HashScheme, func(err error, state *tests.StateTestState) { + var root common.Hash + if state.StateDB != nil { + root = state.StateDB.IntermediateRoot(false) + result.Root = &root + fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root) + // Dump any state to aid debugging. + if ctx.Bool(DumpFlag.Name) { + result.State = dump(state.StateDB) + } + } + // Collect bench stats if requested. + if ctx.Bool(BenchFlag.Name) { + _, stats, _ := timedExec(true, func() ([]byte, uint64, error) { + _, _, gasUsed, _ := test.RunNoVerify(st, cfg, false, rawdb.HashScheme) + return nil, gasUsed, nil + }) + result.Stats = &stats + } + if err != nil { + // Test failed, mark as so. + result.Pass, result.Error = false, err.Error() + return + } + }) + results = append(results, *result) } - results = append(results, *result) } - out, _ := json.MarshalIndent(results, "", " ") - fmt.Println(string(out)) - return nil + return results, nil } diff --git a/core/blockchain.go b/core/blockchain.go index c3da61b281..0fe4812626 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1926,7 +1926,7 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s task := types.NewBlockWithHeader(context).WithBody(*block.Body()) // Run the stateless self-cross-validation - crossStateRoot, crossReceiptRoot, err := ExecuteStateless(bc.chainConfig, task, witness) + crossStateRoot, crossReceiptRoot, err := ExecuteStateless(bc.chainConfig, bc.vmConfig, task, witness) if err != nil { return nil, fmt.Errorf("stateless self-validation failed: %v", err) } diff --git a/core/stateless.go b/core/stateless.go index 5b37d5020e..d21a62b4a5 100644 --- a/core/stateless.go +++ b/core/stateless.go @@ -40,7 +40,7 @@ import ( // - It cannot be placed outside of core, because it needs to construct a dud headerchain // // TODO(karalabe): Would be nice to resolve both issues above somehow and move it. -func ExecuteStateless(config *params.ChainConfig, block *types.Block, witness *stateless.Witness) (common.Hash, common.Hash, error) { +func ExecuteStateless(config *params.ChainConfig, vmconfig vm.Config, block *types.Block, witness *stateless.Witness) (common.Hash, common.Hash, error) { // Sanity check if the supplied block accidentally contains a set root or // receipt hash. If so, be very loud, but still continue. if block.Root() != (common.Hash{}) { @@ -66,7 +66,7 @@ func ExecuteStateless(config *params.ChainConfig, block *types.Block, witness *s validator := NewBlockValidator(config, nil) // No chain, we only validate the state, not the block // Run the stateless blocks processing and self-validate certain fields - res, err := processor.Process(block, db, vm.Config{}) + res, err := processor.Process(block, db, vmconfig) if err != nil { return common.Hash{}, common.Hash{}, err } diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 4779f9756b..fa9752734e 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/internal/version" @@ -995,7 +996,7 @@ func (api *ConsensusAPI) executeStatelessPayload(params engine.ExecutableData, v api.lastNewPayloadLock.Unlock() log.Trace("Executing block statelessly", "number", block.Number(), "hash", params.BlockHash) - stateRoot, receiptRoot, err := core.ExecuteStateless(api.eth.BlockChain().Config(), block, witness) + stateRoot, receiptRoot, err := core.ExecuteStateless(api.eth.BlockChain().Config(), vm.Config{}, block, witness) if err != nil { log.Warn("ExecuteStatelessPayload: execution failed", "err", err) errorMsg := err.Error() From 9848e9b046f67bfef015b4ac7c834c8de9229147 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Mon, 2 Dec 2024 15:43:17 +0100 Subject: [PATCH 11/17] fuzzing: fix oss-fuzz fuzzer (#30845) The fuzzer added recenly to fuzz the eth handler doesn't build on oss-fuzz, because it also has dependencies in the peer_test.go. This change fixes it, I hope, by adding that file also for preprocessing. --- oss-fuzz.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/oss-fuzz.sh b/oss-fuzz.sh index 5e4aa1c253..1f222c433b 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -234,10 +234,6 @@ compile_fuzzer github.com/ethereum/go-ethereum/tests/fuzzers/secp256k1 \ compile_fuzzer github.com/ethereum/go-ethereum/eth/protocols/eth \ FuzzEthProtocolHandlers fuzz_eth_protocol_handlers \ - $repo/eth/protocols/eth/handler_test.go + $repo/eth/protocols/eth/handler_test.go,$repo/eth/protocols/eth/peer_test.go -#compile_fuzzer tests/fuzzers/vflux FuzzClientPool fuzzClientPool -#compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty -#compile_fuzzer tests/fuzzers/les Fuzz fuzzLes - From ae5a16f87038e420fdde7c48371eed3c35defa11 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 2 Dec 2024 18:17:43 +0100 Subject: [PATCH 12/17] internal/debug: rename --trace to --go-execution-trace (#30846) This flag is very rarely needed, so it's OK for it to have a verbose name. The name --trace also conflicts with the concept of EVM tracing, which is much more heavily used. --- internal/debug/flags.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 0e05975c7e..adb652d59b 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -136,8 +136,8 @@ var ( Category: flags.LoggingCategory, } traceFlag = &cli.StringFlag{ - Name: "trace", - Usage: "Write execution trace to the given file", + Name: "go-execution-trace", + Usage: "Write Go execution trace to the given file", Category: flags.LoggingCategory, } ) From 4afab7ef7614b6888abaf367a3a5852ecba9bee1 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 3 Dec 2024 09:30:26 +0100 Subject: [PATCH 13/17] eth/downloader: move SyncMode to package eth/ethconfig (#30847) Lots of packages depend on eth/downloader just for the SyncMode type. Since we have a dedicated package for eth protocol configuration, it makes more sense to define SyncMode there, turning eth/downloader into more of a leaf package. --- cmd/utils/flags.go | 7 ++-- eth/backend.go | 10 +++--- eth/catalyst/api.go | 6 ++-- eth/catalyst/api_test.go | 3 +- eth/catalyst/simulated_beacon_test.go | 3 +- eth/catalyst/tester.go | 4 +-- eth/downloader/beaconsync.go | 13 ++++---- eth/downloader/downloader.go | 32 +++++++++++++------ eth/downloader/queue.go | 7 ++-- eth/ethconfig/config.go | 5 ++- eth/ethconfig/gen_config.go | 5 ++- .../modes.go => ethconfig/syncmode.go} | 2 +- eth/handler.go | 5 +-- eth/handler_eth_test.go | 6 ++-- eth/handler_test.go | 4 +-- eth/sync_test.go | 4 +-- ethclient/simulated/backend.go | 3 +- 17 files changed, 64 insertions(+), 55 deletions(-) rename eth/{downloader/modes.go => ethconfig/syncmode.go} (99%) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e635dd89c3..9f309286bd 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -50,7 +50,6 @@ import ( "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/catalyst" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" @@ -1606,7 +1605,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { godebug.SetGCPercent(int(gogc)) if ctx.IsSet(SyncTargetFlag.Name) { - cfg.SyncMode = downloader.FullSync // dev sync target forces full sync + cfg.SyncMode = ethconfig.FullSync // dev sync target forces full sync } else if ctx.IsSet(SyncModeFlag.Name) { if err = cfg.SyncMode.UnmarshalText([]byte(ctx.String(SyncModeFlag.Name))); err != nil { Fatalf("invalid --syncmode flag: %v", err) @@ -1677,7 +1676,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } if !ctx.Bool(SnapshotFlag.Name) || cfg.SnapshotCache == 0 { // If snap-sync is requested, this flag is also required - if cfg.SyncMode == downloader.SnapSync { + if cfg.SyncMode == ethconfig.SnapSync { if !ctx.Bool(SnapshotFlag.Name) { log.Warn("Snap sync requested, enabling --snapshot") } @@ -1743,7 +1742,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if !ctx.IsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 } - cfg.SyncMode = downloader.FullSync + cfg.SyncMode = ethconfig.FullSync // Create new developer account or reuse existing one var ( developer accounts.Account diff --git a/eth/backend.go b/eth/backend.go index ccfe650f41..a3aa0a7b9b 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -424,17 +424,17 @@ func (s *Ethereum) Stop() error { // SyncMode retrieves the current sync mode, either explicitly set, or derived // from the chain status. -func (s *Ethereum) SyncMode() downloader.SyncMode { +func (s *Ethereum) SyncMode() ethconfig.SyncMode { // If we're in snap sync mode, return that directly if s.handler.snapSync.Load() { - return downloader.SnapSync + return ethconfig.SnapSync } // We are probably in full sync, but we might have rewound to before the // snap sync pivot, check if we should re-enable snap sync. head := s.blockchain.CurrentBlock() if pivot := rawdb.ReadLastPivotNumber(s.chainDb); pivot != nil { if head.Number.Uint64() < *pivot { - return downloader.SnapSync + return ethconfig.SnapSync } } // We are in a full sync, but the associated head state is missing. To complete @@ -442,8 +442,8 @@ func (s *Ethereum) SyncMode() downloader.SyncMode { // persistent state is corrupted, just mismatch with the head block. if !s.blockchain.HasState(head.Root) { log.Info("Reenabled snap sync as chain is stateless") - return downloader.SnapSync + return ethconfig.SnapSync } // Nope, we're really full syncing - return downloader.FullSync + return ethconfig.FullSync } diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index fa9752734e..3e45ad9e4f 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -33,7 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" @@ -918,7 +918,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe // tries to make it import a block. That should be denied as pushing something // into the database directly will conflict with the assumptions of snap sync // that it has an empty db that it can fill itself. - if api.eth.SyncMode() != downloader.FullSync { + if api.eth.SyncMode() != ethconfig.FullSync { return api.delayPayloadImport(block), nil } if !api.eth.BlockChain().HasBlockAndState(block.ParentHash(), block.NumberU64()-1) { @@ -1031,7 +1031,7 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) engine.PayloadSt // payload as non-integratable on top of the existing sync. We'll just // have to rely on the beacon client to forcefully update the head with // a forkchoice update request. - if api.eth.SyncMode() == downloader.FullSync { + if api.eth.SyncMode() == ethconfig.FullSync { // In full sync mode, failure to import a well-formed block can only mean // that the parent state is missing and the syncer rejected extending the // current cycle with the new payload. diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 3ac719c23e..e0a155f12b 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -40,7 +40,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/miner" @@ -452,7 +451,7 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) } mcfg := miner.DefaultConfig - ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: mcfg} + ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: ethconfig.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: mcfg} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) diff --git a/eth/catalyst/simulated_beacon_test.go b/eth/catalyst/simulated_beacon_test.go index 7e9fd7b324..79d9ba738e 100644 --- a/eth/catalyst/simulated_beacon_test.go +++ b/eth/catalyst/simulated_beacon_test.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" @@ -49,7 +48,7 @@ func startSimulatedBeaconEthService(t *testing.T, genesis *core.Genesis, period t.Fatal("can't create node:", err) } - ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: miner.DefaultConfig} + ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: ethconfig.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: miner.DefaultConfig} ethservice, err := eth.New(n, ethcfg) if err != nil { t.Fatal("can't create eth service:", err) diff --git a/eth/catalyst/tester.go b/eth/catalyst/tester.go index 0922ac0ba6..db2d638701 100644 --- a/eth/catalyst/tester.go +++ b/eth/catalyst/tester.go @@ -22,7 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" ) @@ -62,7 +62,7 @@ func (tester *FullSyncTester) Start() error { // Trigger beacon sync with the provided block hash as trusted // chain head. - err := tester.backend.Downloader().BeaconDevSync(downloader.FullSync, tester.target, tester.closed) + err := tester.backend.Downloader().BeaconDevSync(ethconfig.FullSync, tester.target, tester.closed) if err != nil { log.Info("Failed to trigger beacon sync", "err", err) } diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index e682536e07..c142ea7435 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/log" ) @@ -198,9 +199,9 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { var chainHead *types.Header switch d.getMode() { - case FullSync: + case ethconfig.FullSync: chainHead = d.blockchain.CurrentBlock() - case SnapSync: + case ethconfig.SnapSync: chainHead = d.blockchain.CurrentSnapBlock() default: panic("unknown sync mode") @@ -218,9 +219,9 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { } var linked bool switch d.getMode() { - case FullSync: + case ethconfig.FullSync: linked = d.blockchain.HasBlock(beaconTail.ParentHash, beaconTail.Number.Uint64()-1) - case SnapSync: + case ethconfig.SnapSync: linked = d.blockchain.HasFastBlock(beaconTail.ParentHash, beaconTail.Number.Uint64()-1) default: panic("unknown sync mode") @@ -253,9 +254,9 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { var known bool switch d.getMode() { - case FullSync: + case ethconfig.FullSync: known = d.blockchain.HasBlock(h.Hash(), n) - case SnapSync: + case ethconfig.SnapSync: known = d.blockchain.HasFastBlock(h.Hash(), n) default: panic("unknown sync mode") diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index fadb68ef03..8ac5d2eb31 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -69,6 +70,17 @@ var ( errNoPivotHeader = errors.New("pivot header is not found") ) +// SyncMode defines the sync method of the downloader. +// Deprecated: use ethconfig.SyncMode instead +type SyncMode = ethconfig.SyncMode + +const ( + // Deprecated: use ethconfig.FullSync + FullSync = ethconfig.FullSync + // Deprecated: use ethconfig.SnapSync + SnapSync = ethconfig.SnapSync +) + // peerDropFn is a callback type for dropping a peer detected as malicious. type peerDropFn func(id string) @@ -230,9 +242,9 @@ func (d *Downloader) Progress() ethereum.SyncProgress { current := uint64(0) mode := d.getMode() switch mode { - case FullSync: + case ethconfig.FullSync: current = d.blockchain.CurrentBlock().Number.Uint64() - case SnapSync: + case ethconfig.SnapSync: current = d.blockchain.CurrentSnapBlock().Number.Uint64() default: log.Error("Unknown downloader mode", "mode", mode) @@ -326,7 +338,7 @@ func (d *Downloader) synchronise(mode SyncMode, beaconPing chan struct{}) error if d.notified.CompareAndSwap(false, true) { log.Info("Block synchronisation started") } - if mode == SnapSync { + if mode == ethconfig.SnapSync { // Snap sync will directly modify the persistent state, making the entire // trie database unusable until the state is fully synced. To prevent any // subsequent state reads, explicitly disable the trie database and state @@ -434,7 +446,7 @@ func (d *Downloader) syncToHead() (err error) { // threshold (i.e. new chain). In that case we won't really snap sync // anyway, but still need a valid pivot block to avoid some code hitting // nil panics on access. - if mode == SnapSync && pivot == nil { + if mode == ethconfig.SnapSync && pivot == nil { pivot = d.blockchain.CurrentBlock() } height := latest.Number.Uint64() @@ -452,7 +464,7 @@ func (d *Downloader) syncToHead() (err error) { d.syncStatsLock.Unlock() // Ensure our origin point is below any snap sync pivot point - if mode == SnapSync { + if mode == ethconfig.SnapSync { if height <= uint64(fsMinFullBlocks) { origin = 0 } else { @@ -466,10 +478,10 @@ func (d *Downloader) syncToHead() (err error) { } } d.committed.Store(true) - if mode == SnapSync && pivot.Number.Uint64() != 0 { + if mode == ethconfig.SnapSync && pivot.Number.Uint64() != 0 { d.committed.Store(false) } - if mode == SnapSync { + if mode == ethconfig.SnapSync { // Set the ancient data limitation. If we are running snap sync, all block // data older than ancientLimit will be written to the ancient store. More // recent data will be written to the active database and will wait for the @@ -523,13 +535,13 @@ func (d *Downloader) syncToHead() (err error) { func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during snap sync func() error { return d.processHeaders(origin + 1) }, } - if mode == SnapSync { + if mode == ethconfig.SnapSync { d.pivotLock.Lock() d.pivotHeader = pivot d.pivotLock.Unlock() fetchers = append(fetchers, func() error { return d.processSnapSyncContent() }) - } else if mode == FullSync { + } else if mode == ethconfig.FullSync { fetchers = append(fetchers, func() error { return d.processFullSyncContent() }) } return d.spawnSync(fetchers) @@ -676,7 +688,7 @@ func (d *Downloader) processHeaders(origin uint64) error { chunkHashes := hashes[:limit] // In case of header only syncing, validate the chunk immediately - if mode == SnapSync { + if mode == ethconfig.SnapSync { // Although the received headers might be all valid, a legacy // PoW/PoA sync must not accept post-merge headers. Make sure // that any transition is rejected at this point. diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index a2f916ebbc..6c9175a95a 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common/prque" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto/kzg4844" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" @@ -180,7 +181,7 @@ func (q *queue) Reset(blockCacheLimit int, thresholdInitialSize int) { defer q.lock.Unlock() q.closed = false - q.mode = FullSync + q.mode = ethconfig.FullSync q.headerHead = common.Hash{} q.headerPendPool = make(map[string]*fetchRequest) @@ -328,7 +329,7 @@ func (q *queue) Schedule(headers []*types.Header, hashes []common.Hash, from uin q.blockTaskQueue.Push(header, -int64(header.Number.Uint64())) } // Queue for receipt retrieval - if q.mode == SnapSync && !header.EmptyReceipts() { + if q.mode == ethconfig.SnapSync && !header.EmptyReceipts() { if _, ok := q.receiptTaskPool[hash]; ok { log.Warn("Header already scheduled for receipt fetch", "number", header.Number, "hash", hash) } else { @@ -523,7 +524,7 @@ func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common // we can ask the resultcache if this header is within the // "prioritized" segment of blocks. If it is not, we need to throttle - stale, throttle, item, err := q.resultCache.AddFetch(header, q.mode == SnapSync) + stale, throttle, item, err := q.resultCache.AddFetch(header, q.mode == ethconfig.SnapSync) if stale { // Don't put back in the task queue, this item has already been // delivered upstream diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 8542bc97c4..6b75ab816f 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -49,7 +48,7 @@ var FullNodeGPO = gasprice.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ - SyncMode: downloader.SnapSync, + SyncMode: SnapSync, NetworkId: 0, // enable auto configuration of networkID == chainID TxLookupLimit: 2350000, TransactionHistory: 2350000, @@ -80,7 +79,7 @@ type Config struct { // Network ID separates blockchains on the peer-to-peer networking level. When left // zero, the chain ID is used as network ID. NetworkId uint64 - SyncMode downloader.SyncMode + SyncMode SyncMode // This can be set to list of enrtree:// URLs which will be queried for // nodes to connect to. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 0ec0eaddeb..8e954eaefb 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -9,7 +9,6 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/txpool/blobpool" "github.com/ethereum/go-ethereum/core/txpool/legacypool" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/miner" ) @@ -19,7 +18,7 @@ func (c Config) MarshalTOML() (interface{}, error) { type Config struct { Genesis *core.Genesis `toml:",omitempty"` NetworkId uint64 - SyncMode downloader.SyncMode + SyncMode SyncMode EthDiscoveryURLs []string SnapDiscoveryURLs []string NoPruning bool @@ -95,7 +94,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { type Config struct { Genesis *core.Genesis `toml:",omitempty"` NetworkId *uint64 - SyncMode *downloader.SyncMode + SyncMode *SyncMode EthDiscoveryURLs []string SnapDiscoveryURLs []string NoPruning *bool diff --git a/eth/downloader/modes.go b/eth/ethconfig/syncmode.go similarity index 99% rename from eth/downloader/modes.go rename to eth/ethconfig/syncmode.go index 9d8e1f313c..af5dbbb961 100644 --- a/eth/downloader/modes.go +++ b/eth/ethconfig/syncmode.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package downloader +package ethconfig import "fmt" diff --git a/eth/handler.go b/eth/handler.go index b28081eef0..583dc2835d 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/fetcher" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" @@ -87,7 +88,7 @@ type handlerConfig struct { Chain *core.BlockChain // Blockchain to serve data from TxPool txPool // Transaction pool to propagate from Network uint64 // Network identifier to advertise - Sync downloader.SyncMode // Whether to snap or full sync + Sync ethconfig.SyncMode // Whether to snap or full sync BloomCache uint64 // Megabytes to alloc for snap sync bloom EventMux *event.TypeMux // Legacy event mux, deprecate for `feed` RequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges @@ -145,7 +146,7 @@ func newHandler(config *handlerConfig) (*handler, error) { handlerDoneCh: make(chan struct{}), handlerStartCh: make(chan struct{}), } - if config.Sync == downloader.FullSync { + if config.Sync == ethconfig.FullSync { // The database seems empty as the current block is the genesis. Yet the snap // block is ahead, so snap sync was enabled for this node at a certain point. // The scenarios where this can happen is diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 55f7da87dd..ce17345358 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/p2p" @@ -109,7 +109,7 @@ func testForkIDSplit(t *testing.T, protocol uint) { Chain: chainNoFork, TxPool: newTestTxPool(), Network: 1, - Sync: downloader.FullSync, + Sync: ethconfig.FullSync, BloomCache: 1, }) ethProFork, _ = newHandler(&handlerConfig{ @@ -117,7 +117,7 @@ func testForkIDSplit(t *testing.T, protocol uint) { Chain: chainProFork, TxPool: newTestTxPool(), Network: 1, - Sync: downloader.FullSync, + Sync: ethconfig.FullSync, BloomCache: 1, }) ) diff --git a/eth/handler_test.go b/eth/handler_test.go index 7b250df2e9..b63d3e8592 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -164,7 +164,7 @@ func newTestHandlerWithBlocks(blocks int) *testHandler { Chain: chain, TxPool: txpool, Network: 1, - Sync: downloader.SnapSync, + Sync: ethconfig.SnapSync, BloomCache: 1, }) handler.Start(1000) diff --git a/eth/sync_test.go b/eth/sync_test.go index 7ede0a82c5..57eea73790 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -20,7 +20,7 @@ import ( "testing" "time" - "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/p2p" @@ -85,7 +85,7 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) { time.Sleep(250 * time.Millisecond) // Check that snap sync was disabled - if err := empty.handler.downloader.BeaconSync(downloader.SnapSync, full.chain.CurrentBlock(), nil); err != nil { + if err := empty.handler.downloader.BeaconSync(ethconfig.SnapSync, full.chain.CurrentBlock(), nil); err != nil { t.Fatal("sync failed:", err) } empty.handler.enableSyncedFeatures() diff --git a/ethclient/simulated/backend.go b/ethclient/simulated/backend.go index 6e07aa68d0..65d44b9efa 100644 --- a/ethclient/simulated/backend.go +++ b/ethclient/simulated/backend.go @@ -26,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/catalyst" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethclient" @@ -85,7 +84,7 @@ func NewBackend(alloc types.GenesisAlloc, options ...func(nodeConf *node.Config, GasLimit: ethconfig.Defaults.Miner.GasCeil, Alloc: alloc, } - ethConf.SyncMode = downloader.FullSync + ethConf.SyncMode = ethconfig.FullSync ethConf.TxPool.NoLocals = true for _, option := range options { From 84cabb587b4d0a6ba64d82212c7ed2f25bcbdb21 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 3 Dec 2024 16:11:26 +0100 Subject: [PATCH 14/17] CODEOWNERS: add some more entries for auto assignment (#30851) --- .github/CODEOWNERS | 39 +++++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 5852865f94..b569180852 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,25 +1,36 @@ # Lines starting with '#' are comments. # Each line is a file pattern followed by one or more owners. -accounts/usbwallet @karalabe -accounts/scwallet @gballet -accounts/abi @gballet @MariusVanDerWijden -beacon/engine @lightclient -cmd/clef @holiman -cmd/evm @holiman @MariusVanDerWijden @lightclient -consensus @karalabe -core/ @karalabe @holiman @rjl493456442 -eth/ @karalabe @holiman @rjl493456442 -eth/catalyst/ @gballet @lightclient +accounts/usbwallet/ @gballet +accounts/scwallet/ @gballet +accounts/abi/ @gballet @MariusVanDerWijden +beacon/engine/ @MariusVanDerWijden @lightclient @fjl +beacon/light/ @zsfelfoldi +beacon/merkle/ @zsfelfoldi +beacon/types/ @zsfelfoldi @fjl +beacon/params/ @zsfelfoldi @fjl +cmd/clef/ @holiman +cmd/evm/ @holiman @MariusVanDerWijden @lightclient +core/state/ @rjl493456442 @holiman +crypto/ @gballet @jwasinger @holiman @fjl +core/ @holiman @rjl493456442 +eth/ @holiman @rjl493456442 +eth/catalyst/ @MariusVanDerWijden @lightclient @fjl @jwasinger eth/tracers/ @s1na +ethclient/ @fjl +ethdb/ @rjl493456442 +event/ @fjl +trie/ @rjl493456442 +triedb/ @rjl493456442 core/tracing/ @s1na graphql/ @s1na -internal/ethapi @lightclient -internal/era @lightclient -les/ @zsfelfoldi @rjl493456442 -light/ @zsfelfoldi @rjl493456442 +internal/ethapi/ @fjl @s1na @lightclient +internal/era/ @lightclient +metrics/ @holiman +miner/ @MariusVanDerWijden @holiman @fjl @rjl493456442 node/ @fjl p2p/ @fjl @zsfelfoldi +rlp/ @fjl params/ @fjl @holiman @karalabe @gballet @rjl493456442 @zsfelfoldi rpc/ @fjl @holiman signer/ @holiman From f0e7382f38f7b4705ebd030a4dc78a8251a1a103 Mon Sep 17 00:00:00 2001 From: Martin HS Date: Wed, 4 Dec 2024 08:52:59 +0100 Subject: [PATCH 15/17] cmd/evm, eth/tracers: refactor structlogger and make it streaming (#30806) This PR refactors the structlog a bit, making it so that it can be used in a streaming mode. ------------- OBS: this PR makes a change in the input `config` config, the third input-parem field to `debug.traceCall`. Previously, seteting it to e.g. ` {"enableMemory": true, "limit": 1024}` would mean that the response was limited to `1024` items. Since an 'item' may include both memory and storage, the actual size of the response was undertermined. After this change, the response will be limited to `1024` __`bytes`__ (or thereabouts). ----------- The commandline usage of structlog now uses the streaming mode, leaving the non-streaming mode of operation for the eth_Call. There are two benefits of streaming mode 1. Not have to maintain a long list of operations, 2. Not have to duplicate / n-plicate data, e.g. memory / stack / returndata so that each entry has their own private slice. --------- Co-authored-by: Gary Rong --- cmd/evm/main.go | 4 +- cmd/evm/runner.go | 29 ++- core/vm/runtime/runtime_test.go | 19 +- eth/tracers/api_test.go | 2 +- eth/tracers/logger/logger.go | 311 ++++++++++++++++---------------- eth/tracers/tracers_test.go | 24 +-- 6 files changed, 200 insertions(+), 189 deletions(-) diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 750b269906..6e0345ce63 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -240,13 +240,13 @@ func tracerFromFlags(ctx *cli.Context) *tracing.Hooks { } switch { case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "struct": - return logger.NewStructLogger(config).Hooks() + return logger.NewStreamingStructLogger(config, os.Stderr).Hooks() case ctx.Bool(TraceFlag.Name) && ctx.String(TraceFormatFlag.Name) == "json": return logger.NewJSONLogger(config, os.Stderr) case ctx.Bool(MachineFlag.Name): return logger.NewJSONLogger(config, os.Stderr) case ctx.Bool(DebugFlag.Name): - return logger.NewStructLogger(config).Hooks() + return logger.NewStreamingStructLogger(config, os.Stderr).Hooks() default: return nil } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index afb0bec0ef..59ad8687d5 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "encoding/hex" "encoding/json" "fmt" "io" @@ -35,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm/runtime" "github.com/ethereum/go-ethereum/eth/tracers/logger" @@ -205,7 +207,6 @@ func runCmd(ctx *cli.Context) error { var ( tracer *tracing.Hooks - debugLogger *logger.StructLogger prestate *state.StateDB chainConfig *params.ChainConfig sender = common.BytesToAddress([]byte("sender")) @@ -217,10 +218,7 @@ func runCmd(ctx *cli.Context) error { if ctx.Bool(MachineFlag.Name) { tracer = logger.NewJSONLogger(logconfig, os.Stdout) } else if ctx.Bool(DebugFlag.Name) { - debugLogger = logger.NewStructLogger(logconfig) - tracer = debugLogger.Hooks() - } else { - debugLogger = logger.NewStructLogger(logconfig) + tracer = logger.NewStreamingStructLogger(logconfig, os.Stderr).Hooks() } initialGas := ctx.Uint64(GasFlag.Name) @@ -365,12 +363,10 @@ func runCmd(ctx *cli.Context) error { } if ctx.Bool(DebugFlag.Name) { - if debugLogger != nil { - fmt.Fprintln(os.Stderr, "#### TRACE ####") - logger.WriteTrace(os.Stderr, debugLogger.StructLogs()) + if logs := runtimeConfig.State.Logs(); len(logs) > 0 { + fmt.Fprintln(os.Stderr, "### LOGS") + writeLogs(os.Stderr, logs) } - fmt.Fprintln(os.Stderr, "#### LOGS ####") - logger.WriteLogs(os.Stderr, runtimeConfig.State.Logs()) } if bench || ctx.Bool(StatDumpFlag.Name) { @@ -389,3 +385,16 @@ allocated bytes: %d return nil } + +// writeLogs writes vm logs in a readable format to the given writer +func writeLogs(writer io.Writer, logs []*types.Log) { + for _, log := range logs { + fmt.Fprintf(writer, "LOG%d: %x bn=%d txi=%x\n", len(log.Topics), log.Address, log.BlockNumber, log.TxIndex) + + for i, topic := range log.Topics { + fmt.Fprintf(writer, "%08d %x\n", i, topic) + } + fmt.Fprint(writer, hex.Dump(log.Data)) + fmt.Fprintln(writer) + } +} diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 7d1345a57b..0e774a01c2 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/asm" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm/program" @@ -670,17 +671,23 @@ func TestColdAccountAccessCost(t *testing.T) { want: 7600, }, } { - tracer := logger.NewStructLogger(nil) + var step = 0 + var have = uint64(0) Execute(tc.code, nil, &Config{ EVMConfig: vm.Config{ - Tracer: tracer.Hooks(), + Tracer: &tracing.Hooks{ + OnOpcode: func(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { + // Uncomment to investigate failures: + //t.Logf("%d: %v %d", step, vm.OpCode(op).String(), cost) + if step == tc.step { + have = cost + } + step++ + }, + }, }, }) - have := tracer.StructLogs()[tc.step].GasCost if want := tc.want; have != want { - for ii, op := range tracer.StructLogs() { - t.Logf("%d: %v %d", ii, op.OpName(), op.GasCost) - } t.Fatalf("testcase %d, gas report wrong, step %d, have %d want %d", i, tc.step, have, want) } } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index e5940b6828..e786853ede 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -535,7 +535,7 @@ func TestTraceTransaction(t *testing.T) { Gas: params.TxGas, Failed: false, ReturnValue: "", - StructLogs: []logger.StructLogRes{}, + StructLogs: []json.RawMessage{}, }) { t.Error("Transaction tracing result is different") } diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index bdd4e07055..51b1512d03 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" "io" + "maps" "math/big" "strings" "sync/atomic" @@ -38,15 +39,6 @@ import ( // Storage represents a contract's storage. type Storage map[common.Hash]common.Hash -// Copy duplicates the current storage. -func (s Storage) Copy() Storage { - cpy := make(Storage, len(s)) - for key, value := range s { - cpy[key] = value - } - return cpy -} - // Config are the configuration options for structured logger the EVM type Config struct { EnableMemory bool // enable memory capture @@ -54,15 +46,16 @@ type Config struct { DisableStorage bool // disable storage capture EnableReturnData bool // enable return data capture Debug bool // print output during capture end - Limit int // maximum length of output, but zero means unlimited + Limit int // maximum size of output, but zero means unlimited + // Chain overrides, can be used to execute a trace using future fork rules Overrides *params.ChainConfig `json:"overrides,omitempty"` } //go:generate go run github.com/fjl/gencodec -type StructLog -field-override structLogMarshaling -out gen_structlog.go -// StructLog is emitted to the EVM each cycle and lists information about the current internal state -// prior to the execution of the statement. +// StructLog is emitted to the EVM each cycle and lists information about the +// current internal state prior to the execution of the statement. type StructLog struct { Pc uint64 `json:"pc"` Op vm.OpCode `json:"op"` @@ -102,29 +95,144 @@ func (s *StructLog) ErrorString() string { return "" } +// WriteTo writes the human-readable log data into the supplied writer. +func (s *StructLog) WriteTo(writer io.Writer) { + fmt.Fprintf(writer, "%-16spc=%08d gas=%v cost=%v", s.Op, s.Pc, s.Gas, s.GasCost) + if s.Err != nil { + fmt.Fprintf(writer, " ERROR: %v", s.Err) + } + fmt.Fprintln(writer) + + if len(s.Stack) > 0 { + fmt.Fprintln(writer, "Stack:") + for i := len(s.Stack) - 1; i >= 0; i-- { + fmt.Fprintf(writer, "%08d %s\n", len(s.Stack)-i-1, s.Stack[i].Hex()) + } + } + if len(s.Memory) > 0 { + fmt.Fprintln(writer, "Memory:") + fmt.Fprint(writer, hex.Dump(s.Memory)) + } + if len(s.Storage) > 0 { + fmt.Fprintln(writer, "Storage:") + for h, item := range s.Storage { + fmt.Fprintf(writer, "%x: %x\n", h, item) + } + } + if len(s.ReturnData) > 0 { + fmt.Fprintln(writer, "ReturnData:") + fmt.Fprint(writer, hex.Dump(s.ReturnData)) + } + fmt.Fprintln(writer) +} + +// structLogLegacy stores a structured log emitted by the EVM while replaying a +// transaction in debug mode. It's the legacy format used in tracer. The differences +// between the structLog json and the 'legacy' json are: +// +// op: +// Legacy uses string (e.g. "SSTORE"), non-legacy uses a byte. +// non-legacy has an 'opName' field containing the op name. +// +// gas, gasCost: +// Legacy uses integers, non-legacy hex-strings +// +// memory: +// Legacy uses a list of 64-char strings, each representing 32-byte chunks +// of evm memory. Non-legacy just uses a string of hexdata, no chunking. +// +// storage: +// Legacy has a storage field while non-legacy doesn't. +type structLogLegacy struct { + Pc uint64 `json:"pc"` + Op string `json:"op"` + Gas uint64 `json:"gas"` + GasCost uint64 `json:"gasCost"` + Depth int `json:"depth"` + Error string `json:"error,omitempty"` + Stack *[]string `json:"stack,omitempty"` + ReturnData string `json:"returnData,omitempty"` + Memory *[]string `json:"memory,omitempty"` + Storage *map[string]string `json:"storage,omitempty"` + RefundCounter uint64 `json:"refund,omitempty"` +} + +// toLegacyJSON converts the structLog to legacy json-encoded legacy form. +func (s *StructLog) toLegacyJSON() json.RawMessage { + msg := structLogLegacy{ + Pc: s.Pc, + Op: s.Op.String(), + Gas: s.Gas, + GasCost: s.GasCost, + Depth: s.Depth, + Error: s.ErrorString(), + RefundCounter: s.RefundCounter, + } + if s.Stack != nil { + stack := make([]string, len(s.Stack)) + for i, stackValue := range s.Stack { + stack[i] = stackValue.Hex() + } + msg.Stack = &stack + } + if len(s.ReturnData) > 0 { + msg.ReturnData = hexutil.Bytes(s.ReturnData).String() + } + if s.Memory != nil { + memory := make([]string, 0, (len(s.Memory)+31)/32) + for i := 0; i+32 <= len(s.Memory); i += 32 { + memory = append(memory, fmt.Sprintf("%x", s.Memory[i:i+32])) + } + msg.Memory = &memory + } + if s.Storage != nil { + storage := make(map[string]string) + for i, storageValue := range s.Storage { + storage[fmt.Sprintf("%x", i)] = fmt.Sprintf("%x", storageValue) + } + msg.Storage = &storage + } + element, _ := json.Marshal(msg) + return element +} + // StructLogger is an EVM state logger and implements EVMLogger. // // StructLogger can capture state based on the given Log configuration and also keeps // a track record of modified storage which is used in reporting snapshots of the // contract their storage. +// +// A StructLogger can either yield it's output immediately (streaming) or store for +// later output. type StructLogger struct { cfg Config env *tracing.VMContext storage map[common.Address]Storage - logs []StructLog output []byte err error usedGas uint64 + writer io.Writer // If set, the logger will stream instead of store logs + logs []json.RawMessage // buffer of json-encoded logs + resultSize int + interrupt atomic.Bool // Atomic flag to signal execution interruption reason error // Textual reason for the interruption } -// NewStructLogger returns a new logger +// NewStreamingStructLogger returns a new streaming logger. +func NewStreamingStructLogger(cfg *Config, writer io.Writer) *StructLogger { + l := NewStructLogger(cfg) + l.writer = writer + return l +} + +// NewStructLogger construct a new (non-streaming) struct logger. func NewStructLogger(cfg *Config) *StructLogger { logger := &StructLogger{ storage: make(map[common.Address]Storage), + logs: make([]json.RawMessage, 0), } if cfg != nil { logger.cfg = *cfg @@ -141,44 +249,36 @@ func (l *StructLogger) Hooks() *tracing.Hooks { } } -// Reset clears the data held by the logger. -func (l *StructLogger) Reset() { - l.storage = make(map[common.Address]Storage) - l.output = make([]byte, 0) - l.logs = l.logs[:0] - l.err = nil -} - // OnOpcode logs a new structured log message and pushes it out to the environment // // OnOpcode also tracks SLOAD/SSTORE ops to track storage change. func (l *StructLogger) OnOpcode(pc uint64, opcode byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { - // If tracing was interrupted, set the error and stop + // If tracing was interrupted, exit if l.interrupt.Load() { return } - // check if already accumulated the specified number of logs - if l.cfg.Limit != 0 && l.cfg.Limit <= len(l.logs) { + // check if already accumulated the size of the response. + if l.cfg.Limit != 0 && l.resultSize > l.cfg.Limit { return } - - op := vm.OpCode(opcode) - memory := scope.MemoryData() - stack := scope.StackData() - // Copy a snapshot of the current memory state to a new buffer - var mem []byte + var ( + op = vm.OpCode(opcode) + memory = scope.MemoryData() + contractAddr = scope.Address() + stack = scope.StackData() + stackLen = len(stack) + ) + log := StructLog{pc, op, gas, cost, nil, len(memory), nil, nil, nil, depth, l.env.StateDB.GetRefund(), err} if l.cfg.EnableMemory { - mem = make([]byte, len(memory)) - copy(mem, memory) + log.Memory = memory } - // Copy a snapshot of the current stack state to a new buffer - var stck []uint256.Int if !l.cfg.DisableStack { - stck = make([]uint256.Int, len(stack)) - copy(stck, stack) + log.Stack = scope.StackData() } - contractAddr := scope.Address() - stackLen := len(stack) + if l.cfg.EnableReturnData { + log.ReturnData = rData + } + // Copy a snapshot of the current storage to a new container var storage Storage if !l.cfg.DisableStorage && (op == vm.SLOAD || op == vm.SSTORE) { @@ -194,7 +294,7 @@ func (l *StructLogger) OnOpcode(pc uint64, opcode byte, gas, cost uint64, scope value = l.env.StateDB.GetState(contractAddr, address) ) l.storage[contractAddr][address] = value - storage = l.storage[contractAddr].Copy() + storage = maps.Clone(l.storage[contractAddr]) } else if op == vm.SSTORE && stackLen >= 2 { // capture SSTORE opcodes and record the written entry in the local storage. var ( @@ -202,17 +302,19 @@ func (l *StructLogger) OnOpcode(pc uint64, opcode byte, gas, cost uint64, scope address = common.Hash(stack[stackLen-1].Bytes32()) ) l.storage[contractAddr][address] = value - storage = l.storage[contractAddr].Copy() + storage = maps.Clone(l.storage[contractAddr]) } } - var rdata []byte - if l.cfg.EnableReturnData { - rdata = make([]byte, len(rData)) - copy(rdata, rData) + log.Storage = storage + + // create a log + if l.writer == nil { + entry := log.toLegacyJSON() + l.resultSize += len(entry) + l.logs = append(l.logs, entry) + return } - // create a new snapshot of the EVM. - log := StructLog{pc, op, gas, cost, mem, len(memory), stck, rdata, storage, depth, l.env.StateDB.GetRefund(), err} - l.logs = append(l.logs, log) + log.WriteTo(l.writer) } // OnExit is called a call frame finishes processing. @@ -246,7 +348,7 @@ func (l *StructLogger) GetResult() (json.RawMessage, error) { Gas: l.usedGas, Failed: failed, ReturnValue: returnVal, - StructLogs: formatLogs(l.StructLogs()), + StructLogs: l.logs, }) } @@ -273,9 +375,6 @@ func (l *StructLogger) OnTxEnd(receipt *types.Receipt, err error) { } } -// StructLogs returns the captured log entries. -func (l *StructLogger) StructLogs() []StructLog { return l.logs } - // Error returns the VM error captured by the trace. func (l *StructLogger) Error() error { return l.err } @@ -283,49 +382,10 @@ func (l *StructLogger) Error() error { return l.err } func (l *StructLogger) Output() []byte { return l.output } // WriteTrace writes a formatted trace to the given writer +// @deprecated func WriteTrace(writer io.Writer, logs []StructLog) { for _, log := range logs { - fmt.Fprintf(writer, "%-16spc=%08d gas=%v cost=%v", log.Op, log.Pc, log.Gas, log.GasCost) - if log.Err != nil { - fmt.Fprintf(writer, " ERROR: %v", log.Err) - } - fmt.Fprintln(writer) - - if len(log.Stack) > 0 { - fmt.Fprintln(writer, "Stack:") - for i := len(log.Stack) - 1; i >= 0; i-- { - fmt.Fprintf(writer, "%08d %s\n", len(log.Stack)-i-1, log.Stack[i].Hex()) - } - } - if len(log.Memory) > 0 { - fmt.Fprintln(writer, "Memory:") - fmt.Fprint(writer, hex.Dump(log.Memory)) - } - if len(log.Storage) > 0 { - fmt.Fprintln(writer, "Storage:") - for h, item := range log.Storage { - fmt.Fprintf(writer, "%x: %x\n", h, item) - } - } - if len(log.ReturnData) > 0 { - fmt.Fprintln(writer, "ReturnData:") - fmt.Fprint(writer, hex.Dump(log.ReturnData)) - } - fmt.Fprintln(writer) - } -} - -// WriteLogs writes vm logs in a readable format to the given writer -func WriteLogs(writer io.Writer, logs []*types.Log) { - for _, log := range logs { - fmt.Fprintf(writer, "LOG%d: %x bn=%d txi=%x\n", len(log.Topics), log.Address, log.BlockNumber, log.TxIndex) - - for i, topic := range log.Topics { - fmt.Fprintf(writer, "%08d %x\n", i, topic) - } - - fmt.Fprint(writer, hex.Dump(log.Data)) - fmt.Fprintln(writer) + log.WriteTo(writer) } } @@ -425,65 +485,8 @@ func (t *mdLogger) OnFault(pc uint64, op byte, gas, cost uint64, scope tracing.O // while replaying a transaction in debug mode as well as transaction // execution status, the amount of gas used and the return value type ExecutionResult struct { - Gas uint64 `json:"gas"` - Failed bool `json:"failed"` - ReturnValue string `json:"returnValue"` - StructLogs []StructLogRes `json:"structLogs"` -} - -// StructLogRes stores a structured log emitted by the EVM while replaying a -// transaction in debug mode -type StructLogRes struct { - Pc uint64 `json:"pc"` - Op string `json:"op"` - Gas uint64 `json:"gas"` - GasCost uint64 `json:"gasCost"` - Depth int `json:"depth"` - Error string `json:"error,omitempty"` - Stack *[]string `json:"stack,omitempty"` - ReturnData string `json:"returnData,omitempty"` - Memory *[]string `json:"memory,omitempty"` - Storage *map[string]string `json:"storage,omitempty"` - RefundCounter uint64 `json:"refund,omitempty"` -} - -// formatLogs formats EVM returned structured logs for json output -func formatLogs(logs []StructLog) []StructLogRes { - formatted := make([]StructLogRes, len(logs)) - for index, trace := range logs { - formatted[index] = StructLogRes{ - Pc: trace.Pc, - Op: trace.Op.String(), - Gas: trace.Gas, - GasCost: trace.GasCost, - Depth: trace.Depth, - Error: trace.ErrorString(), - RefundCounter: trace.RefundCounter, - } - if trace.Stack != nil { - stack := make([]string, len(trace.Stack)) - for i, stackValue := range trace.Stack { - stack[i] = stackValue.Hex() - } - formatted[index].Stack = &stack - } - if len(trace.ReturnData) > 0 { - formatted[index].ReturnData = hexutil.Bytes(trace.ReturnData).String() - } - if trace.Memory != nil { - memory := make([]string, 0, (len(trace.Memory)+31)/32) - for i := 0; i+32 <= len(trace.Memory); i += 32 { - memory = append(memory, fmt.Sprintf("%x", trace.Memory[i:i+32])) - } - formatted[index].Memory = &memory - } - if trace.Storage != nil { - storage := make(map[string]string) - for i, storageValue := range trace.Storage { - storage[fmt.Sprintf("%x", i)] = fmt.Sprintf("%x", storageValue) - } - formatted[index].Storage = &storage - } - } - return formatted + Gas uint64 `json:"gas"` + Failed bool `json:"failed"` + ReturnValue string `json:"returnValue"` + StructLogs []json.RawMessage `json:"structLogs"` } diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 9d8f70386f..13c36379e7 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/tests" ) -func BenchmarkTransactionTrace(b *testing.B) { +func BenchmarkTransactionTraceV2(b *testing.B) { key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") from := crypto.PubkeyToAddress(key.PublicKey) gas := uint64(1000000) // 1M gas @@ -78,14 +78,8 @@ func BenchmarkTransactionTrace(b *testing.B) { state := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false, rawdb.HashScheme) defer state.Close() - // Create the tracer, the EVM environment and run it - tracer := logger.NewStructLogger(&logger.Config{ - Debug: false, - //DisableStorage: true, - //EnableMemory: false, - //EnableReturnData: false, - }) - evm := vm.NewEVM(context, state.StateDB, params.AllEthashProtocolChanges, vm.Config{Tracer: tracer.Hooks()}) + evm := vm.NewEVM(context, state.StateDB, params.AllEthashProtocolChanges, vm.Config{}) + msg, err := core.TransactionToMessage(tx, signer, context.BaseFee) if err != nil { b.Fatalf("failed to prepare transaction for tracing: %v", err) @@ -94,17 +88,15 @@ func BenchmarkTransactionTrace(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { - snap := state.StateDB.Snapshot() + tracer := logger.NewStructLogger(&logger.Config{Debug: false}).Hooks() tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - res, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + evm.Config.Tracer = tracer + + snap := state.StateDB.Snapshot() + _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) if err != nil { b.Fatal(err) } - tracer.OnTxEnd(&types.Receipt{GasUsed: res.UsedGas}, nil) state.StateDB.RevertToSnapshot(snap) - if have, want := len(tracer.StructLogs()), 244752; have != want { - b.Fatalf("trace wrong, want %d steps, have %d", want, have) - } - tracer.Reset() } } From 67a3b087951a3f3a8e341ae32b6ec18f3553e5cc Mon Sep 17 00:00:00 2001 From: Nebojsa Urosevic Date: Wed, 4 Dec 2024 00:40:37 -0800 Subject: [PATCH 16/17] core/tracing: extends tracing.Hooks with OnSystemCallStartV2 (#30786) This PR extends the Hooks interface with a new method, `OnSystemCallStartV2`, which takes `VMContext` as its parameter. Motivation By including `VMContext` as a parameter, the `OnSystemCallStartV2` hook achieves parity with the `OnTxStart` hook in terms of provided insights. This alignment simplifies the inner tracer logic, enabling consistent handling of state changes and internal calls within the same framework. --------- Co-authored-by: Sina Mahmoodi --- core/state_processor.go | 21 ++++++++++++--------- core/tracing/hooks.go | 21 +++++++++++++-------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/core/state_processor.go b/core/state_processor.go index d8637a2a52..3eb83a673a 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/tracing" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -212,9 +213,7 @@ func ApplyTransaction(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header * // contract. This method is exported to be used in tests. func ProcessBeaconBlockRoot(beaconRoot common.Hash, evm *vm.EVM) { if tracer := evm.Config.Tracer; tracer != nil { - if tracer.OnSystemCallStart != nil { - tracer.OnSystemCallStart() - } + onSystemCallStart(tracer, evm.GetVMContext()) if tracer.OnSystemCallEnd != nil { defer tracer.OnSystemCallEnd() } @@ -238,9 +237,7 @@ func ProcessBeaconBlockRoot(beaconRoot common.Hash, evm *vm.EVM) { // as per EIP-2935. func ProcessParentBlockHash(prevHash common.Hash, evm *vm.EVM) { if tracer := evm.Config.Tracer; tracer != nil { - if tracer.OnSystemCallStart != nil { - tracer.OnSystemCallStart() - } + onSystemCallStart(tracer, evm.GetVMContext()) if tracer.OnSystemCallEnd != nil { defer tracer.OnSystemCallEnd() } @@ -274,9 +271,7 @@ func ProcessConsolidationQueue(requests *[][]byte, evm *vm.EVM) { func processRequestsSystemCall(requests *[][]byte, evm *vm.EVM, requestType byte, addr common.Address) { if tracer := evm.Config.Tracer; tracer != nil { - if tracer.OnSystemCallStart != nil { - tracer.OnSystemCallStart() - } + onSystemCallStart(tracer, evm.GetVMContext()) if tracer.OnSystemCallEnd != nil { defer tracer.OnSystemCallEnd() } @@ -322,3 +317,11 @@ func ParseDepositLogs(requests *[][]byte, logs []*types.Log, config *params.Chai } return nil } + +func onSystemCallStart(tracer *tracing.Hooks, ctx *tracing.VMContext) { + if tracer.OnSystemCallStartV2 != nil { + tracer.OnSystemCallStartV2(ctx) + } else if tracer.OnSystemCallStart != nil { + tracer.OnSystemCallStart() + } +} diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go index 5218e80c9b..728f15069b 100644 --- a/core/tracing/hooks.go +++ b/core/tracing/hooks.go @@ -145,6 +145,10 @@ type ( // will not be invoked. OnSystemCallStartHook = func() + // OnSystemCallStartHookV2 is called when a system call is about to be executed. Refer + // to `OnSystemCallStartHook` for more information. + OnSystemCallStartHookV2 = func(vm *VMContext) + // OnSystemCallEndHook is called when a system call has finished executing. Today, // this hook is invoked when the EIP-4788 system call is about to be executed to set the // beacon block root. @@ -180,14 +184,15 @@ type Hooks struct { OnFault FaultHook OnGasChange GasChangeHook // Chain events - OnBlockchainInit BlockchainInitHook - OnClose CloseHook - OnBlockStart BlockStartHook - OnBlockEnd BlockEndHook - OnSkippedBlock SkippedBlockHook - OnGenesisBlock GenesisBlockHook - OnSystemCallStart OnSystemCallStartHook - OnSystemCallEnd OnSystemCallEndHook + OnBlockchainInit BlockchainInitHook + OnClose CloseHook + OnBlockStart BlockStartHook + OnBlockEnd BlockEndHook + OnSkippedBlock SkippedBlockHook + OnGenesisBlock GenesisBlockHook + OnSystemCallStart OnSystemCallStartHook + OnSystemCallStartV2 OnSystemCallStartHookV2 + OnSystemCallEnd OnSystemCallEndHook // State events OnBalanceChange BalanceChangeHook OnNonceChange NonceChangeHook From 08e6bdb550712503873fb2a138b30132cc36c481 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Fri, 6 Dec 2024 12:14:05 +0100 Subject: [PATCH 17/17] trie/utils: ensure master can generate a correct genesis for kaustinen7 (#30856) This imports the following fixes: - update gnark to 1.1.0 - update go-verkle to 0.2.2 - fix: main storage offset bug (gballet/go-ethereum#329) - fix: tree key generation (gballet/go-ethereum#401) --------- Signed-off-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Co-authored-by: Ignacio Hagopian --- core/genesis_test.go | 2 +- go.mod | 26 +++++++++---------- go.sum | 59 ++++++++++++++++++++++---------------------- trie/utils/verkle.go | 16 +++--------- trie/verkle_test.go | 4 +-- 5 files changed, 50 insertions(+), 57 deletions(-) diff --git a/core/genesis_test.go b/core/genesis_test.go index 9eacf2024c..3ec87474e5 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -293,7 +293,7 @@ func TestVerkleGenesisCommit(t *testing.T) { }, } - expected := common.FromHex("4a83dc39eb688dbcfaf581d60e82de18f875e38786ebce5833342011d6fef37b") + expected := common.FromHex("018d20eebb130b5e2b796465fe36aafab650650729a92435aec071bf2386f080") got := genesis.ToBlock().Root().Bytes() if !bytes.Equal(got, expected) { t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) diff --git a/go.mod b/go.mod index c467cef169..b33c30f41d 100644 --- a/go.mod +++ b/go.mod @@ -15,16 +15,16 @@ require ( github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.79.0 github.com/cockroachdb/pebble v1.1.2 - github.com/consensys/gnark-crypto v0.12.1 - github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c - github.com/crate-crypto/go-kzg-4844 v1.0.0 + github.com/consensys/gnark-crypto v0.14.0 + github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a + github.com/crate-crypto/go-kzg-4844 v1.1.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.6.0 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 github.com/ethereum/c-kzg-4844 v1.0.0 - github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 + github.com/ethereum/go-verkle v0.2.2 github.com/fatih/color v1.16.0 github.com/ferranbt/fastssz v0.1.2 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e @@ -64,13 +64,13 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/urfave/cli/v2 v2.25.7 go.uber.org/automaxprocs v1.5.2 - golang.org/x/crypto v0.22.0 + golang.org/x/crypto v0.26.0 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa - golang.org/x/sync v0.7.0 - golang.org/x/sys v0.22.0 - golang.org/x/text v0.14.0 + golang.org/x/sync v0.10.0 + golang.org/x/sys v0.28.0 + golang.org/x/text v0.17.0 golang.org/x/time v0.5.0 - golang.org/x/tools v0.20.0 + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d google.golang.org/protobuf v1.34.2 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 @@ -91,14 +91,14 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect github.com/aws/smithy-go v1.15.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.13.0 // indirect + github.com/bits-and-blooms/bitset v1.17.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/errors v1.11.3 // indirect github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/bavard v0.1.22 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/deepmap/oapi-codegen v1.6.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect @@ -135,13 +135,13 @@ require ( github.com/prometheus/common v0.32.1 // indirect github.com/prometheus/procfs v0.7.3 // indirect github.com/rivo/uniseg v0.2.0 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.24.0 // indirect + golang.org/x/net v0.25.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) diff --git a/go.sum b/go.sum index 19a7b9d25c..b61121664f 100644 --- a/go.sum +++ b/go.sum @@ -90,8 +90,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= -github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.17.0 h1:1X2TS7aHz1ELcC0yU1y2stUs/0ig5oMU6STFZGrhvHI= +github.com/bits-and-blooms/bitset v1.17.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= @@ -124,16 +124,16 @@ github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwP github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= -github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= -github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/consensys/bavard v0.1.22 h1:Uw2CGvbXSZWhqK59X0VG/zOjpTFuOMcPLStrp1ihI0A= +github.com/consensys/bavard v0.1.22/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= +github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E= +github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCeWHHrb1li/n1XoU0= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= -github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= -github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= -github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= +github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= +github.com/crate-crypto/go-kzg-4844 v1.1.0 h1:EN/u9k2TF6OWSHrCCDBBU6GLNMq88OspHHlMnHfoyU4= +github.com/crate-crypto/go-kzg-4844 v1.1.0/go.mod h1:JolLjpSff1tCCJKaJx4psrlEdlXuJEC996PL3tTAFks= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -166,8 +166,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A= -github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= +github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= +github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/ferranbt/fastssz v0.1.2 h1:Dky6dXlngF6Qjc+EfDipAkE83N5I5DE68bY6O0VLNPk= @@ -261,8 +261,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -363,8 +363,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= -github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= +github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= @@ -469,8 +469,9 @@ github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -527,8 +528,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -600,8 +601,8 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -620,8 +621,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -683,8 +684,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -697,8 +698,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -749,8 +750,8 @@ golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.20.0 h1:hz/CVckiOxybQvFw6h7b/q80NTr9IUQb4s1IIzW7KNY= -golang.org/x/tools v0.20.0/go.mod h1:WvitBU7JJf6A4jOdg4S1tviW9bhUxkgeCui/0JHctQg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/trie/utils/verkle.go b/trie/utils/verkle.go index b785e512b1..1d33f6c3e5 100644 --- a/trie/utils/verkle.go +++ b/trie/utils/verkle.go @@ -41,10 +41,10 @@ var ( zero = uint256.NewInt(0) verkleNodeWidthLog2 = 8 headerStorageOffset = uint256.NewInt(64) - mainStorageOffsetLshVerkleNodeWidth = new(uint256.Int).Lsh(uint256.NewInt(256), 31-uint(verkleNodeWidthLog2)) codeOffset = uint256.NewInt(128) verkleNodeWidth = uint256.NewInt(256) codeStorageDelta = uint256.NewInt(0).Sub(codeOffset, headerStorageOffset) + mainStorageOffsetLshVerkleNodeWidth = new(uint256.Int).Lsh(uint256.NewInt(1), 248-uint(verkleNodeWidthLog2)) index0Point *verkle.Point // pre-computed commitment of polynomial [2+256*64] @@ -273,17 +273,9 @@ func StorageSlotKeyWithEvaluatedAddress(evaluated *verkle.Point, storageKey []by } func pointToHash(evaluated *verkle.Point, suffix byte) []byte { - // The output of Byte() is big endian for banderwagon. This - // introduces an imbalance in the tree, because hashes are - // elements of a 253-bit field. This means more than half the - // tree would be empty. To avoid this problem, use a little - // endian commitment and chop the MSB. - bytes := evaluated.Bytes() - for i := 0; i < 16; i++ { - bytes[31-i], bytes[i] = bytes[i], bytes[31-i] - } - bytes[31] = suffix - return bytes[:] + retb := verkle.HashPointToBytes(evaluated) + retb[31] = suffix + return retb[:] } func evaluateAddressPoint(address []byte) *verkle.Point { diff --git a/trie/verkle_test.go b/trie/verkle_test.go index 4cd1717c0e..84eec2ed30 100644 --- a/trie/verkle_test.go +++ b/trie/verkle_test.go @@ -136,8 +136,8 @@ func TestVerkleRollBack(t *testing.T) { } } - // ensure there is some code in the 2nd group - keyOf2ndGroup := []byte{141, 124, 185, 236, 50, 22, 185, 39, 244, 47, 97, 209, 96, 235, 22, 13, 205, 38, 18, 201, 128, 223, 0, 59, 146, 199, 222, 119, 133, 13, 91, 0} + // ensure there is some code in the 2nd group of the 1st account + keyOf2ndGroup := utils.CodeChunkKeyWithEvaluatedAddress(tr.cache.Get(common.Address{1}.Bytes()), uint256.NewInt(128)) chunk, err := tr.root.Get(keyOf2ndGroup, nil) if err != nil { t.Fatalf("Failed to get account, %v", err)