diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4bc13aff92..39554550c4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,6 +10,7 @@ beacon/merkle/ @zsfelfoldi beacon/types/ @zsfelfoldi @fjl beacon/params/ @zsfelfoldi @fjl cmd/evm/ @MariusVanDerWijden @lightclient +cmd/keeper/ @gballet core/state/ @rjl493456442 crypto/ @gballet @jwasinger @fjl core/ @rjl493456442 diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 50c9fe7f75..507057afe5 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -69,8 +69,8 @@ jobs: - name: Install cross toolchain run: | - apt-get update - apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib + sudo apt-get update + sudo apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib - name: Build run: go run build/ci.go test -arch 386 -short -p 8 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000..91032c8f17 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,102 @@ +# AGENTS + +## Guidelines + +- **Keep changes minimal and focused.** Only modify code directly related to the task at hand. Do not refactor unrelated code, rename existing variables or functions for style, or bundle unrelated fixes into the same commit or PR. +- **Do not add, remove, or update dependencies** unless the task explicitly requires it. + +## Pre-Commit Checklist + +Before every commit, run **all** of the following checks and ensure they pass: + +### 1. Formatting + +Before committing, always run `gofmt` and `goimports` on all modified files: + +```sh +gofmt -w +goimports -w +``` + +### 2. Build All Commands + +Verify that all tools compile successfully: + +```sh +make all +``` + +This builds all executables under `cmd/`, including `keeper` which has special build requirements. + +### 3. Tests + +While iterating during development, use `-short` for faster feedback: + +```sh +go run ./build/ci.go test -short +``` + +Before committing, run the full test suite **without** `-short` to ensure all tests pass, including the Ethereum execution-spec tests and all state/block test permutations: + +```sh +go run ./build/ci.go test +``` + +### 4. Linting + +```sh +go run ./build/ci.go lint +``` + +This runs additional style checks. Fix any issues before committing. + +### 5. Generated Code + +```sh +go run ./build/ci.go check_generate +``` + +Ensures that all generated files (e.g., `gen_*.go`) are up to date. If this fails, first install the required code generators by running `make devtools`, then run the appropriate `go generate` commands and include the updated files in your commit. + +### 6. Dependency Hygiene + +```sh +go run ./build/ci.go check_baddeps +``` + +Verifies that no forbidden dependencies have been introduced. + +## What to include in commits + +Do not commit binaries, whether they are produced by the main build or byproducts of investigations. + +## Commit Message Format + +Commit messages must be prefixed with the package(s) they modify, followed by a short lowercase description: + +``` +: description +``` + +Examples: +- `core/vm: fix stack overflow in PUSH instruction` +- `eth, rpc: make trace configs optional` +- `cmd/geth: add new flag for sync mode` + +Use comma-separated package names when multiple areas are affected. Keep the description concise. + +## Pull Request Title Format + +PR titles follow the same convention as commit messages: + +``` +: description +``` + +Examples: +- `core/vm: fix stack overflow in PUSH instruction` +- `core, eth: add arena allocator support` +- `cmd/geth, internal/ethapi: refactor transaction args` +- `trie/archiver: streaming subtree archival to fix OOM` + +Use the top-level package paths, comma-separated if multiple areas are affected. Only mention the directories with functional changes, interface changes that trickle all over the codebase should not generate an exhaustive list. The description should be a short, lowercase summary of the change. diff --git a/Dockerfile b/Dockerfile index 9b70e9e8a0..940e568cab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.24-alpine AS builder +FROM golang:1.26-alpine AS builder RUN apk add --no-cache gcc musl-dev linux-headers git diff --git a/Dockerfile.alltools b/Dockerfile.alltools index ac9303c678..ae2ca931ae 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.24-alpine AS builder +FROM golang:1.26-alpine AS builder RUN apk add --no-cache gcc musl-dev linux-headers git diff --git a/beacon/blsync/engineclient.go b/beacon/blsync/engineclient.go index 9fc6a18a57..6cac2bc5a8 100644 --- a/beacon/blsync/engineclient.go +++ b/beacon/blsync/engineclient.go @@ -87,6 +87,10 @@ func (ec *engineClient) updateLoop(headCh <-chan types.ChainHeadEvent) { if status, err := ec.callForkchoiceUpdated(forkName, event); err == nil { log.Info("Successful ForkchoiceUpdated", "head", event.Block.Hash(), "status", status) } else { + if err.Error() == "beacon syncer reorging" { + log.Debug("Failed ForkchoiceUpdated", "head", event.Block.Hash(), "error", err) + continue // ignore beacon syncer reorging errors, this error can occur if the blsync is skipping a block + } log.Error("Failed ForkchoiceUpdated", "head", event.Block.Hash(), "error", err) } } diff --git a/beacon/engine/gen_blockparams.go b/beacon/engine/gen_blockparams.go index b1f01b50ff..3a5d7ae593 100644 --- a/beacon/engine/gen_blockparams.go +++ b/beacon/engine/gen_blockparams.go @@ -21,6 +21,7 @@ func (p PayloadAttributes) MarshalJSON() ([]byte, error) { SuggestedFeeRecipient common.Address `json:"suggestedFeeRecipient" gencodec:"required"` Withdrawals []*types.Withdrawal `json:"withdrawals"` BeaconRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *hexutil.Uint64 `json:"slotNumber"` } var enc PayloadAttributes enc.Timestamp = hexutil.Uint64(p.Timestamp) @@ -28,6 +29,7 @@ func (p PayloadAttributes) MarshalJSON() ([]byte, error) { enc.SuggestedFeeRecipient = p.SuggestedFeeRecipient enc.Withdrawals = p.Withdrawals enc.BeaconRoot = p.BeaconRoot + enc.SlotNumber = (*hexutil.Uint64)(p.SlotNumber) return json.Marshal(&enc) } @@ -39,6 +41,7 @@ func (p *PayloadAttributes) UnmarshalJSON(input []byte) error { SuggestedFeeRecipient *common.Address `json:"suggestedFeeRecipient" gencodec:"required"` Withdrawals []*types.Withdrawal `json:"withdrawals"` BeaconRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *hexutil.Uint64 `json:"slotNumber"` } var dec PayloadAttributes if err := json.Unmarshal(input, &dec); err != nil { @@ -62,5 +65,8 @@ func (p *PayloadAttributes) UnmarshalJSON(input []byte) error { if dec.BeaconRoot != nil { p.BeaconRoot = dec.BeaconRoot } + if dec.SlotNumber != nil { + p.SlotNumber = (*uint64)(dec.SlotNumber) + } return nil } diff --git a/beacon/engine/gen_ed.go b/beacon/engine/gen_ed.go index 6893d64a16..b460368b84 100644 --- a/beacon/engine/gen_ed.go +++ b/beacon/engine/gen_ed.go @@ -34,6 +34,7 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) { Withdrawals []*types.Withdrawal `json:"withdrawals"` BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` + SlotNumber *hexutil.Uint64 `json:"slotNumber"` } var enc ExecutableData enc.ParentHash = e.ParentHash @@ -58,6 +59,7 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) { enc.Withdrawals = e.Withdrawals enc.BlobGasUsed = (*hexutil.Uint64)(e.BlobGasUsed) enc.ExcessBlobGas = (*hexutil.Uint64)(e.ExcessBlobGas) + enc.SlotNumber = (*hexutil.Uint64)(e.SlotNumber) return json.Marshal(&enc) } @@ -81,6 +83,7 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { Withdrawals []*types.Withdrawal `json:"withdrawals"` BlobGasUsed *hexutil.Uint64 `json:"blobGasUsed"` ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas"` + SlotNumber *hexutil.Uint64 `json:"slotNumber"` } var dec ExecutableData if err := json.Unmarshal(input, &dec); err != nil { @@ -154,5 +157,8 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { if dec.ExcessBlobGas != nil { e.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas) } + if dec.SlotNumber != nil { + e.SlotNumber = (*uint64)(dec.SlotNumber) + } return nil } diff --git a/beacon/engine/types.go b/beacon/engine/types.go index da9b6568f2..5c94e67de1 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -50,6 +50,13 @@ var ( // ExecutionPayloadV3 has the syntax of ExecutionPayloadV2 and appends the new // fields: blobGasUsed and excessBlobGas. PayloadV3 PayloadVersion = 0x3 + + // PayloadV4 is the identifier of ExecutionPayloadV4 introduced in amsterdam fork. + // + // https://github.com/ethereum/execution-apis/blob/main/src/engine/amsterdam.md#executionpayloadv4 + // ExecutionPayloadV4 has the syntax of ExecutionPayloadV3 and appends the new + // field slotNumber. + PayloadV4 PayloadVersion = 0x4 ) //go:generate go run github.com/fjl/gencodec -type PayloadAttributes -field-override payloadAttributesMarshaling -out gen_blockparams.go @@ -62,11 +69,13 @@ type PayloadAttributes struct { SuggestedFeeRecipient common.Address `json:"suggestedFeeRecipient" gencodec:"required"` Withdrawals []*types.Withdrawal `json:"withdrawals"` BeaconRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *uint64 `json:"slotNumber"` } // JSON type overrides for PayloadAttributes. type payloadAttributesMarshaling struct { - Timestamp hexutil.Uint64 + Timestamp hexutil.Uint64 + SlotNumber *hexutil.Uint64 } //go:generate go run github.com/fjl/gencodec -type ExecutableData -field-override executableDataMarshaling -out gen_ed.go @@ -90,6 +99,7 @@ type ExecutableData struct { Withdrawals []*types.Withdrawal `json:"withdrawals"` BlobGasUsed *uint64 `json:"blobGasUsed"` ExcessBlobGas *uint64 `json:"excessBlobGas"` + SlotNumber *uint64 `json:"slotNumber"` } // JSON type overrides for executableData. @@ -104,6 +114,7 @@ type executableDataMarshaling struct { Transactions []hexutil.Bytes BlobGasUsed *hexutil.Uint64 ExcessBlobGas *hexutil.Uint64 + SlotNumber *hexutil.Uint64 } // StatelessPayloadStatusV1 is the result of a stateless payload execution. @@ -213,7 +224,7 @@ func encodeTransactions(txs []*types.Transaction) [][]byte { return enc } -func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) { +func DecodeTransactions(enc [][]byte) ([]*types.Transaction, error) { var txs = make([]*types.Transaction, len(enc)) for i, encTx := range enc { var tx types.Transaction @@ -251,7 +262,7 @@ func ExecutableDataToBlock(data ExecutableData, versionedHashes []common.Hash, b // for stateless execution, so it skips checking if the executable data hashes to // the requested hash (stateless has to *compute* the root hash, it's not given). func ExecutableDataToBlockNoHash(data ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, requests [][]byte) (*types.Block, error) { - txs, err := decodeTransactions(data.Transactions) + txs, err := DecodeTransactions(data.Transactions) if err != nil { return nil, err } @@ -313,6 +324,7 @@ func ExecutableDataToBlockNoHash(data ExecutableData, versionedHashes []common.H BlobGasUsed: data.BlobGasUsed, ParentBeaconRoot: beaconRoot, RequestsHash: requestsHash, + SlotNumber: data.SlotNumber, } return types.NewBlockWithHeader(header). WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: data.Withdrawals}), @@ -340,6 +352,7 @@ func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types. Withdrawals: block.Withdrawals(), BlobGasUsed: block.BlobGasUsed(), ExcessBlobGas: block.ExcessBlobGas(), + SlotNumber: block.SlotNumber(), } // Add blobs. diff --git a/build/checksums.txt b/build/checksums.txt index 98ee3a91ef..ba80a3e201 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -5,81 +5,102 @@ # https://github.com/ethereum/execution-spec-tests/releases/download/v5.1.0 a3192784375acec7eaec492799d5c5d0c47a2909a3cc40178898e4ecd20cc416 fixtures_develop.tar.gz -# version:golang 1.25.1 +# version:golang 1.25.7 # https://go.dev/dl/ -d010c109cee94d80efe681eab46bdea491ac906bf46583c32e9f0dbb0bd1a594 go1.25.1.src.tar.gz -1d622468f767a1b9fe1e1e67bd6ce6744d04e0c68712adc689748bbeccb126bb go1.25.1.darwin-amd64.tar.gz -68deebb214f39d542e518ebb0598a406ab1b5a22bba8ec9ade9f55fb4dd94a6c go1.25.1.darwin-arm64.tar.gz -d03cdcbc9bd8baf5cf028de390478e9e2b3e4d0afe5a6582dedc19bfe6a263b2 go1.25.1.linux-386.tar.gz -7716a0d940a0f6ae8e1f3b3f4f36299dc53e31b16840dbd171254312c41ca12e go1.25.1.linux-amd64.tar.gz -65a3e34fb2126f55b34e1edfc709121660e1be2dee6bdf405fc399a63a95a87d go1.25.1.linux-arm64.tar.gz -eb949be683e82a99e9861dafd7057e31ea40b161eae6c4cd18fdc0e8c4ae6225 go1.25.1.linux-armv6l.tar.gz -be13d5479b8c75438f2efcaa8c191fba3af684b3228abc9c99c7aa8502f34424 go1.25.1.windows-386.zip -4a974de310e7ee1d523d2fcedb114ba5fa75408c98eb3652023e55ccf3fa7cab go1.25.1.windows-amd64.zip -45ab4290adbd6ee9e7f18f0d57eaa9008fdbef590882778ed93eac3c8cca06c5 go1.25.1.aix-ppc64.tar.gz -2e3c1549bed3124763774d648f291ac42611232f48320ebbd23517c909c09b81 go1.25.1.dragonfly-amd64.tar.gz -dc0198dd4ec520e13f26798def8750544edf6448d8e9c43fd2a814e4885932af go1.25.1.freebsd-386.tar.gz -c4f1a7e7b258406e6f3b677ecdbd97bbb23ff9c0d44be4eb238a07d360f69ac8 go1.25.1.freebsd-amd64.tar.gz -7772fc5ff71ed39297ec0c1599fc54e399642c9b848eac989601040923b0de9c go1.25.1.freebsd-arm.tar.gz -5bb011d5d5b6218b12189f07aa0be618ab2002662fff1ca40afba7389735c207 go1.25.1.freebsd-arm64.tar.gz -ccac716240cb049bebfafcb7eebc3758512178a4c51fc26da9cc032035d850c8 go1.25.1.freebsd-riscv64.tar.gz -cc53910ffb9fcfdd988a9fa25b5423bae1cfa01b19616be646700e1f5453b466 go1.25.1.illumos-amd64.tar.gz -efe809f923bcedab44bf7be2b3af8d182b512b1bf9c07d302e0c45d26c8f56f3 go1.25.1.linux-loong64.tar.gz -c0de33679f6ed68991dc42dc4a602e74a666e3e166c1748ee1b5d1a7ea2ffbb2 go1.25.1.linux-mips.tar.gz -c270f7b0c0bdfbcd54fef4481227c40d41bb518f9ae38ee930870f04a0a6a589 go1.25.1.linux-mips64.tar.gz -80be871ba9c944f34d1868cdf5047e1cf2e1289fe08cdb90e2453d2f0d6965ae go1.25.1.linux-mips64le.tar.gz -9f09defa9bb22ebf2cde76162f40958564e57ce5c2b3649bc063bebcbc9294c1 go1.25.1.linux-mipsle.tar.gz -2c76b7d278c1d43ad19d478ad3f0f05e7b782b64b90870701b314fa48b5f43c6 go1.25.1.linux-ppc64.tar.gz -8b0c8d3ee5b1b5c28b6bd63dc4438792012e01d03b4bf7a61d985c87edab7d1f go1.25.1.linux-ppc64le.tar.gz -22fe934a9d0c9c57275716c55b92d46ebd887cec3177c9140705efa9f84ba1e2 go1.25.1.linux-riscv64.tar.gz -9cfe517ba423f59f3738ca5c3d907c103253cffbbcc2987142f79c5de8c1bf93 go1.25.1.linux-s390x.tar.gz -6af8a08353e76205d5b743dd7a3f0126684f96f62be0a31b75daf9837e512c46 go1.25.1.netbsd-386.tar.gz -e5d534ff362edb1bd8c8e10892b6a027c4c1482454245d1529167676498684c7 go1.25.1.netbsd-amd64.tar.gz -88bcf39254fdcea6a199c1c27d787831b652427ce60851ae9e41a3d7eb477f45 go1.25.1.netbsd-arm.tar.gz -d7c2eabe1d04ee47bcaea2816fdd90dbd25d90d4dfa756faa9786c788e4f3a4e go1.25.1.netbsd-arm64.tar.gz -14a2845977eb4dde11d929858c437a043467c427db87899935e90cee04a38d72 go1.25.1.openbsd-386.tar.gz -d27ac54b38a13a09c81e67c82ac70d387037341c85c3399291c73e13e83fdd8c go1.25.1.openbsd-amd64.tar.gz -0f4ab5f02500afa4befd51fed1e8b45e4d07ca050f641cc3acc76eaa4027b2c3 go1.25.1.openbsd-arm.tar.gz -d46c3bd156843656f7f3cb0dec27ea51cd926ec3f7b80744bf8156e67c1c812f go1.25.1.openbsd-arm64.tar.gz -c550514c67f22e409be10e40eace761e2e43069f4ef086ae6e60aac736c2b679 go1.25.1.openbsd-ppc64.tar.gz -8a09a8714a2556eb13fc1f10b7ce2553fcea4971e3330fc3be0efd24aab45734 go1.25.1.openbsd-riscv64.tar.gz -b0e1fefaf0c7abd71f139a54eee9767944aff5f0bc9d69c968234804884e552f go1.25.1.plan9-386.tar.gz -e94732c94f149690aa0ab11c26090577211b4a988137cb2c03ec0b54e750402e go1.25.1.plan9-amd64.tar.gz -7eb80e9de1e817d9089a54e8c7c5c8d8ed9e5fb4d4a012fc0f18fc422a484f0c go1.25.1.plan9-arm.tar.gz -1261dfad7c4953c0ab90381bc1242dc54e394db7485c59349428d532b2273343 go1.25.1.solaris-amd64.tar.gz -04bc3c078e9e904c4d58d6ac2532a5bdd402bd36a9ff0b5949b3c5e6006a05ee go1.25.1.windows-arm64.zip +178f2832820274b43e177d32f06a3ebb0129e427dd20a5e4c88df2c1763cf10a go1.25.7.src.tar.gz +81bf2a1f20633f62d55d826d82dde3b0570cf1408a91e15781b266037299285b go1.25.7.aix-ppc64.tar.gz +bf5050a2152f4053837b886e8d9640c829dbacbc3370f913351eb0904cb706f5 go1.25.7.darwin-amd64.tar.gz +ff18369ffad05c57d5bed888b660b31385f3c913670a83ef557cdfd98ea9ae1b go1.25.7.darwin-arm64.tar.gz +c5dccd7f192dd7b305dc209fb316ac1917776d74bd8e4d532ef2772f305bf42a go1.25.7.dragonfly-amd64.tar.gz +a2de97c8ac74bf64b0ae73fe9d379e61af530e061bc7f8f825044172ffe61a8b go1.25.7.freebsd-386.tar.gz +055f9e138787dcafa81eb0314c8ff70c6dd0f6dba1e8a6957fef5d5efd1ab8fd go1.25.7.freebsd-amd64.tar.gz +60e7f7a7c990f0b9539ac8ed668155746997d404643a4eecd47b3dee1b7e710b go1.25.7.freebsd-arm.tar.gz +631e03d5fd4c526e2f499154d8c6bf4cb081afb2fff171c428722afc9539d53a go1.25.7.freebsd-arm64.tar.gz +8a264fd685823808140672812e3ad9c43f6ad59444c0dc14cdd3a1351839ddd5 go1.25.7.freebsd-riscv64.tar.gz +57c672447d906a1bcab98f2b11492d54521a791aacbb4994a25169e59cbe289a go1.25.7.illumos-amd64.tar.gz +2866517e9ca81e6a2e85a930e9b11bc8a05cfeb2fc6dc6cb2765e7fb3c14b715 go1.25.7.linux-386.tar.gz +12e6d6a191091ae27dc31f6efc630e3a3b8ba409baf3573d955b196fdf086005 go1.25.7.linux-amd64.tar.gz +ba611a53534135a81067240eff9508cd7e256c560edd5d8c2fef54f083c07129 go1.25.7.linux-arm64.tar.gz +1ba07e0eb86b839e72467f4b5c7a5597d07f30bcf5563c951410454f7cda5266 go1.25.7.linux-armv6l.tar.gz +775753fc5952a334c415f08768df2f0b73a3228a16e8f5f63d545daacb4e3357 go1.25.7.linux-loong64.tar.gz +1a023bb367c5fbb4c637a2f6dc23ff17c6591ad929ce16ea88c74d857153b307 go1.25.7.linux-mips.tar.gz +a8e97223d8aa6fdfd45f132a4784d2f536bbac5f3d63a24b63d33b6bfe1549af go1.25.7.linux-mips64.tar.gz +eb9edb6223330d5e20275667c65dea076b064c08e595fe4eba5d7d6055cfaccf go1.25.7.linux-mips64le.tar.gz +9c1e693552a5f9bb9e0012d1c5e01456ecefbc59bef53a77305222ce10aba368 go1.25.7.linux-mipsle.tar.gz +28a788798e7329acbbc0ac2caa5e4368b1e5ede646cc24429c991214cfb45c63 go1.25.7.linux-ppc64.tar.gz +42124c0edc92464e2b37b2d7fcd3658f0c47ebd6a098732415a522be8cb88e3f go1.25.7.linux-ppc64le.tar.gz +88d59c6893c8425875d6eef8e3434bc2fa2552e5ad4c058c6cd8cd710a0301c8 go1.25.7.linux-riscv64.tar.gz +c6b77facf666dc68195ecab05dbf0ebb4e755b2a8b7734c759880557f1c29b0c go1.25.7.linux-s390x.tar.gz +f14c184d9ade0ee04c7735d4071257b90896ecbde1b32adae84135f055e6399b go1.25.7.netbsd-386.tar.gz +7e7389e404dca1088c31f0fc07f1dd60891d7182bcd621469c14f7e79eceb3ff go1.25.7.netbsd-amd64.tar.gz +70388bb3ef2f03dbf1357e9056bd09034a67e018262557354f8cf549766b3f9d go1.25.7.netbsd-arm.tar.gz +8c1cda9d25bfc9b18d24d5f95fc23949dd3ff99fa408a6cfa40e2cf12b07e362 go1.25.7.netbsd-arm64.tar.gz +42f0d1bfbe39b8401cccb84dd66b30795b97bfc9620dfdc17c5cd4fcf6495cb0 go1.25.7.openbsd-386.tar.gz +e514879c0a28bc32123cd52c4c093de912477fe83f36a6d07517d066ef55391a go1.25.7.openbsd-amd64.tar.gz +8cd22530695a0218232bf7efea8f162df1697a3106942ac4129b8c3de39ce4ef go1.25.7.openbsd-arm.tar.gz +938720f6ebc0d1c53d7840321d3a31f29fd02496e84a6538f442a9311dc1cc9a go1.25.7.openbsd-arm64.tar.gz +a4c378b73b98f89a3596c2ef51aabbb28783d9ca29f7e317d8ca07939660ce6f go1.25.7.openbsd-ppc64.tar.gz +937b58734fbeaa8c7941a0e4285e7e84b7885396e8d11c23f9ab1a8ff10ff20e go1.25.7.openbsd-riscv64.tar.gz +61a093c8c5244916f25740316386bb9f141545dcf01b06a79d1c78ece488403e go1.25.7.plan9-386.tar.gz +7fc8f6689c9de8ccb7689d2278035fa83c2d601409101840df6ddfe09ba58699 go1.25.7.plan9-amd64.tar.gz +9661dff8eaeeb62f1c3aadbc5ff189a2e6744e1ec885e32dbcb438f58a34def5 go1.25.7.plan9-arm.tar.gz +28ecba0e1d7950c8b29a4a04962dd49c3bf5221f55a44f17d98f369f82859cf4 go1.25.7.solaris-amd64.tar.gz +baa6b488291801642fa620026169e38bec2da2ac187cd3ae2145721cf826bbc3 go1.25.7.windows-386.zip +c75e5f4ff62d085cc0017be3ad19d5536f46825fa05db06ec468941f847e3228 go1.25.7.windows-amd64.zip +807033f85931bc4a589ca8497535dcbeb1f30d506e47fa200f5f04c4a71c3d9f go1.25.7.windows-arm64.zip -# version:golangci 2.4.0 +# version:golangci 2.10.1 # https://github.com/golangci/golangci-lint/releases/ -# https://github.com/golangci/golangci-lint/releases/download/v2.4.0/ -7904ce63f79db44934939cf7a063086ea0ea98e9b19eba0a9d52ccdd0d21951c golangci-lint-2.4.0-darwin-amd64.tar.gz -cd4dd53fa09b6646baff5fd22b8c64d91db02c21c7496df27992d75d34feec59 golangci-lint-2.4.0-darwin-arm64.tar.gz -d58f426ebe14cc257e81562b4bf37a488ffb4ffbbb3ec73041eb3b38bb25c0e1 golangci-lint-2.4.0-freebsd-386.tar.gz -6ec4a6177fc6c0dd541fbcb3a7612845266d020d35cc6fa92959220cdf64ca39 golangci-lint-2.4.0-freebsd-amd64.tar.gz -4d473e3e71c01feaa915a0604fb35758b41284fb976cdeac3f842118d9ee7e17 golangci-lint-2.4.0-freebsd-armv6.tar.gz -58727746c6530801a3f9a702a5945556a5eb7e88809222536dd9f9d54cafaeff golangci-lint-2.4.0-freebsd-armv7.tar.gz -fbf28c662760e24c32f82f8d16dffdb4a82de7726a52ba1fad94f890c22997ea golangci-lint-2.4.0-illumos-amd64.tar.gz -a15a000a8981ef665e971e0f67e2acda9066a9e37a59344393b7351d8fb49c81 golangci-lint-2.4.0-linux-386.tar.gz -fae792524c04424c0ac369f5b8076f04b45cf29fc945a370e55d369a8dc11840 golangci-lint-2.4.0-linux-amd64.tar.gz -70ac11f55b80ec78fd3a879249cc9255121b8dfd7f7ed4fc46ed137f4abf17e7 golangci-lint-2.4.0-linux-arm64.tar.gz -4acdc40e5cebe99e4e7ced358a05b2e71789f409b41cb4f39bbb86ccfa14b1dc golangci-lint-2.4.0-linux-armv6.tar.gz -2a68749568fa22b4a97cb88dbea655595563c795076536aa6c087f7968784bf3 golangci-lint-2.4.0-linux-armv7.tar.gz -9e3369afb023711036dcb0b4f45c9fe2792af962fa1df050c9f6ac101a6c5d73 golangci-lint-2.4.0-linux-loong64.tar.gz -bb9143d6329be2c4dbfffef9564078e7da7d88e7dde6c829b6263d98e072229e golangci-lint-2.4.0-linux-mips64.tar.gz -5ad1765b40d56cd04d4afd805b3ba6f4bfd9b36181da93c31e9b17e483d8608d golangci-lint-2.4.0-linux-mips64le.tar.gz -918936fb9c0d5ba96bef03cf4348b03938634cfcced49be1e9bb29cb5094fa73 golangci-lint-2.4.0-linux-ppc64le.tar.gz -f7474c638e1fb67ebbdc654b55ca0125377ea0bc88e8fee8d964a4f24eacf828 golangci-lint-2.4.0-linux-riscv64.tar.gz -b617a9543997c8bfceaffa88a75d4e595030c6add69fba800c1e4d8f5fe253dd golangci-lint-2.4.0-linux-s390x.tar.gz -7db027b03a9ba328f795215b04f594036837bc7dd0dd7cd16776b02a6167981c golangci-lint-2.4.0-netbsd-386.tar.gz -52d8f9393f4313df0a62b752c37775e3af0b818e43e8dd28954351542d7c60bc golangci-lint-2.4.0-netbsd-amd64.tar.gz -5c0086027fb5a4af3829e530c8115db4b35d11afe1914322eef528eb8cd38c69 golangci-lint-2.4.0-netbsd-arm64.tar.gz -6b779d6ed1aed87cefe195cc11759902b97a76551b593312c6833f2635a3488f golangci-lint-2.4.0-netbsd-armv6.tar.gz -f00d1f4b7ec3468a0f9fffd0d9ea036248b029b7621cbc9a59c449ef94356d09 golangci-lint-2.4.0-netbsd-armv7.tar.gz -3ce671b0b42b58e35066493aab75a7e2826c9e079988f1ba5d814a4029faaf87 golangci-lint-2.4.0-windows-386.zip -003112f7a56746feaabf20b744054bf9acdf900c9e77176383623c4b1d76aaa9 golangci-lint-2.4.0-windows-amd64.zip -dc0c2092af5d47fc2cd31a1dfe7b4c7e765fab22de98bd21ef2ffcc53ad9f54f golangci-lint-2.4.0-windows-arm64.zip -0263d23e20a260cb1592d35e12a388f99efe2c51b3611fdc66fbd9db1fce664d golangci-lint-2.4.0-windows-armv6.zip -9403c03bf648e6313036e0273149d44bad1b9ad53889b6d00e4ccb842ba3c058 golangci-lint-2.4.0-windows-armv7.zip +# https://github.com/golangci/golangci-lint/releases/download/v2.10.1 +66fb0da81b8033b477f97eea420d4b46b230ca172b8bb87c6610109f3772b6b6 golangci-lint-2.10.1-darwin-amd64.tar.gz +03bfadf67e52b441b7ec21305e501c717df93c959836d66c7f97312654acb297 golangci-lint-2.10.1-darwin-arm64.tar.gz +c9a44658ccc8f7b8dbbd4ae6020ba91c1a5d3987f4d91ced0f7d2bea013e57ca golangci-lint-2.10.1-freebsd-386.tar.gz +a513c5cb4e0f5bd5767001af9d5e97e7868cfc2d9c46739a4df93e713cfb24af golangci-lint-2.10.1-freebsd-amd64.tar.gz +2ef38eefc4b5cee2febacb75a30579526e5656c16338a921d80e59a8e87d4425 golangci-lint-2.10.1-freebsd-arm64.tar.gz +8fea6766318b4829e766bbe325f10191d75297dcc44ae35bf374816037878e38 golangci-lint-2.10.1-freebsd-armv6.tar.gz +30b629870574d6254f3e8804e5a74b34f98e1263c9d55465830d739c88b862ed golangci-lint-2.10.1-freebsd-armv7.tar.gz +c0db839f866ce80b1b6c96167aa101cfe50d9c936f42d942a3c1cbdc1801af68 golangci-lint-2.10.1-illumos-amd64.tar.gz +280eb56636e9175f671cd7b755d7d67f628ae2ed00a164d1e443c43c112034e5 golangci-lint-2.10.1-linux-386.deb +065a7d99da61dc7dfbfef2e2d7053dd3fa6672598f2747117aa4bb5f45e7df7f golangci-lint-2.10.1-linux-386.rpm +a55918c03bb413b2662287653ab2ae2fef4e37428b247dad6348724adde9d770 golangci-lint-2.10.1-linux-386.tar.gz +8aa9b3aa14f39745eeb7fc7ff50bcac683e785397d1e4bc9afd2184b12c4ce86 golangci-lint-2.10.1-linux-amd64.deb +62a111688e9e305032334a2cbc84f4d971b64bb3bffc99d3f80081d57fb25e32 golangci-lint-2.10.1-linux-amd64.rpm +dfa775874cf0561b404a02a8f4481fc69b28091da95aa697259820d429b09c99 golangci-lint-2.10.1-linux-amd64.tar.gz +b3f36937e8ea1660739dc0f5c892ea59c9c21ed4e75a91a25957c561f7f79a55 golangci-lint-2.10.1-linux-arm64.deb +36d50314d53683b1f1a2a6cedfb5a9468451b481c64ab9e97a8e843ea088074d golangci-lint-2.10.1-linux-arm64.rpm +6652b42ae02915eb2f9cb2a2e0cac99514c8eded8388d88ae3e06e1a52c00de8 golangci-lint-2.10.1-linux-arm64.tar.gz +a32d8d318e803496812dd3461f250e52ccc7f53c47b95ce404a9cf55778ceb6a golangci-lint-2.10.1-linux-armv6.deb +41d065f4c8ea165a1531abea644988ee2e973e4f0b49f9725ed3b979dac45112 golangci-lint-2.10.1-linux-armv6.rpm +59159a4df03aabbde69d15c7b7b3df143363cbb41f4bd4b200caffb8e34fb734 golangci-lint-2.10.1-linux-armv6.tar.gz +b2e8ec0e050a1e2251dfe1561434999d202f5a3f9fa47ce94378b0fd1662ea5a golangci-lint-2.10.1-linux-armv7.deb +28c9331429a497da27e9c77846063bd0e8275e878ffedb4eb9e9f21d24771cc0 golangci-lint-2.10.1-linux-armv7.rpm +818f33e95b273e3769284b25563b51ef6a294e9e25acf140fda5830c075a1a59 golangci-lint-2.10.1-linux-armv7.tar.gz +6b6b85ed4b7c27f51097dd681523000409dde835e86e6e314e87be4bb013e2ab golangci-lint-2.10.1-linux-loong64.deb +94050a0cf06169e2ae44afb307dcaafa7d7c3b38c0c23b5652cf9cb60f0c337f golangci-lint-2.10.1-linux-loong64.rpm +25820300fccb8c961c1cdcb1f77928040c079e04c43a3a5ceb34b1cb4a1c5c8d golangci-lint-2.10.1-linux-loong64.tar.gz +98bf39d10139fdcaa37f94950e9bbb8888660ae468847ae0bf1cb5bf67c1f68b golangci-lint-2.10.1-linux-mips64.deb +df3ce5f03808dcceaa8b683d1d06e95c885f09b59dc8e15deb840fbe2b3e3299 golangci-lint-2.10.1-linux-mips64.rpm +972508dda523067e6e6a1c8e6609d63bc7c4153819c11b947d439235cf17bac2 golangci-lint-2.10.1-linux-mips64.tar.gz +1d37f2919e183b5bf8b1777ed8c4b163d3b491d0158355a7999d647655cbbeb6 golangci-lint-2.10.1-linux-mips64le.deb +e341d031002cd09a416329ed40f674231051a38544b8f94deb2d1708ce1f4a6f golangci-lint-2.10.1-linux-mips64le.rpm +393560122b9cb5538df0c357d30eb27b6ee563533fbb9b138c8db4fd264002af golangci-lint-2.10.1-linux-mips64le.tar.gz +21ca46b6a96442e8957677a3ca059c6b93674a68a01b1c71f4e5df0ea2e96d19 golangci-lint-2.10.1-linux-ppc64le.deb +57fe0cbca0a9bbdf1547c5e8aa7d278e6896b438d72a541bae6bc62c38b43d1e golangci-lint-2.10.1-linux-ppc64le.rpm +e2883db9fa51584e5e203c64456f29993550a7faadc84e3faccdb48f0669992e golangci-lint-2.10.1-linux-ppc64le.tar.gz +aa6da0e98ab0ba3bb7582e112174c349907d5edfeff90a551dca3c6eecf92fc0 golangci-lint-2.10.1-linux-riscv64.deb +3c68d76cd884a7aad206223a980b9c20bb9ea74b560fa27ed02baf2389189234 golangci-lint-2.10.1-linux-riscv64.rpm +3bca11bfac4197205639cbd4676a5415054e629ac6c12ea10fcbe33ef852d9c3 golangci-lint-2.10.1-linux-riscv64.tar.gz +0c6aed2ce49db2586adbac72c80d871f06feb1caf4c0763a5ca98fec809a8f0b golangci-lint-2.10.1-linux-s390x.deb +16c285adfe1061d69dd8e503be69f87c7202857c6f4add74ac02e3571158fbec golangci-lint-2.10.1-linux-s390x.rpm +21011ad368eb04f024201b832095c6b5f96d0888de194cca5bfe4d9307d6364b golangci-lint-2.10.1-linux-s390x.tar.gz +7b5191e77a70485918712e31ed55159956323e4911bab1b67569c9d86e1b75eb golangci-lint-2.10.1-netbsd-386.tar.gz +07801fd38d293ebad10826f8285525a39ea91ce5ddad77d05bfa90bda9c884a9 golangci-lint-2.10.1-netbsd-amd64.tar.gz +7e7219d71c1bf33b98c328c93dc0560706dd896a1c43c44696e5222fc9d7446e golangci-lint-2.10.1-netbsd-arm64.tar.gz +92fbc90b9eec0e572269b0f5492a2895c426b086a68372fde49b7e4d4020863e golangci-lint-2.10.1-netbsd-armv6.tar.gz +f67b3ae1f47caeefa507a4ebb0c8336958a19011fe48766443212030f75d004b golangci-lint-2.10.1-netbsd-armv7.tar.gz +a40bc091c10cea84eaee1a90b84b65f5e8652113b0a600bb099e4e4d9d7caddb golangci-lint-2.10.1-windows-386.zip +c60c87695e79db8e320f0e5be885059859de52bb5ee5f11be5577828570bc2a3 golangci-lint-2.10.1-windows-amd64.zip +636ab790c8dcea8034aa34aba6031ca3893d68f7eda000460ab534341fadbab1 golangci-lint-2.10.1-windows-arm64.zip # This is the builder on PPA that will build Go itself (inception-y), don't modify! # diff --git a/build/ci.go b/build/ci.go index abb7c4997f..4d0a1d7e35 100644 --- a/build/ci.go +++ b/build/ci.go @@ -168,8 +168,6 @@ var ( "focal", // 20.04, EOL: 04/2030 "jammy", // 22.04, EOL: 04/2032 "noble", // 24.04, EOL: 04/2034 - "oracular", // 24.10, EOL: 07/2025 - "plucky", // 25.04, EOL: 01/2026 } // This is where the tests should be unpacked. diff --git a/cmd/devp2p/internal/ethtest/conn.go b/cmd/devp2p/internal/ethtest/conn.go index 5182d71ce1..98baba81a4 100644 --- a/cmd/devp2p/internal/ethtest/conn.go +++ b/cmd/devp2p/internal/ethtest/conn.go @@ -155,7 +155,7 @@ func (c *Conn) ReadEth() (any, error) { var msg any switch int(code) { case eth.StatusMsg: - msg = new(eth.StatusPacket69) + msg = new(eth.StatusPacket) case eth.GetBlockHeadersMsg: msg = new(eth.GetBlockHeadersPacket) case eth.BlockHeadersMsg: @@ -164,10 +164,6 @@ func (c *Conn) ReadEth() (any, error) { msg = new(eth.GetBlockBodiesPacket) case eth.BlockBodiesMsg: msg = new(eth.BlockBodiesPacket) - case eth.NewBlockMsg: - msg = new(eth.NewBlockPacket) - case eth.NewBlockHashesMsg: - msg = new(eth.NewBlockHashesPacket) case eth.TransactionsMsg: msg = new(eth.TransactionsPacket) case eth.NewPooledTransactionHashesMsg: @@ -229,7 +225,7 @@ func (c *Conn) ReadSnap() (any, error) { } // dialAndPeer creates a peer connection and runs the handshake. -func (s *Suite) dialAndPeer(status *eth.StatusPacket69) (*Conn, error) { +func (s *Suite) dialAndPeer(status *eth.StatusPacket) (*Conn, error) { c, err := s.dial() if err != nil { return nil, err @@ -242,7 +238,7 @@ func (s *Suite) dialAndPeer(status *eth.StatusPacket69) (*Conn, error) { // peer performs both the protocol handshake and the status message // exchange with the node in order to peer with it. -func (c *Conn) peer(chain *Chain, status *eth.StatusPacket69) error { +func (c *Conn) peer(chain *Chain, status *eth.StatusPacket) error { if err := c.handshake(); err != nil { return fmt.Errorf("handshake failed: %v", err) } @@ -315,7 +311,7 @@ func (c *Conn) negotiateEthProtocol(caps []p2p.Cap) { } // statusExchange performs a `Status` message exchange with the given node. -func (c *Conn) statusExchange(chain *Chain, status *eth.StatusPacket69) error { +func (c *Conn) statusExchange(chain *Chain, status *eth.StatusPacket) error { loop: for { code, data, err := c.Read() @@ -324,7 +320,7 @@ loop: } switch code { case eth.StatusMsg + protoOffset(ethProto): - msg := new(eth.StatusPacket69) + msg := new(eth.StatusPacket) if err := rlp.DecodeBytes(data, &msg); err != nil { return fmt.Errorf("error decoding status packet: %w", err) } @@ -363,7 +359,7 @@ loop: } if status == nil { // default status message - status = ð.StatusPacket69{ + status = ð.StatusPacket{ ProtocolVersion: uint32(c.negotiatedProtoVersion), NetworkID: chain.config.ChainID.Uint64(), Genesis: chain.blocks[0].Hash(), diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 8bb488e358..7560c13137 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -447,7 +447,7 @@ func (s *Suite) TestGetReceipts(t *utesting.T) { t.Fatalf("could not write to connection: %v", err) } // Wait for response. - resp := new(eth.ReceiptsPacket[*eth.ReceiptList69]) + resp := new(eth.ReceiptsPacket) if err := conn.ReadMsg(ethProto, eth.ReceiptsMsg, &resp); err != nil { t.Fatalf("error reading block bodies msg: %v", err) } diff --git a/cmd/devp2p/internal/v5test/discv5tests.go b/cmd/devp2p/internal/v5test/discv5tests.go index 2139cd8ca6..efe9144069 100644 --- a/cmd/devp2p/internal/v5test/discv5tests.go +++ b/cmd/devp2p/internal/v5test/discv5tests.go @@ -52,7 +52,7 @@ func (s *Suite) AllTests() []utesting.Test { {Name: "Ping", Fn: s.TestPing}, {Name: "PingLargeRequestID", Fn: s.TestPingLargeRequestID}, {Name: "PingMultiIP", Fn: s.TestPingMultiIP}, - {Name: "PingHandshakeInterrupted", Fn: s.TestPingHandshakeInterrupted}, + {Name: "HandshakeResend", Fn: s.TestHandshakeResend}, {Name: "TalkRequest", Fn: s.TestTalkRequest}, {Name: "FindnodeZeroDistance", Fn: s.TestFindnodeZeroDistance}, {Name: "FindnodeResults", Fn: s.TestFindnodeResults}, @@ -158,22 +158,20 @@ the attempt from a different IP.`) } } -// TestPingHandshakeInterrupted starts a handshake, but doesn't finish it and sends a second ordinary message -// packet instead of a handshake message packet. The remote node should respond with -// another WHOAREYOU challenge for the second packet. -func (s *Suite) TestPingHandshakeInterrupted(t *utesting.T) { - t.Log(`TestPingHandshakeInterrupted starts a handshake, but doesn't finish it and sends a second ordinary message -packet instead of a handshake message packet. The remote node should respond with -another WHOAREYOU challenge for the second packet.`) - +// TestHandshakeResend starts a handshake, but doesn't finish it and sends a second ordinary message +// packet instead of a handshake message packet. The remote node should repeat the previous WHOAREYOU +// challenge for the first PING. +func (s *Suite) TestHandshakeResend(t *utesting.T) { conn, l1 := s.listen1(t) defer conn.close() // First PING triggers challenge. ping := &v5wire.Ping{ReqID: conn.nextReqID()} conn.write(l1, ping, nil) + var challenge1 *v5wire.Whoareyou switch resp := conn.read(l1).(type) { case *v5wire.Whoareyou: + challenge1 = resp t.Logf("got WHOAREYOU for PING") default: t.Fatal("expected WHOAREYOU, got", resp) @@ -181,9 +179,16 @@ another WHOAREYOU challenge for the second packet.`) // Send second PING. ping2 := &v5wire.Ping{ReqID: conn.nextReqID()} - switch resp := conn.reqresp(l1, ping2).(type) { - case *v5wire.Pong: - checkPong(t, resp, ping2, l1) + conn.write(l1, ping2, nil) + switch resp := conn.read(l1).(type) { + case *v5wire.Whoareyou: + if resp.Nonce != challenge1.Nonce { + t.Fatalf("wrong nonce %x in WHOAREYOU (want %x)", resp.Nonce[:], challenge1.Nonce[:]) + } + if !bytes.Equal(resp.ChallengeData, challenge1.ChallengeData) { + t.Fatalf("wrong ChallengeData in resent WHOAREYOU (want %x)", resp.ChallengeData, challenge1.ChallengeData) + } + resp.Node = conn.remote default: t.Fatal("expected WHOAREYOU, got", resp) } diff --git a/cmd/evm/internal/t8ntool/block.go b/cmd/evm/internal/t8ntool/block.go index 37a6db9ffc..6148e9e248 100644 --- a/cmd/evm/internal/t8ntool/block.go +++ b/cmd/evm/internal/t8ntool/block.go @@ -56,6 +56,7 @@ type header struct { BlobGasUsed *uint64 `json:"blobGasUsed" rlp:"optional"` ExcessBlobGas *uint64 `json:"excessBlobGas" rlp:"optional"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` + SlotNumber *uint64 `json:"slotNumber" rlp:"optional"` } type headerMarshaling struct { @@ -68,6 +69,7 @@ type headerMarshaling struct { BaseFee *math.HexOrDecimal256 BlobGasUsed *math.HexOrDecimal64 ExcessBlobGas *math.HexOrDecimal64 + SlotNumber *math.HexOrDecimal64 } type bbInput struct { @@ -136,6 +138,7 @@ func (i *bbInput) ToBlock() *types.Block { BlobGasUsed: i.Header.BlobGasUsed, ExcessBlobGas: i.Header.ExcessBlobGas, ParentBeaconRoot: i.Header.ParentBeaconBlockRoot, + SlotNumber: i.Header.SlotNumber, } // Fill optional values. diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 693ec9f4a9..b3fb79bc4a 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -102,6 +102,7 @@ type stEnv struct { ParentExcessBlobGas *uint64 `json:"parentExcessBlobGas,omitempty"` ParentBlobGasUsed *uint64 `json:"parentBlobGasUsed,omitempty"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *uint64 `json:"slotNumber"` } type stEnvMarshaling struct { @@ -120,6 +121,7 @@ type stEnvMarshaling struct { ExcessBlobGas *math.HexOrDecimal64 ParentExcessBlobGas *math.HexOrDecimal64 ParentBlobGasUsed *math.HexOrDecimal64 + SlotNumber *math.HexOrDecimal64 } type rejectedTx struct { @@ -147,15 +149,13 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, isEIP4762 = chainConfig.IsVerkle(big.NewInt(int64(pre.Env.Number)), pre.Env.Timestamp) statedb = MakePreState(rawdb.NewMemoryDatabase(), pre.Pre, isEIP4762) signer = types.MakeSigner(chainConfig, new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp) - gaspool = new(core.GasPool) + gaspool = core.NewGasPool(pre.Env.GasLimit) blockHash = common.Hash{0x13, 0x37} rejectedTxs []*rejectedTx includedTxs types.Transactions - gasUsed = uint64(0) blobGasUsed = uint64(0) receipts = make(types.Receipts, 0) ) - gaspool.AddGas(pre.Env.GasLimit) vmContext := vm.BlockContext{ CanTransfer: core.CanTransfer, Transfer: core.Transfer, @@ -195,6 +195,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, ExcessBlobGas: pre.Env.ParentExcessBlobGas, BlobGasUsed: pre.Env.ParentBlobGasUsed, BaseFee: pre.Env.ParentBaseFee, + SlotNumber: pre.Env.SlotNumber, } header := &types.Header{ Time: pre.Env.Timestamp, @@ -255,16 +256,19 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, statedb.SetTxContext(tx.Hash(), len(receipts)) var ( snapshot = statedb.Snapshot() - prevGas = gaspool.Gas() + gp = gaspool.Snapshot() ) - receipt, err := core.ApplyTransactionWithEVM(msg, gaspool, statedb, vmContext.BlockNumber, blockHash, pre.Env.Timestamp, tx, &gasUsed, evm) + receipt, err := core.ApplyTransactionWithEVM(msg, gaspool, statedb, vmContext.BlockNumber, blockHash, pre.Env.Timestamp, tx, evm) if err != nil { statedb.RevertToSnapshot(snapshot) log.Info("rejected tx", "index", i, "hash", tx.Hash(), "from", msg.From, "error", err) rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()}) - gaspool.SetGas(prevGas) + gaspool.Set(gp) continue } + if receipt.Logs == nil { + receipt.Logs = []*types.Log{} + } includedTxs = append(includedTxs, tx) if hashError != nil { return nil, nil, nil, NewError(ErrorMissingBlockhash, hashError) @@ -346,7 +350,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, Receipts: receipts, Rejected: rejectedTxs, Difficulty: (*math.HexOrDecimal256)(vmContext.Difficulty), - GasUsed: (math.HexOrDecimal64)(gasUsed), + GasUsed: (math.HexOrDecimal64)(gaspool.Used()), BaseFee: (*math.HexOrDecimal256)(vmContext.BaseFee), } if pre.Env.Withdrawals != nil { diff --git a/cmd/evm/internal/t8ntool/file_tracer.go b/cmd/evm/internal/t8ntool/file_tracer.go index 38fc35bd32..d032252cba 100644 --- a/cmd/evm/internal/t8ntool/file_tracer.go +++ b/cmd/evm/internal/t8ntool/file_tracer.go @@ -56,27 +56,35 @@ func (l *fileWritingTracer) Write(p []byte) (n int, err error) { return n, nil } -// newFileWriter creates a set of hooks which wraps inner hooks (typically a logger), +// newFileWriter creates a tracer which wraps inner hooks (typically a logger), // and writes the output to a file, one file per transaction. -func newFileWriter(baseDir string, innerFn func(out io.Writer) *tracing.Hooks) *tracing.Hooks { +func newFileWriter(baseDir string, innerFn func(out io.Writer) *tracing.Hooks) *tracers.Tracer { t := &fileWritingTracer{ baseDir: baseDir, suffix: "jsonl", } t.inner = innerFn(t) // instantiate the inner tracer - return t.hooks() + return &tracers.Tracer{ + Hooks: t.hooks(), + GetResult: func() (json.RawMessage, error) { return json.RawMessage("{}"), nil }, + Stop: func(err error) {}, + } } -// newResultWriter creates a set of hooks wraps and invokes an underlying tracer, +// newResultWriter creates a tracer that wraps and invokes an underlying tracer, // and writes the result (getResult-output) to file, one per transaction. -func newResultWriter(baseDir string, tracer *tracers.Tracer) *tracing.Hooks { +func newResultWriter(baseDir string, tracer *tracers.Tracer) *tracers.Tracer { t := &fileWritingTracer{ baseDir: baseDir, getResult: tracer.GetResult, inner: tracer.Hooks, suffix: "json", } - return t.hooks() + return &tracers.Tracer{ + Hooks: t.hooks(), + GetResult: func() (json.RawMessage, error) { return json.RawMessage("{}"), nil }, + Stop: func(err error) {}, + } } // OnTxStart creates a new output-file specific for this transaction, and invokes diff --git a/cmd/evm/internal/t8ntool/flags.go b/cmd/evm/internal/t8ntool/flags.go index a6ec33eacf..078d304927 100644 --- a/cmd/evm/internal/t8ntool/flags.go +++ b/cmd/evm/internal/t8ntool/flags.go @@ -162,6 +162,11 @@ var ( strings.Join(vm.ActivateableEips(), ", ")), Value: "GrayGlacier", } + OpcodeCountFlag = &cli.StringFlag{ + Name: "opcode.count", + Usage: "If set, opcode execution counts will be written to this file (relative to output.basedir).", + Value: "", + } VerbosityFlag = &cli.IntFlag{ Name: "verbosity", Usage: "sets the verbosity level", diff --git a/cmd/evm/internal/t8ntool/gen_header.go b/cmd/evm/internal/t8ntool/gen_header.go index a8c8668978..f430feb6d2 100644 --- a/cmd/evm/internal/t8ntool/gen_header.go +++ b/cmd/evm/internal/t8ntool/gen_header.go @@ -38,6 +38,7 @@ func (h header) MarshalJSON() ([]byte, error) { BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed" rlp:"optional"` ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas" rlp:"optional"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` + SlotNumber *math.HexOrDecimal64 `json:"slotNumber" rlp:"optional"` } var enc header enc.ParentHash = h.ParentHash @@ -60,6 +61,7 @@ func (h header) MarshalJSON() ([]byte, error) { enc.BlobGasUsed = (*math.HexOrDecimal64)(h.BlobGasUsed) enc.ExcessBlobGas = (*math.HexOrDecimal64)(h.ExcessBlobGas) enc.ParentBeaconBlockRoot = h.ParentBeaconBlockRoot + enc.SlotNumber = (*math.HexOrDecimal64)(h.SlotNumber) return json.Marshal(&enc) } @@ -86,6 +88,7 @@ func (h *header) UnmarshalJSON(input []byte) error { BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed" rlp:"optional"` ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas" rlp:"optional"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` + SlotNumber *math.HexOrDecimal64 `json:"slotNumber" rlp:"optional"` } var dec header if err := json.Unmarshal(input, &dec); err != nil { @@ -155,5 +158,8 @@ func (h *header) UnmarshalJSON(input []byte) error { if dec.ParentBeaconBlockRoot != nil { h.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot } + if dec.SlotNumber != nil { + h.SlotNumber = (*uint64)(dec.SlotNumber) + } return nil } diff --git a/cmd/evm/internal/t8ntool/gen_stenv.go b/cmd/evm/internal/t8ntool/gen_stenv.go index d47db4a876..1b8e14abc6 100644 --- a/cmd/evm/internal/t8ntool/gen_stenv.go +++ b/cmd/evm/internal/t8ntool/gen_stenv.go @@ -37,6 +37,7 @@ func (s stEnv) MarshalJSON() ([]byte, error) { ParentExcessBlobGas *math.HexOrDecimal64 `json:"parentExcessBlobGas,omitempty"` ParentBlobGasUsed *math.HexOrDecimal64 `json:"parentBlobGasUsed,omitempty"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *math.HexOrDecimal64 `json:"slotNumber"` } var enc stEnv enc.Coinbase = common.UnprefixedAddress(s.Coinbase) @@ -59,6 +60,7 @@ func (s stEnv) MarshalJSON() ([]byte, error) { enc.ParentExcessBlobGas = (*math.HexOrDecimal64)(s.ParentExcessBlobGas) enc.ParentBlobGasUsed = (*math.HexOrDecimal64)(s.ParentBlobGasUsed) enc.ParentBeaconBlockRoot = s.ParentBeaconBlockRoot + enc.SlotNumber = (*math.HexOrDecimal64)(s.SlotNumber) return json.Marshal(&enc) } @@ -85,6 +87,7 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { ParentExcessBlobGas *math.HexOrDecimal64 `json:"parentExcessBlobGas,omitempty"` ParentBlobGasUsed *math.HexOrDecimal64 `json:"parentBlobGasUsed,omitempty"` ParentBeaconBlockRoot *common.Hash `json:"parentBeaconBlockRoot"` + SlotNumber *math.HexOrDecimal64 `json:"slotNumber"` } var dec stEnv if err := json.Unmarshal(input, &dec); err != nil { @@ -154,5 +157,8 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { if dec.ParentBeaconBlockRoot != nil { s.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot } + if dec.SlotNumber != nil { + s.SlotNumber = (*uint64)(dec.SlotNumber) + } return nil } diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go index 4ba7b5f130..3a457eeaec 100644 --- a/cmd/evm/internal/t8ntool/transaction.go +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -27,7 +27,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" @@ -177,9 +179,12 @@ func Transaction(ctx *cli.Context) error { r.Error = errors.New("gas * maxFeePerGas exceeds 256 bits") } // Check whether the init code size has been exceeded. - if chainConfig.IsShanghai(new(big.Int), 0) && tx.To() == nil && len(tx.Data()) > params.MaxInitCodeSize { - r.Error = errors.New("max initcode size exceeded") + if tx.To() == nil { + if err := vm.CheckMaxInitCodeSize(&rules, uint64(len(tx.Data()))); err != nil { + r.Error = err + } } + if chainConfig.IsOsaka(new(big.Int), 0) && tx.Gas() > params.MaxTxGas { r.Error = errors.New("gas limit exceeds maximum") } diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index af60333cbd..d7cdc98e74 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/eth/tracers/logger" + "github.com/ethereum/go-ethereum/eth/tracers/native" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/tests" @@ -167,14 +168,15 @@ func Transition(ctx *cli.Context) error { } // Configure tracer + var tracer *tracers.Tracer if ctx.IsSet(TraceTracerFlag.Name) { // Custom tracing config := json.RawMessage(ctx.String(TraceTracerConfigFlag.Name)) - tracer, err := tracers.DefaultDirectory.New(ctx.String(TraceTracerFlag.Name), + innerTracer, err := tracers.DefaultDirectory.New(ctx.String(TraceTracerFlag.Name), nil, config, chainConfig) if err != nil { return NewError(ErrorConfig, fmt.Errorf("failed instantiating tracer: %v", err)) } - vmConfig.Tracer = newResultWriter(baseDir, tracer) + tracer = newResultWriter(baseDir, innerTracer) } else if ctx.Bool(TraceFlag.Name) { // JSON opcode tracing logConfig := &logger.Config{ DisableStack: ctx.Bool(TraceDisableStackFlag.Name), @@ -182,20 +184,45 @@ func Transition(ctx *cli.Context) error { EnableReturnData: ctx.Bool(TraceEnableReturnDataFlag.Name), } if ctx.Bool(TraceEnableCallFramesFlag.Name) { - vmConfig.Tracer = newFileWriter(baseDir, func(out io.Writer) *tracing.Hooks { + tracer = newFileWriter(baseDir, func(out io.Writer) *tracing.Hooks { return logger.NewJSONLoggerWithCallFrames(logConfig, out) }) } else { - vmConfig.Tracer = newFileWriter(baseDir, func(out io.Writer) *tracing.Hooks { + tracer = newFileWriter(baseDir, func(out io.Writer) *tracing.Hooks { return logger.NewJSONLogger(logConfig, out) }) } } + // Configure opcode counter + var opcodeTracer *tracers.Tracer + if ctx.IsSet(OpcodeCountFlag.Name) && ctx.String(OpcodeCountFlag.Name) != "" { + opcodeTracer = native.NewOpcodeCounter() + if tracer != nil { + // If we have an existing tracer, multiplex with the opcode tracer + mux, _ := native.NewMuxTracer([]string{"trace", "opcode"}, []*tracers.Tracer{tracer, opcodeTracer}) + vmConfig.Tracer = mux.Hooks + } else { + vmConfig.Tracer = opcodeTracer.Hooks + } + } else if tracer != nil { + vmConfig.Tracer = tracer.Hooks + } // Run the test and aggregate the result s, result, body, err := prestate.Apply(vmConfig, chainConfig, txIt, ctx.Int64(RewardFlag.Name)) if err != nil { return err } + // Write opcode counts if enabled + if opcodeTracer != nil { + fname := ctx.String(OpcodeCountFlag.Name) + result, err := opcodeTracer.GetResult() + if err != nil { + return NewError(ErrorJson, fmt.Errorf("failed getting opcode counts: %v", err)) + } + if err := saveFile(baseDir, fname, result); err != nil { + return err + } + } // Dump the execution result var ( collector = make(Alloc) diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 57741b5f9c..77a06bec26 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -161,6 +161,7 @@ var ( t8ntool.ForknameFlag, t8ntool.ChainIDFlag, t8ntool.RewardFlag, + t8ntool.OpcodeCountFlag, }, } diff --git a/cmd/evm/testdata/1/exp.json b/cmd/evm/testdata/1/exp.json index 6537c9517d..f596cb5e67 100644 --- a/cmd/evm/testdata/1/exp.json +++ b/cmd/evm/testdata/1/exp.json @@ -24,7 +24,7 @@ "status": "0x1", "cumulativeGasUsed": "0x5208", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x0557bacce3375c98d806609b8d5043072f0b6a8bae45ae5a67a00d3a1a18d673", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", diff --git a/cmd/evm/testdata/13/exp2.json b/cmd/evm/testdata/13/exp2.json index f716289cf7..260721e22f 100644 --- a/cmd/evm/testdata/13/exp2.json +++ b/cmd/evm/testdata/13/exp2.json @@ -12,7 +12,7 @@ "status": "0x0", "cumulativeGasUsed": "0x84d0", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0xa98a24882ea90916c6a86da650fbc6b14238e46f0af04a131ce92be897507476", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x84d0", @@ -27,7 +27,7 @@ "status": "0x0", "cumulativeGasUsed": "0x109a0", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x36bad80acce7040c45fd32764b5c2b2d2e6f778669fb41791f73f546d56e739a", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x84d0", diff --git a/cmd/evm/testdata/23/exp.json b/cmd/evm/testdata/23/exp.json index 2d9cd492db..803411de46 100644 --- a/cmd/evm/testdata/23/exp.json +++ b/cmd/evm/testdata/23/exp.json @@ -11,7 +11,7 @@ "status": "0x1", "cumulativeGasUsed": "0x520b", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x72fadbef39cd251a437eea619cfeda752271a5faaaa2147df012e112159ffb81", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x520b", diff --git a/cmd/evm/testdata/24/exp.json b/cmd/evm/testdata/24/exp.json index 0dd552e112..bbde70c631 100644 --- a/cmd/evm/testdata/24/exp.json +++ b/cmd/evm/testdata/24/exp.json @@ -27,7 +27,7 @@ "status": "0x1", "cumulativeGasUsed": "0xa861", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x92ea4a28224d033afb20e0cc2b290d4c7c2d61f6a4800a680e4e19ac962ee941", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0xa861", @@ -41,7 +41,7 @@ "status": "0x1", "cumulativeGasUsed": "0x10306", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x16b1d912f1d664f3f60f4e1b5f296f3c82a64a1a253117b4851d18bc03c4f1da", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5aa5", diff --git a/cmd/evm/testdata/25/exp.json b/cmd/evm/testdata/25/exp.json index 3dac46aa60..25cac55dc0 100644 --- a/cmd/evm/testdata/25/exp.json +++ b/cmd/evm/testdata/25/exp.json @@ -23,7 +23,7 @@ "status": "0x1", "cumulativeGasUsed": "0x5208", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x92ea4a28224d033afb20e0cc2b290d4c7c2d61f6a4800a680e4e19ac962ee941", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", diff --git a/cmd/evm/testdata/28/exp.json b/cmd/evm/testdata/28/exp.json index 15b29bc0ac..f67ff76087 100644 --- a/cmd/evm/testdata/28/exp.json +++ b/cmd/evm/testdata/28/exp.json @@ -28,7 +28,7 @@ "status": "0x1", "cumulativeGasUsed": "0xa865", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x7508d7139d002a4b3a26a4f12dec0d87cb46075c78bf77a38b569a133b509262", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0xa865", diff --git a/cmd/evm/testdata/29/exp.json b/cmd/evm/testdata/29/exp.json index 69c8661aa8..cb4b5eaa28 100644 --- a/cmd/evm/testdata/29/exp.json +++ b/cmd/evm/testdata/29/exp.json @@ -26,7 +26,7 @@ "status": "0x1", "cumulativeGasUsed": "0x5208", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x84f70aba406a55628a0620f26d260f90aeb6ccc55fed6ec2ac13dd4f727032ed", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", diff --git a/cmd/evm/testdata/3/exp.json b/cmd/evm/testdata/3/exp.json index 807cdccfb4..36bf2604af 100644 --- a/cmd/evm/testdata/3/exp.json +++ b/cmd/evm/testdata/3/exp.json @@ -24,7 +24,7 @@ "status": "0x1", "cumulativeGasUsed": "0x521f", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x72fadbef39cd251a437eea619cfeda752271a5faaaa2147df012e112159ffb81", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x521f", diff --git a/cmd/evm/testdata/30/exp.json b/cmd/evm/testdata/30/exp.json index 9861f5a071..7df3b78820 100644 --- a/cmd/evm/testdata/30/exp.json +++ b/cmd/evm/testdata/30/exp.json @@ -25,7 +25,7 @@ "status": "0x1", "cumulativeGasUsed": "0x5208", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0xa98a24882ea90916c6a86da650fbc6b14238e46f0af04a131ce92be897507476", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", @@ -40,7 +40,7 @@ "status": "0x1", "cumulativeGasUsed": "0xa410", "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "logs": null, + "logs": [], "transactionHash": "0x36bad80acce7040c45fd32764b5c2b2d2e6f778669fb41791f73f546d56e739a", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x5208", diff --git a/cmd/evm/testdata/33/exp.json b/cmd/evm/testdata/33/exp.json index b40ca9fee2..73aaf80a28 100644 --- a/cmd/evm/testdata/33/exp.json +++ b/cmd/evm/testdata/33/exp.json @@ -44,7 +44,7 @@ "root": "0x", "status": "0x1", "cumulativeGasUsed": "0x15fa9", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","logs": null,"transactionHash": "0x0417aab7c1d8a3989190c3167c132876ce9b8afd99262c5a0f9d06802de3d7ef", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","logs": [],"transactionHash": "0x0417aab7c1d8a3989190c3167c132876ce9b8afd99262c5a0f9d06802de3d7ef", "contractAddress": "0x0000000000000000000000000000000000000000", "gasUsed": "0x15fa9", "effectiveGasPrice": null, diff --git a/cmd/fetchpayload/main.go b/cmd/fetchpayload/main.go new file mode 100644 index 0000000000..eafc05fbe8 --- /dev/null +++ b/cmd/fetchpayload/main.go @@ -0,0 +1,177 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +// fetchpayload queries an Ethereum node over RPC, fetches a block and its +// execution witness, and writes the combined Payload (ChainID + Block + +// Witness) to disk in the format consumed by cmd/keeper. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "math/big" + "os" + "path/filepath" + "strings" + "time" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/stateless" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rpc" +) + +// Payload is duplicated from cmd/keeper/main.go (package main, not importable). +type Payload struct { + ChainID uint64 + Block *types.Block + Witness *stateless.Witness +} + +func main() { + var ( + rpcURL = flag.String("rpc", "http://localhost:8545", "RPC endpoint URL") + blockArg = flag.String("block", "latest", `Block number: decimal, 0x-hex, or "latest"`) + format = flag.String("format", "rlp", "Comma-separated output formats: rlp, hex, json") + outDir = flag.String("out", "", "Output directory (default: current directory)") + ) + flag.Parse() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + // Parse block number (nil means "latest" in ethclient). + blockNum, err := parseBlockNumber(*blockArg) + if err != nil { + fatal("invalid block number %q: %v", *blockArg, err) + } + + // Connect to the node. + client, err := ethclient.DialContext(ctx, *rpcURL) + if err != nil { + fatal("failed to connect to %s: %v", *rpcURL, err) + } + defer client.Close() + + chainID, err := client.ChainID(ctx) + if err != nil { + fatal("failed to get chain ID: %v", err) + } + + // Fetch the block first so we have a concrete number for the witness call, + // avoiding a race where "latest" advances between the two RPCs. + block, err := client.BlockByNumber(ctx, blockNum) + if err != nil { + fatal("failed to fetch block: %v", err) + } + fmt.Printf("Fetched block %d (%#x)\n", block.NumberU64(), block.Hash()) + + // Fetch the execution witness via the debug namespace. + var extWitness stateless.ExtWitness + err = client.Client().CallContext(ctx, &extWitness, "debug_executionWitness", rpc.BlockNumber(block.NumberU64())) + if err != nil { + fatal("failed to fetch execution witness: %v", err) + } + + witness := new(stateless.Witness) + err = witness.FromExtWitness(&extWitness) + if err != nil { + fatal("failed to convert witness: %v", err) + } + + payload := Payload{ + ChainID: chainID.Uint64(), + Block: block, + Witness: witness, + } + + // Encode payload as RLP (shared by "rlp" and "hex" formats). + rlpBytes, err := rlp.EncodeToBytes(payload) + if err != nil { + fatal("failed to RLP-encode payload: %v", err) + } + + // Write one output file per requested format. + blockHex := fmt.Sprintf("%x", block.NumberU64()) + for f := range strings.SplitSeq(*format, ",") { + f = strings.TrimSpace(f) + outPath := filepath.Join(*outDir, fmt.Sprintf("%s_payload.%s", blockHex, f)) + + var data []byte + switch f { + case "rlp": + data = rlpBytes + case "hex": + data = []byte(hexutil.Encode(rlpBytes)) + case "json": + data, err = marshalJSONPayload(chainID, block, &extWitness) + if err != nil { + fatal("failed to JSON-encode payload: %v", err) + } + default: + fatal("unknown format %q (valid: rlp, hex, json)", f) + } + + if err := os.WriteFile(outPath, data, 0644); err != nil { + fatal("failed to write %s: %v", outPath, err) + } + fmt.Printf("Wrote %s (%d bytes)\n", outPath, len(data)) + } +} + +// parseBlockNumber converts a CLI string to *big.Int. +// Returns nil for "latest" (ethclient convention for the head block). +func parseBlockNumber(s string) (*big.Int, error) { + if strings.EqualFold(s, "latest") { + return nil, nil + } + n := new(big.Int) + if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { + if _, ok := n.SetString(s[2:], 16); !ok { + return nil, fmt.Errorf("invalid hex number") + } + return n, nil + } + if _, ok := n.SetString(s, 10); !ok { + return nil, fmt.Errorf("invalid decimal number") + } + return n, nil +} + +// jsonPayload is a JSON-friendly representation of Payload. It uses ExtWitness +// instead of the internal Witness (which has no JSON marshaling). +type jsonPayload struct { + ChainID uint64 `json:"chainId"` + Block *types.Block `json:"block"` + Witness *stateless.ExtWitness `json:"witness"` +} + +func marshalJSONPayload(chainID *big.Int, block *types.Block, ext *stateless.ExtWitness) ([]byte, error) { + return json.MarshalIndent(jsonPayload{ + ChainID: chainID.Uint64(), + Block: block, + Witness: ext, + }, "", " ") +} + +func fatal(format string, args ...any) { + fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +} diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 1ccb78d622..7e14ec1c60 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -111,6 +111,7 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, + utils.MetricsInfluxDBIntervalFlag, utils.MetricsInfluxDBTokenFlag, utils.MetricsInfluxDBBucketFlag, utils.MetricsInfluxDBOrganizationFlag, @@ -207,13 +208,19 @@ This command dumps out the state for a given block (or latest, if none provided) pruneHistoryCommand = &cli.Command{ Action: pruneHistory, Name: "prune-history", - Usage: "Prune blockchain history (block bodies and receipts) up to the merge block", + Usage: "Prune blockchain history (block bodies and receipts) up to a specified point", ArgsUsage: "", - Flags: utils.DatabaseFlags, + Flags: slices.Concat(utils.DatabaseFlags, []cli.Flag{ + utils.ChainHistoryFlag, + }), Description: ` The prune-history command removes historical block bodies and receipts from the -blockchain database up to the merge block, while preserving block headers. This -helps reduce storage requirements for nodes that don't need full historical data.`, +blockchain database up to a specified point, while preserving block headers. This +helps reduce storage requirements for nodes that don't need full historical data. + +The --history.chain flag is required to specify the pruning target: + - postmerge: Prune up to the merge block. The node will keep the merge block and everything thereafter. + - postprague: Prune up to the Prague (Pectra) upgrade block. The node will keep the prague block and everything thereafter.`, } downloadEraCommand = &cli.Command{ @@ -702,47 +709,74 @@ func hashish(x string) bool { } func pruneHistory(ctx *cli.Context) error { + // Parse and validate the history mode flag. + if !ctx.IsSet(utils.ChainHistoryFlag.Name) { + return errors.New("--history.chain flag is required") + } + var mode history.HistoryMode + if err := mode.UnmarshalText([]byte(ctx.String(utils.ChainHistoryFlag.Name))); err != nil { + return err + } + if mode == history.KeepAll { + return errors.New("--history.chain=all is not valid for pruning. To restore history, use 'geth import-history'") + } + stack, _ := makeConfigNode(ctx) defer stack.Close() - // Open the chain database + // Open the chain database. chain, chaindb := utils.MakeChain(ctx, stack, false) defer chaindb.Close() defer chain.Stop() - // Determine the prune point. This will be the first PoS block. - prunePoint, ok := history.PrunePoints[chain.Genesis().Hash()] - if !ok || prunePoint == nil { - return errors.New("prune point not found") + // Determine the prune point based on the history mode. + genesisHash := chain.Genesis().Hash() + prunePoint := history.GetPrunePoint(genesisHash, mode) + if prunePoint == nil { + return fmt.Errorf("prune point for %q not found for this network", mode.String()) } var ( - mergeBlock = prunePoint.BlockNumber - mergeBlockHash = prunePoint.BlockHash.Hex() + targetBlock = prunePoint.BlockNumber + targetBlockHash = prunePoint.BlockHash ) - // Check we're far enough past merge to ensure all data is in freezer + // Check the current freezer tail to see if pruning is needed/possible. + freezerTail, _ := chaindb.Tail() + if freezerTail > 0 { + if freezerTail == targetBlock { + log.Info("Database already pruned to target block", "tail", freezerTail) + return nil + } + if freezerTail > targetBlock { + // Database is pruned beyond the target - can't unprune. + return fmt.Errorf("database is already pruned to block %d, which is beyond target %d. Cannot unprune. To restore history, use 'geth import-history'", freezerTail, targetBlock) + } + // freezerTail < targetBlock: we can prune further, continue below. + } + + // Check we're far enough past the target to ensure all data is in freezer. currentHeader := chain.CurrentHeader() if currentHeader == nil { return errors.New("current header not found") } - if currentHeader.Number.Uint64() < mergeBlock+params.FullImmutabilityThreshold { - return fmt.Errorf("chain not far enough past merge block, need %d more blocks", - mergeBlock+params.FullImmutabilityThreshold-currentHeader.Number.Uint64()) + if currentHeader.Number.Uint64() < targetBlock+params.FullImmutabilityThreshold { + return fmt.Errorf("chain not far enough past target block %d, need %d more blocks", + targetBlock, targetBlock+params.FullImmutabilityThreshold-currentHeader.Number.Uint64()) } - // Double-check the prune block in db has the expected hash. - hash := rawdb.ReadCanonicalHash(chaindb, mergeBlock) - if hash != common.HexToHash(mergeBlockHash) { - return fmt.Errorf("merge block hash mismatch: got %s, want %s", hash.Hex(), mergeBlockHash) + // Double-check the target block in db has the expected hash. + hash := rawdb.ReadCanonicalHash(chaindb, targetBlock) + if hash != targetBlockHash { + return fmt.Errorf("target block hash mismatch: got %s, want %s", hash.Hex(), targetBlockHash.Hex()) } - log.Info("Starting history pruning", "head", currentHeader.Number, "tail", mergeBlock, "tailHash", mergeBlockHash) + log.Info("Starting history pruning", "head", currentHeader.Number, "target", targetBlock, "targetHash", targetBlockHash.Hex()) start := time.Now() - rawdb.PruneTransactionIndex(chaindb, mergeBlock) - if _, err := chaindb.TruncateTail(mergeBlock); err != nil { + rawdb.PruneTransactionIndex(chaindb, targetBlock) + if _, err := chaindb.TruncateTail(targetBlock); err != nil { return fmt.Errorf("failed to truncate ancient data: %v", err) } - log.Info("History pruning completed", "tail", mergeBlock, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("History pruning completed", "tail", targetBlock, "elapsed", common.PrettyDuration(time.Since(start))) // TODO(s1na): what if there is a crash between the two prune operations? diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 7943d95d65..720d1ef9fc 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -377,6 +377,9 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { if ctx.IsSet(utils.MetricsInfluxDBTagsFlag.Name) { cfg.Metrics.InfluxDBTags = ctx.String(utils.MetricsInfluxDBTagsFlag.Name) } + if ctx.IsSet(utils.MetricsInfluxDBIntervalFlag.Name) { + cfg.Metrics.InfluxDBInterval = ctx.Duration(utils.MetricsInfluxDBIntervalFlag.Name) + } if ctx.IsSet(utils.MetricsEnableInfluxDBV2Flag.Name) { cfg.Metrics.EnableInfluxDBV2 = ctx.Bool(utils.MetricsEnableInfluxDBV2Flag.Name) } diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index 871e8c175f..12ee7e7dd1 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -30,7 +30,7 @@ import ( ) const ( - ipcAPIs = "admin:1.0 debug:1.0 engine:1.0 eth:1.0 miner:1.0 net:1.0 rpc:1.0 txpool:1.0 web3:1.0" + ipcAPIs = "admin:1.0 debug:1.0 engine:1.0 eth:1.0 miner:1.0 net:1.0 rpc:1.0 testing:1.0 txpool:1.0 web3:1.0" httpAPIs = "eth:1.0 net:1.0 rpc:1.0 web3:1.0" ) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index fb688793e3..455dd05aca 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -19,6 +19,7 @@ package main import ( "bytes" "fmt" + "math" "os" "os/signal" "path/filepath" @@ -37,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/tablewriter" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -51,7 +53,24 @@ var ( } removeChainDataFlag = &cli.BoolFlag{ Name: "remove.chain", - Usage: "If set, selects the state data for removal", + Usage: "If set, selects the chain data for removal", + } + inspectTrieTopFlag = &cli.IntFlag{ + Name: "top", + Usage: "Print the top N results per ranking category", + Value: 10, + } + inspectTrieDumpPathFlag = &cli.StringFlag{ + Name: "dump-path", + Usage: "Path for the trie statistics dump file", + } + inspectTrieSummarizeFlag = &cli.StringFlag{ + Name: "summarize", + Usage: "Summarize an existing trie dump file (skip trie traversal)", + } + inspectTrieContractFlag = &cli.StringFlag{ + Name: "contract", + Usage: "Inspect only the storage of the given contract address (skips full account trie walk)", } removedbCommand = &cli.Command{ @@ -74,6 +93,7 @@ Remove blockchain and state databases`, dbCompactCmd, dbGetCmd, dbDeleteCmd, + dbInspectTrieCmd, dbPutCmd, dbGetSlotsCmd, dbDumpFreezerIndex, @@ -92,6 +112,22 @@ Remove blockchain and state databases`, Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } + dbInspectTrieCmd = &cli.Command{ + Action: inspectTrie, + Name: "inspect-trie", + ArgsUsage: "", + Flags: slices.Concat([]cli.Flag{ + utils.ExcludeStorageFlag, + inspectTrieTopFlag, + utils.OutputFileFlag, + inspectTrieDumpPathFlag, + inspectTrieSummarizeFlag, + inspectTrieContractFlag, + }, utils.NetworkFlags, utils.DatabaseFlags), + Usage: "Print detailed trie information about the structure of account trie and storage tries.", + Description: `This commands iterates the entrie trie-backed state. If the 'blocknum' is not specified, +the latest block number will be used by default.`, + } dbCheckStateContentCmd = &cli.Command{ Action: checkStateContent, Name: "check-state-content", @@ -385,6 +421,88 @@ func checkStateContent(ctx *cli.Context) error { return nil } +func inspectTrie(ctx *cli.Context) error { + topN := ctx.Int(inspectTrieTopFlag.Name) + if topN <= 0 { + return fmt.Errorf("invalid --%s value %d (must be > 0)", inspectTrieTopFlag.Name, topN) + } + config := &trie.InspectConfig{ + NoStorage: ctx.Bool(utils.ExcludeStorageFlag.Name), + TopN: topN, + Path: ctx.String(utils.OutputFileFlag.Name), + } + + if summarizePath := ctx.String(inspectTrieSummarizeFlag.Name); summarizePath != "" { + if ctx.NArg() > 0 { + return fmt.Errorf("block number argument is not supported with --%s", inspectTrieSummarizeFlag.Name) + } + config.DumpPath = summarizePath + log.Info("Summarizing trie dump", "path", summarizePath, "top", topN) + return trie.Summarize(summarizePath, config) + } + if ctx.NArg() > 1 { + return fmt.Errorf("excessive number of arguments: %v", ctx.Command.ArgsUsage) + } + + stack, _ := makeConfigNode(ctx) + db := utils.MakeChainDatabase(ctx, stack, false) + defer stack.Close() + defer db.Close() + + var ( + trieRoot common.Hash + hash common.Hash + number uint64 + ) + switch { + case ctx.NArg() == 0 || ctx.Args().Get(0) == "latest": + head := rawdb.ReadHeadHeaderHash(db) + n, ok := rawdb.ReadHeaderNumber(db, head) + if !ok { + return fmt.Errorf("could not load head block hash") + } + number = n + case ctx.Args().Get(0) == "snapshot": + trieRoot = rawdb.ReadSnapshotRoot(db) + number = math.MaxUint64 + default: + var err error + number, err = strconv.ParseUint(ctx.Args().Get(0), 10, 64) + if err != nil { + return fmt.Errorf("failed to parse blocknum, Args[0]: %v, err: %v", ctx.Args().Get(0), err) + } + } + + if number != math.MaxUint64 { + hash = rawdb.ReadCanonicalHash(db, number) + if hash == (common.Hash{}) { + return fmt.Errorf("canonical hash for block %d not found", number) + } + blockHeader := rawdb.ReadHeader(db, hash, number) + trieRoot = blockHeader.Root + } + if trieRoot == (common.Hash{}) { + log.Error("Empty root hash") + } + + config.DumpPath = ctx.String(inspectTrieDumpPathFlag.Name) + if config.DumpPath == "" { + config.DumpPath = stack.ResolvePath("trie-dump.bin") + } + + triedb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false) + defer triedb.Close() + + if contractAddr := ctx.String(inspectTrieContractFlag.Name); contractAddr != "" { + address := common.HexToAddress(contractAddr) + log.Info("Inspecting contract", "address", address, "root", trieRoot, "block", number) + return trie.InspectContract(triedb, db, trieRoot, address) + } + + log.Info("Inspecting trie", "root", trieRoot, "block", number, "dump", config.DumpPath, "top", topN) + return trie.Inspect(triedb, trieRoot, config) +} + func showDBStats(db ethdb.KeyValueStater) { stats, err := db.Stat() if err != nil { @@ -759,7 +877,7 @@ func showMetaData(ctx *cli.Context) error { data = append(data, []string{"headHeader.Root", fmt.Sprintf("%v", h.Root)}) data = append(data, []string{"headHeader.Number", fmt.Sprintf("%d (%#x)", h.Number, h.Number)}) } - table := rawdb.NewTableWriter(os.Stdout) + table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Field", "Value"}) table.AppendBulk(data) table.Render() diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 1a197743d6..48e189d98c 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -22,7 +22,6 @@ import ( "os" "slices" "sort" - "strconv" "time" "github.com/ethereum/go-ethereum/accounts" @@ -217,6 +216,7 @@ var ( utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, + utils.MetricsInfluxDBIntervalFlag, utils.MetricsEnableInfluxDBV2Flag, utils.MetricsInfluxDBTokenFlag, utils.MetricsInfluxDBBucketFlag, @@ -316,18 +316,6 @@ func prepare(ctx *cli.Context) { case !ctx.IsSet(utils.NetworkIdFlag.Name): log.Info("Starting Geth on Ethereum mainnet...") } - // If we're a full node on mainnet without --cache specified, bump default cache allowance - if !ctx.IsSet(utils.CacheFlag.Name) && !ctx.IsSet(utils.NetworkIdFlag.Name) { - // Make sure we're not on any supported preconfigured testnet either - if !ctx.IsSet(utils.HoleskyFlag.Name) && - !ctx.IsSet(utils.SepoliaFlag.Name) && - !ctx.IsSet(utils.HoodiFlag.Name) && - !ctx.IsSet(utils.DeveloperFlag.Name) { - // Nope, we're really on mainnet. Bump that cache up! - log.Info("Bumping default cache on mainnet", "provided", ctx.Int(utils.CacheFlag.Name), "updated", 4096) - ctx.Set(utils.CacheFlag.Name, strconv.Itoa(4096)) - } - } } // geth is the main entry point into the system if no special subcommand is run. diff --git a/cmd/keeper/go.mod b/cmd/keeper/go.mod index c193f4fc2d..8303b4ab2e 100644 --- a/cmd/keeper/go.mod +++ b/cmd/keeper/go.mod @@ -13,13 +13,15 @@ require ( github.com/bits-and-blooms/bitset v1.20.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/consensys/gnark-crypto v0.18.1 // indirect - github.com/crate-crypto/go-eth-kzg v1.4.0 // indirect + github.com/crate-crypto/go-eth-kzg v1.5.0 // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/emicklei/dot v1.6.2 // indirect - github.com/ethereum/c-kzg-4844/v2 v2.1.5 // indirect + github.com/ethereum/c-kzg-4844/v2 v2.1.6 // indirect github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab // indirect github.com/ferranbt/fastssz v0.1.4 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect github.com/gofrs/flock v0.12.1 // indirect github.com/golang/snappy v1.0.0 // indirect @@ -29,12 +31,16 @@ require ( github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect - github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe // indirect + github.com/supranational/blst v0.3.16 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.40.0 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect + go.opentelemetry.io/otel/trace v1.40.0 // indirect golang.org/x/crypto v0.44.0 // indirect golang.org/x/sync v0.18.0 // indirect - golang.org/x/sys v0.39.0 // indirect + golang.org/x/sys v0.40.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/cmd/keeper/go.sum b/cmd/keeper/go.sum index e9c081e605..a162537c88 100644 --- a/cmd/keeper/go.sum +++ b/cmd/keeper/go.sum @@ -28,8 +28,8 @@ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAK github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDdoGqFt3fI= github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= -github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= -github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= +github.com/crate-crypto/go-eth-kzg v1.5.0 h1:FYRiJMJG2iv+2Dy3fi14SVGjcPteZ5HAAUe4YWlJygc= +github.com/crate-crypto/go-eth-kzg v1.5.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80NsVHagjM= @@ -40,14 +40,19 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= -github.com/ethereum/c-kzg-4844/v2 v2.1.5 h1:aVtoLK5xwJ6c5RiqO8g8ptJ5KU+2Hdquf6G3aXiHh5s= -github.com/ethereum/c-kzg-4844/v2 v2.1.5/go.mod h1:u59hRTTah4Co6i9fDWtiCjTrblJv0UwsqZKCc0GfgUs= +github.com/ethereum/c-kzg-4844/v2 v2.1.6 h1:xQymkKCT5E2Jiaoqf3v4wsNgjZLY0lRSkZn27fRjSls= +github.com/ethereum/c-kzg-4844/v2 v2.1.6/go.mod h1:8HMkUZ5JRv4hpw/XUrYWSQNAUzhHMg2UDb/U+5m+XNw= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJhy07IMfEKuARQ9TKojGqLVNxQajaXEp/BoqSk= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= github.com/ferranbt/fastssz v0.1.4 h1:OCDB+dYDEQDvAgtAGnTSidK1Pe2tW3nFV40XyMkTeDY= github.com/ferranbt/fastssz v0.1.4/go.mod h1:Ea3+oeoRGGLGm5shYAeDgu6PGUlcvQhE2fILyD9+tGg= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= @@ -59,6 +64,10 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= @@ -102,12 +111,22 @@ github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw= -github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.16 h1:bTDadT+3fK497EvLdWRQEjiGnUtzJ7jjIUMF0jqwYhE= +github.com/supranational/blst v0.3.16/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= golang.org/x/crypto v0.44.0 h1:A97SsFvM3AIwEEmTBiaxPPTYpDC47w720rdiiUvgoAU= golang.org/x/crypto v0.44.0/go.mod h1:013i+Nw79BMiQiMsOPcVCB5ZIJbYkerPrGnOa00tvmc= golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME= @@ -118,8 +137,8 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/cmd/keeper/main.go b/cmd/keeper/main.go index 9b459f6f36..df6881acbf 100644 --- a/cmd/keeper/main.go +++ b/cmd/keeper/main.go @@ -17,6 +17,7 @@ package main import ( + "context" "fmt" "os" "runtime/debug" @@ -52,7 +53,7 @@ func main() { } vmConfig := vm.Config{} - crossStateRoot, crossReceiptRoot, err := core.ExecuteStateless(chainConfig, vmConfig, payload.Block, payload.Witness) + crossStateRoot, crossReceiptRoot, err := core.ExecuteStateless(context.Background(), chainConfig, vmConfig, payload.Block, payload.Witness) if err != nil { fmt.Fprintf(os.Stderr, "stateless self-validation failed: %v\n", err) os.Exit(10) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c172579da5..5676f0e1af 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -218,7 +218,11 @@ var ( Usage: "Max number of elements (0 = no limit)", Value: 0, } - + OutputFileFlag = &cli.StringFlag{ + Name: "output", + Usage: "Writes the result in json to the output", + Value: "", + } SnapshotFlag = &cli.BoolFlag{ Name: "snapshot", Usage: `Enables snapshot-database mode (default = enable)`, @@ -321,7 +325,7 @@ var ( } ChainHistoryFlag = &cli.StringFlag{ Name: "history.chain", - Usage: `Blockchain history retention ("all" or "postmerge")`, + Usage: `Blockchain history retention ("all", "postmerge", or "postprague")`, Value: ethconfig.Defaults.HistoryMode.String(), Category: flags.StateCategory, } @@ -486,8 +490,8 @@ var ( // Performance tuning settings CacheFlag = &cli.IntFlag{ Name: "cache", - Usage: "Megabytes of memory allocated to internal caching (default = 4096 mainnet full node, 128 light mode)", - Value: 1024, + Usage: "Megabytes of memory allocated to internal caching", + Value: 4096, Category: flags.PerfCategory, } CacheDatabaseFlag = &cli.IntFlag{ @@ -1022,6 +1026,13 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server. Category: flags.MetricsCategory, } + MetricsInfluxDBIntervalFlag = &cli.DurationFlag{ + Name: "metrics.influxdb.interval", + Usage: "Interval between metrics reports to InfluxDB (with time unit, e.g. 10s)", + Value: metrics.DefaultConfig.InfluxDBInterval, + Category: flags.MetricsCategory, + } + MetricsEnableInfluxDBV2Flag = &cli.BoolFlag{ Name: "metrics.influxdbv2", Usage: "Enable metrics export/push to an external InfluxDB v2 database", @@ -2255,13 +2266,14 @@ func SetupMetrics(cfg *metrics.Config) { bucket = cfg.InfluxDBBucket organization = cfg.InfluxDBOrganization tagsMap = SplitTagsFlag(cfg.InfluxDBTags) + interval = cfg.InfluxDBInterval ) if enableExport { - log.Info("Enabling metrics export to InfluxDB") - go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap) + log.Info("Enabling metrics export to InfluxDB", "interval", interval) + go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, interval, endpoint, database, username, password, "geth.", tagsMap) } else if enableExportV2 { - log.Info("Enabling metrics export to InfluxDB (v2)") - go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, token, bucket, organization, "geth.", tagsMap) + log.Info("Enabling metrics export to InfluxDB (v2)", "interval", interval) + go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, interval, endpoint, token, bucket, organization, "geth.", tagsMap) } // Expvar exporter. @@ -2466,8 +2478,6 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh } vmcfg := vm.Config{ EnablePreimageRecording: ctx.Bool(VMEnableDebugFlag.Name), - EnableWitnessStats: ctx.Bool(VMWitnessStatsFlag.Name), - StatelessSelfValidation: ctx.Bool(VMStatelessSelfValidationFlag.Name) || ctx.Bool(VMWitnessStatsFlag.Name), } if ctx.IsSet(VMTraceFlag.Name) { if name := ctx.String(VMTraceFlag.Name); name != "" { @@ -2481,6 +2491,9 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh } options.VmConfig = vmcfg + options.StatelessSelfValidation = ctx.Bool(VMStatelessSelfValidationFlag.Name) || ctx.Bool(VMWitnessStatsFlag.Name) + options.EnableWitnessStats = ctx.Bool(VMWitnessStatsFlag.Name) + chain, err := core.NewBlockChain(chainDb, gspec, engine, options) if err != nil { Fatalf("Can't create BlockChain: %v", err) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index eed27407a5..45a480c50e 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -272,6 +272,14 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa return err } } + + amsterdam := chain.Config().IsAmsterdam(header.Number, header.Time) + if amsterdam && header.SlotNumber == nil { + return errors.New("header is missing slotNumber") + } + if !amsterdam && header.SlotNumber != nil { + return fmt.Errorf("invalid slotNumber: have %d, expected nil", *header.SlotNumber) + } return nil } diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 87cd407a71..28095011c1 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -310,6 +310,8 @@ func (c *Clique) verifyHeader(chain consensus.ChainHeaderReader, header *types.H return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", *header.BlobGasUsed) case header.ParentBeaconRoot != nil: return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", *header.ParentBeaconRoot) + case header.SlotNumber != nil: + return fmt.Errorf("invalid slotNumber, have %#x, expected nil", *header.SlotNumber) } // All basic checks passed, verify cascading fields return c.verifyCascadingFields(chain, header, parents) @@ -694,6 +696,9 @@ func encodeSigHeader(w io.Writer, header *types.Header) { if header.ParentBeaconRoot != nil { panic("unexpected parent beacon root value in clique") } + if header.SlotNumber != nil { + panic("unexpected slot number value in clique") + } if err := rlp.Encode(w, enc); err != nil { panic("can't encode: " + err.Error()) } diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index f90001fc1a..61371e0636 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -283,6 +283,8 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainHeaderReader, header, pa return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", *header.BlobGasUsed) case header.ParentBeaconRoot != nil: return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", *header.ParentBeaconRoot) + case header.SlotNumber != nil: + return fmt.Errorf("invalid slotNumber, have %#x, expected nil", *header.SlotNumber) } // Add some fake checks for tests if ethash.fakeDelay != nil { @@ -559,6 +561,9 @@ func (ethash *Ethash) SealHash(header *types.Header) (hash common.Hash) { if header.ParentBeaconRoot != nil { panic("parent beacon root set on ethash") } + if header.SlotNumber != nil { + panic("slot number set on ethash") + } rlp.Encode(hasher, enc) hasher.Sum(hash[:0]) return hash diff --git a/consensus/misc/eip1559/eip1559.go b/consensus/misc/eip1559/eip1559.go index a90bd744b2..9a4cd320a8 100644 --- a/consensus/misc/eip1559/eip1559.go +++ b/consensus/misc/eip1559/eip1559.go @@ -43,6 +43,10 @@ func VerifyEIP1559Header(config *params.ChainConfig, parent, header *types.Heade if header.BaseFee == nil { return errors.New("header is missing baseFee") } + // Verify the parent header is not malformed + if config.IsLondon(parent.Number) && parent.BaseFee == nil { + return errors.New("parent header is missing baseFee") + } // Verify the baseFee is correct based on the parent header. expectedBaseFee := CalcBaseFee(config, parent) if header.BaseFee.Cmp(expectedBaseFee) != 0 { diff --git a/core/block_validator_test.go b/core/block_validator_test.go index fcc99effd0..0280862e3c 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -17,6 +17,7 @@ package core import ( + "context" "math/big" "testing" "time" @@ -210,7 +211,7 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { t.Fatalf("post-block %d: unexpected result returned: %v", i, result) case <-time.After(25 * time.Millisecond): } - chain.InsertBlockWithoutSetHead(postBlocks[i], false) + chain.InsertBlockWithoutSetHead(context.Background(), postBlocks[i], false) } // Verify the blocks with pre-merge blocks and post-merge blocks diff --git a/core/blockchain.go b/core/blockchain.go index 4756fde632..95fb2c9ea8 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -18,6 +18,7 @@ package core import ( + "context" "errors" "fmt" "io" @@ -47,6 +48,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/syncx" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -91,9 +93,7 @@ var ( accountReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/account/single/reads", nil) storageReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/storage/single/reads", nil) codeReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/code/single/reads", nil) - - snapshotCommitTimer = metrics.NewRegisteredResettingTimer("chain/snapshot/commits", nil) - triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) + triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil) blockInsertTimer = metrics.NewRegisteredResettingTimer("chain/inserts", nil) blockValidationTimer = metrics.NewRegisteredResettingTimer("chain/validation", nil) @@ -220,6 +220,10 @@ type BlockChainConfig struct { // detailed statistics will be logged. Negative value means disabled (default), // zero logs all blocks, positive value filters blocks by execution time. SlowBlockThreshold time.Duration + + // Execution configs + StatelessSelfValidation bool // Generate execution witnesses and self-check against them (testing purpose) + EnableWitnessStats bool // Whether trie access statistics collection is enabled } // DefaultConfig returns the default config. @@ -323,7 +327,7 @@ type BlockChain struct { lastWrite uint64 // Last block when the state was flushed flushInterval atomic.Int64 // Time interval (processing time) after which to flush a state triedb *triedb.Database // The database handler for maintaining trie nodes. - statedb *state.CachingDB // State database to reuse between imports (contains state cache) + codedb *state.CodeDB // The database handler for maintaining contract codes. txIndexer *txIndexer // Transaction indexer, might be nil if not enabled hc *HeaderChain @@ -405,6 +409,7 @@ func NewBlockChain(db ethdb.Database, genesis *Genesis, engine consensus.Engine, cfg: cfg, db: db, triedb: triedb, + codedb: state.NewCodeDB(db), triegc: prque.New[int64, common.Hash](nil), chainmu: syncx.NewClosableMutex(), bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit), @@ -421,7 +426,6 @@ func NewBlockChain(db ethdb.Database, genesis *Genesis, engine consensus.Engine, return nil, err } bc.flushInterval.Store(int64(cfg.TrieTimeLimit)) - bc.statedb = state.NewDatabase(bc.triedb, nil) bc.validator = NewBlockValidator(chainConfig, bc) bc.prefetcher = newStatePrefetcher(chainConfig, bc.hc) bc.processor = NewStateProcessor(bc.hc) @@ -598,9 +602,6 @@ func (bc *BlockChain) setupSnapshot() { AsyncBuild: !bc.cfg.SnapshotWait, } bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root) - - // Re-initialize the state database with snapshot - bc.statedb = state.NewDatabase(bc.triedb, bc.snaps) } } @@ -717,8 +718,12 @@ func (bc *BlockChain) loadLastState() error { // initializeHistoryPruning sets bc.historyPrunePoint. func (bc *BlockChain) initializeHistoryPruning(latest uint64) error { - freezerTail, _ := bc.db.Tail() - + var ( + freezerTail, _ = bc.db.Tail() + genesisHash = bc.genesisBlock.Hash() + mergePoint = history.MergePrunePoints[genesisHash] + praguePoint = history.PraguePrunePoints[genesisHash] + ) switch bc.cfg.ChainHistoryMode { case history.KeepAll: if freezerTail == 0 { @@ -726,33 +731,65 @@ func (bc *BlockChain) initializeHistoryPruning(latest uint64) error { } // The database was pruned somehow, so we need to figure out if it's a known // configuration or an error. - predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] - if predefinedPoint == nil || freezerTail != predefinedPoint.BlockNumber { - log.Error("Chain history database is pruned with unknown configuration", "tail", freezerTail) - return errors.New("unexpected database tail") + if mergePoint != nil && freezerTail == mergePoint.BlockNumber { + bc.historyPrunePoint.Store(mergePoint) + return nil } - bc.historyPrunePoint.Store(predefinedPoint) - return nil + if praguePoint != nil && freezerTail == praguePoint.BlockNumber { + bc.historyPrunePoint.Store(praguePoint) + return nil + } + log.Error("Chain history database is pruned with unknown configuration", "tail", freezerTail) + return errors.New("unexpected database tail") case history.KeepPostMerge: + if mergePoint == nil { + return errors.New("history pruning requested for unknown network") + } if freezerTail == 0 && latest != 0 { - // This is the case where a user is trying to run with --history.chain - // postmerge directly on an existing DB. We could just trigger the pruning - // here, but it'd be a bit dangerous since they may not have intended this - // action to happen. So just tell them how to do it. log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cfg.ChainHistoryMode.String())) - log.Error(fmt.Sprintf("Run 'geth prune-history' to prune pre-merge history.")) + log.Error("Run 'geth prune-history --history.chain postmerge' to prune pre-merge history.") return errors.New("history pruning requested via configuration") } - predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()] - if predefinedPoint == nil { - log.Error("Chain history pruning is not supported for this network", "genesis", bc.genesisBlock.Hash()) + // Check if DB is pruned further than requested (to Prague). + if praguePoint != nil && freezerTail == praguePoint.BlockNumber { + log.Error("Chain history database is pruned to Prague block, but postmerge mode was requested.") + log.Error("History cannot be unpruned. To restore history, use 'geth import-history'.") + log.Error("If you intended to keep post-Prague history, use '--history.chain postprague' instead.") + return errors.New("database pruned beyond requested history mode") + } + if freezerTail > 0 && freezerTail != mergePoint.BlockNumber { + return errors.New("chain history database pruned to unknown block") + } + bc.historyPrunePoint.Store(mergePoint) + return nil + + case history.KeepPostPrague: + if praguePoint == nil { return errors.New("history pruning requested for unknown network") - } else if freezerTail > 0 && freezerTail != predefinedPoint.BlockNumber { + } + // Check if already at the prague prune point. + if freezerTail == praguePoint.BlockNumber { + bc.historyPrunePoint.Store(praguePoint) + return nil + } + // Check if database needs pruning. + if latest != 0 { + if freezerTail == 0 { + log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cfg.ChainHistoryMode.String())) + log.Error("Run 'geth prune-history --history.chain postprague' to prune pre-Prague history.") + return errors.New("history pruning requested via configuration") + } + if mergePoint != nil && freezerTail == mergePoint.BlockNumber { + log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is only pruned to merge block.", bc.cfg.ChainHistoryMode.String())) + log.Error("Run 'geth prune-history --history.chain postprague' to prune pre-Prague history.") + return errors.New("history pruning requested via configuration") + } log.Error("Chain history database is pruned to unknown block", "tail", freezerTail) return errors.New("unexpected database tail") } - bc.historyPrunePoint.Store(predefinedPoint) + // Fresh database (latest == 0), will sync from prague point. + bc.historyPrunePoint.Store(praguePoint) return nil default: @@ -1280,6 +1317,8 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { func (bc *BlockChain) writeHeadBlock(block *types.Block) { // Add the block to the canonical chain number scheme and mark as the head batch := bc.db.NewBatch() + defer batch.Close() + rawdb.WriteHeadHeaderHash(batch, block.Hash()) rawdb.WriteHeadFastBlockHash(batch, block.Hash()) rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64()) @@ -1660,6 +1699,8 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. batch = bc.db.NewBatch() start = time.Now() ) + defer batch.Close() + rawdb.WriteBlock(batch, block) rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) rawdb.WritePreimages(batch, statedb.Preimages()) @@ -1827,7 +1868,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { } defer bc.chainmu.Unlock() - _, n, err := bc.insertChain(chain, true, false) // No witness collection for mass inserts (would get super large) + _, n, err := bc.insertChain(context.Background(), chain, true, false) // No witness collection for mass inserts (would get super large) return n, err } @@ -1839,7 +1880,7 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // racey behaviour. If a sidechain import is in progress, and the historic state // is imported, but then new canon-head is added before the actual sidechain // completes, then the historic state could be pruned again -func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness bool) (*stateless.Witness, int, error) { +func (bc *BlockChain) insertChain(ctx context.Context, chain types.Blocks, setHead bool, makeWitness bool) (*stateless.Witness, int, error) { // If the chain is terminating, don't even bother starting up. if bc.insertStopped() { return nil, 0, nil @@ -1921,11 +1962,11 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness if setHead { // First block is pruned, insert as sidechain and reorg only if TD grows enough log.Debug("Pruned ancestor, inserting as sidechain", "number", block.Number(), "hash", block.Hash()) - return bc.insertSideChain(block, it, makeWitness) + return bc.insertSideChain(ctx, block, it, makeWitness) } else { // We're post-merge and the parent is pruned, try to recover the parent state log.Debug("Pruned ancestor", "number", block.Number(), "hash", block.Hash()) - _, err := bc.recoverAncestors(block, makeWitness) + _, err := bc.recoverAncestors(ctx, block, makeWitness) return nil, it.index, err } // Some other error(except ErrKnownBlock) occurred, abort. @@ -1997,7 +2038,15 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness } // The traced section of block import. start := time.Now() - res, err := bc.ProcessBlock(parent.Root, block, setHead, makeWitness && len(chain) == 1) + config := ExecuteConfig{ + WriteState: true, + WriteHead: setHead, + EnableTracer: true, + MakeWitness: makeWitness && len(chain) == 1, + StatelessSelfValidation: bc.cfg.StatelessSelfValidation, + EnableWitnessStats: bc.cfg.EnableWitnessStats, + } + res, err := bc.ProcessBlock(ctx, parent.Root, block, config) if err != nil { return nil, it.index, err } @@ -2080,19 +2129,47 @@ func (bpr *blockProcessingResult) Stats() *ExecuteStats { return bpr.stats } +// ExecuteConfig defines optional behaviors during execution. +type ExecuteConfig struct { + // WriteState controls whether the computed state changes are persisted to + // the underlying storage. If false, execution is performed in-memory only. + WriteState bool + + // WriteHead indicates whether the execution result should update the canonical + // chain head. It's only relevant with WriteState == True. + WriteHead bool + + // EnableTracer enables execution tracing. This is typically used for debugging + // or analysis and may significantly impact performance. + EnableTracer bool + + // MakeWitness indicates whether to generate execution witness data during + // execution. Enabling this may introduce additional memory and CPU overhead. + MakeWitness bool + + // StatelessSelfValidation indicates whether the execution witnesses generation + // and self-validation (testing purpose) is enabled. + StatelessSelfValidation bool + + // EnableWitnessStats indicates whether to enable collection of witness trie + // access statistics + EnableWitnessStats bool +} + // ProcessBlock executes and validates the given block. If there was no error // it writes the block and associated state to database. -func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, setHead bool, makeWitness bool) (result *blockProcessingResult, blockEndErr error) { +func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash, block *types.Block, config ExecuteConfig) (result *blockProcessingResult, blockEndErr error) { var ( err error startTime = time.Now() statedb *state.StateDB interrupt atomic.Bool + sdb = state.NewDatabase(bc.triedb, bc.codedb).WithSnapshot(bc.snaps) ) defer interrupt.Store(true) // terminate the prefetch at the end if bc.cfg.NoPrefetch { - statedb, err = state.New(parentRoot, bc.statedb) + statedb, err = state.New(parentRoot, sdb) if err != nil { return nil, err } @@ -2102,23 +2179,27 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s // // Note: the main processor and prefetcher share the same reader with a local // cache for mitigating the overhead of state access. - prefetch, process, err := bc.statedb.ReadersWithCacheStats(parentRoot) + prefetch, process, err := sdb.ReadersWithCacheStats(parentRoot) if err != nil { return nil, err } - throwaway, err := state.NewWithReader(parentRoot, bc.statedb, prefetch) + throwaway, err := state.NewWithReader(parentRoot, sdb, prefetch) if err != nil { return nil, err } - statedb, err = state.NewWithReader(parentRoot, bc.statedb, process) + statedb, err = state.NewWithReader(parentRoot, sdb, process) if err != nil { return nil, err } // Upload the statistics of reader at the end defer func() { if result != nil { - result.stats.StatePrefetchCacheStats = prefetch.GetStats() - result.stats.StateReadCacheStats = process.GetStats() + if stater, ok := prefetch.(state.ReaderStater); ok { + result.stats.StatePrefetchCacheStats = stater.GetStats() + } + if stater, ok := process.(state.ReaderStater); ok { + result.stats.StateReadCacheStats = stater.GetStats() + } } }() go func(start time.Time, throwaway *state.StateDB, block *types.Block) { @@ -2145,12 +2226,12 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s // Generate witnesses either if we're self-testing, or if it's the // only block being inserted. A bit crude, but witnesses are huge, // so we refuse to make an entire chain of them. - if bc.cfg.VmConfig.StatelessSelfValidation || makeWitness { + if config.StatelessSelfValidation || config.MakeWitness { witness, err = stateless.NewWitness(block.Header(), bc) if err != nil { return nil, err } - if bc.cfg.VmConfig.EnableWitnessStats { + if config.EnableWitnessStats { witnessStats = stateless.NewWitnessStats() } } @@ -2158,22 +2239,27 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s defer statedb.StopPrefetcher() } - if bc.logger != nil && bc.logger.OnBlockStart != nil { - bc.logger.OnBlockStart(tracing.BlockEvent{ - Block: block, - Finalized: bc.CurrentFinalBlock(), - Safe: bc.CurrentSafeBlock(), - }) - } - if bc.logger != nil && bc.logger.OnBlockEnd != nil { - defer func() { - bc.logger.OnBlockEnd(blockEndErr) - }() + // Instrument the blockchain tracing + if config.EnableTracer { + if bc.logger != nil && bc.logger.OnBlockStart != nil { + bc.logger.OnBlockStart(tracing.BlockEvent{ + Block: block, + Finalized: bc.CurrentFinalBlock(), + Safe: bc.CurrentSafeBlock(), + }) + } + if bc.logger != nil && bc.logger.OnBlockEnd != nil { + defer func() { + bc.logger.OnBlockEnd(blockEndErr) + }() + } } // Process block using the parent state as reference point pstart := time.Now() - res, err := bc.processor.Process(block, statedb, bc.cfg.VmConfig) + pctx, _, spanEnd := telemetry.StartSpan(ctx, "bc.processor.Process") + res, err := bc.processor.Process(pctx, block, statedb, bc.cfg.VmConfig) + spanEnd(&err) if err != nil { bc.reportBadBlock(block, res, err) return nil, err @@ -2181,7 +2267,10 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s ptime := time.Since(pstart) vstart := time.Now() - if err := bc.validator.ValidateState(block, statedb, res, false); err != nil { + _, _, spanEnd = telemetry.StartSpan(ctx, "bc.validator.ValidateState") + err = bc.validator.ValidateState(block, statedb, res, false) + spanEnd(&err) + if err != nil { bc.reportBadBlock(block, res, err) return nil, err } @@ -2193,7 +2282,7 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s // witness builder/runner, which would otherwise be impossible due to the // various invalid chain states/behaviors being contained in those tests. xvstart := time.Now() - if witness := statedb.Witness(); witness != nil && bc.cfg.VmConfig.StatelessSelfValidation { + if witness := statedb.Witness(); witness != nil && config.StatelessSelfValidation { log.Warn("Running stateless self-validation", "block", block.Number(), "hash", block.Hash()) // Remove critical computed fields from the block to force true recalculation @@ -2204,7 +2293,7 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s task := types.NewBlockWithHeader(context).WithBody(*block.Body()) // Run the stateless self-cross-validation - crossStateRoot, crossReceiptRoot, err := ExecuteStateless(bc.chainConfig, bc.cfg.VmConfig, task, witness) + crossStateRoot, crossReceiptRoot, err := ExecuteStateless(ctx, bc.chainConfig, bc.cfg.VmConfig, task, witness) if err != nil { return nil, fmt.Errorf("stateless self-validation failed: %v", err) } @@ -2246,31 +2335,28 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s stats.CrossValidation = xvtime // The time spent on stateless cross validation // Write the block to the chain and get the status. - var ( - wstart = time.Now() - status WriteStatus - ) - if !setHead { - // Don't set the head, only insert the block - err = bc.writeBlockWithState(block, res.Receipts, statedb) - } else { - status, err = bc.writeBlockAndSetHead(block, res.Receipts, res.Logs, statedb, false) - } - if err != nil { - return nil, err + var status WriteStatus + if config.WriteState { + wstart := time.Now() + if !config.WriteHead { + // Don't set the head, only insert the block + err = bc.writeBlockWithState(block, res.Receipts, statedb) + } else { + status, err = bc.writeBlockAndSetHead(block, res.Receipts, res.Logs, statedb, false) + } + if err != nil { + return nil, err + } + // Update the metrics touched during block commit + stats.AccountCommits = statedb.AccountCommits // Account commits are complete, we can mark them + stats.StorageCommits = statedb.StorageCommits // Storage commits are complete, we can mark them + stats.DatabaseCommit = statedb.DatabaseCommits // Database commits are complete, we can mark them + stats.BlockWrite = time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.DatabaseCommits } // Report the collected witness statistics if witnessStats != nil { witnessStats.ReportMetrics(block.NumberU64()) } - - // Update the metrics touched during block commit - stats.AccountCommits = statedb.AccountCommits // Account commits are complete, we can mark them - stats.StorageCommits = statedb.StorageCommits // Storage commits are complete, we can mark them - stats.SnapshotCommit = statedb.SnapshotCommits // Snapshot commits are complete, we can mark them - stats.TrieDBCommit = statedb.TrieDBCommits // Trie database commits are complete, we can mark them - stats.BlockWrite = time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.SnapshotCommits - statedb.TrieDBCommits - elapsed := time.Since(startTime) + 1 // prevent zero division stats.TotalTime = elapsed stats.MgasPerSecond = float64(res.GasUsed) * 1000 / float64(elapsed) @@ -2291,7 +2377,7 @@ func (bc *BlockChain) ProcessBlock(parentRoot common.Hash, block *types.Block, s // The method writes all (header-and-body-valid) blocks to disk, then tries to // switch over to the new chain if the TD exceeded the current chain. // insertSideChain is only used pre-merge. -func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, makeWitness bool) (*stateless.Witness, int, error) { +func (bc *BlockChain) insertSideChain(ctx context.Context, block *types.Block, it *insertIterator, makeWitness bool) (*stateless.Witness, int, error) { var current = bc.CurrentBlock() // The first sidechain block error is already verified to be ErrPrunedAncestor. @@ -2372,7 +2458,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ma // memory here. if len(blocks) >= 2048 || memory > 64*1024*1024 { log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64()) - if _, _, err := bc.insertChain(blocks, true, false); err != nil { + if _, _, err := bc.insertChain(ctx, blocks, true, false); err != nil { return nil, 0, err } blocks, memory = blocks[:0], 0 @@ -2386,7 +2472,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ma } if len(blocks) > 0 { log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64()) - return bc.insertChain(blocks, true, makeWitness) + return bc.insertChain(ctx, blocks, true, makeWitness) } return nil, 0, nil } @@ -2395,7 +2481,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ma // all the ancestor blocks since that. // recoverAncestors is only used post-merge. // We return the hash of the latest block that we could correctly validate. -func (bc *BlockChain) recoverAncestors(block *types.Block, makeWitness bool) (common.Hash, error) { +func (bc *BlockChain) recoverAncestors(ctx context.Context, block *types.Block, makeWitness bool) (common.Hash, error) { // Gather all the sidechain hashes (full blocks may be memory heavy) var ( hashes []common.Hash @@ -2435,7 +2521,7 @@ func (bc *BlockChain) recoverAncestors(block *types.Block, makeWitness bool) (co } else { b = bc.GetBlock(hashes[i], numbers[i]) } - if _, _, err := bc.insertChain(types.Blocks{b}, false, makeWitness && i == 0); err != nil { + if _, _, err := bc.insertChain(ctx, types.Blocks{b}, false, makeWitness && i == 0); err != nil { return b.ParentHash(), err } } @@ -2628,6 +2714,8 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error // Delete useless indexes right now which includes the non-canonical // transaction indexes, canonical chain indexes which above the head. batch := bc.db.NewBatch() + defer batch.Close() + for _, tx := range types.HashDifference(deletedTxs, rebirthTxs) { rawdb.DeleteTxLookupEntry(batch, tx) } @@ -2662,14 +2750,16 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error // The key difference between the InsertChain is it won't do the canonical chain // updating. It relies on the additional SetCanonical call to finalize the entire // procedure. -func (bc *BlockChain) InsertBlockWithoutSetHead(block *types.Block, makeWitness bool) (*stateless.Witness, error) { +func (bc *BlockChain) InsertBlockWithoutSetHead(ctx context.Context, block *types.Block, makeWitness bool) (witness *stateless.Witness, err error) { + _, _, spanEnd := telemetry.StartSpan(ctx, "core.blockchain.InsertBlockWithoutSetHead") + defer spanEnd(&err) if !bc.chainmu.TryLock() { return nil, errChainStopped } defer bc.chainmu.Unlock() - witness, _, err := bc.insertChain(types.Blocks{block}, false, makeWitness) - return witness, err + witness, _, err = bc.insertChain(ctx, types.Blocks{block}, false, makeWitness) + return } // SetCanonical rewinds the chain to set the new head block as the specified @@ -2683,7 +2773,7 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) { // Re-execute the reorged chain in case the head state is missing. if !bc.HasState(head.Root()) { - if latestValidHash, err := bc.recoverAncestors(head, false); err != nil { + if latestValidHash, err := bc.recoverAncestors(context.Background(), head, false); err != nil { return latestValidHash, err } log.Info("Recovered head state", "number", head.Number(), "hash", head.Hash()) diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index ee15c152c4..f1b40d0d0c 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -371,7 +371,7 @@ func (bc *BlockChain) TxIndexDone() bool { // HasState checks if state trie is fully present in the database or not. func (bc *BlockChain) HasState(hash common.Hash) bool { - _, err := bc.statedb.OpenTrie(hash) + _, err := bc.triedb.NodeReader(hash) return err == nil } @@ -403,7 +403,7 @@ func (bc *BlockChain) stateRecoverable(root common.Hash) bool { func (bc *BlockChain) ContractCodeWithPrefix(hash common.Hash) []byte { // TODO(rjl493456442) The associated account address is also required // in Verkle scheme. Fix it once snap-sync is supported for Verkle. - return bc.statedb.ContractCodeWithPrefix(common.Address{}, hash) + return bc.codedb.Reader().CodeWithPrefix(common.Address{}, hash) } // State returns a new mutable state based on the current HEAD block. @@ -413,14 +413,14 @@ func (bc *BlockChain) State() (*state.StateDB, error) { // StateAt returns a new mutable state based on a particular point in time. func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { - return state.New(root, bc.statedb) + return state.New(root, state.NewDatabase(bc.triedb, bc.codedb).WithSnapshot(bc.snaps)) } // HistoricState returns a historic state specified by the given root. // Live states are not available and won't be served, please use `State` // or `StateAt` instead. func (bc *BlockChain) HistoricState(root common.Hash) (*state.StateDB, error) { - return state.New(root, state.NewHistoricDatabase(bc.db, bc.triedb)) + return state.New(root, state.NewHistoricDatabase(bc.triedb, bc.codedb)) } // Config retrieves the chain's fork configuration. @@ -444,11 +444,6 @@ func (bc *BlockChain) Processor() Processor { return bc.processor } -// StateCache returns the caching database underpinning the blockchain instance. -func (bc *BlockChain) StateCache() state.Database { - return bc.statedb -} - // GasLimit returns the gas limit of the current HEAD block. func (bc *BlockChain) GasLimit() uint64 { return bc.CurrentBlock().GasLimit @@ -492,6 +487,11 @@ func (bc *BlockChain) TrieDB() *triedb.Database { return bc.triedb } +// CodeDB retrieves the low level contract code database used for data storage. +func (bc *BlockChain) CodeDB() *state.CodeDB { + return bc.codedb +} + // HeaderChain returns the underlying header chain. func (bc *BlockChain) HeaderChain() *HeaderChain { return bc.hc diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 72ca15d7f6..f2fbc003f1 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb/pebble" "github.com/ethereum/go-ethereum/params" @@ -2041,7 +2040,6 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme dbconfig.HashDB = hashdb.Defaults } chain.triedb = triedb.NewDatabase(chain.db, dbconfig) - chain.statedb = state.NewDatabase(chain.triedb, chain.snaps) // Force run a freeze cycle type freezer interface { diff --git a/core/blockchain_stats.go b/core/blockchain_stats.go index adc66266c4..d753b0b700 100644 --- a/core/blockchain_stats.go +++ b/core/blockchain_stats.go @@ -52,8 +52,7 @@ type ExecuteStats struct { Execution time.Duration // Time spent on the EVM execution Validation time.Duration // Time spent on the block validation CrossValidation time.Duration // Optional, time spent on the block cross validation - SnapshotCommit time.Duration // Time spent on snapshot commit - TrieDBCommit time.Duration // Time spent on database commit + DatabaseCommit time.Duration // Time spent on database commit BlockWrite time.Duration // Time spent on block write TotalTime time.Duration // The total time spent on block execution MgasPerSecond float64 // The million gas processed per second @@ -87,22 +86,21 @@ func (s *ExecuteStats) reportMetrics() { blockExecutionTimer.Update(s.Execution) // The time spent on EVM processing blockValidationTimer.Update(s.Validation) // The time spent on block validation blockCrossValidationTimer.Update(s.CrossValidation) // The time spent on stateless cross validation - snapshotCommitTimer.Update(s.SnapshotCommit) // Snapshot commits are complete, we can mark them - triedbCommitTimer.Update(s.TrieDBCommit) // Trie database commits are complete, we can mark them + triedbCommitTimer.Update(s.DatabaseCommit) // Trie database commits are complete, we can mark them blockWriteTimer.Update(s.BlockWrite) // The time spent on block write blockInsertTimer.Update(s.TotalTime) // The total time spent on block execution chainMgaspsMeter.Update(time.Duration(s.MgasPerSecond)) // TODO(rjl493456442) generalize the ResettingTimer // Cache hit rates - accountCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.AccountCacheHit) - accountCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.AccountCacheMiss) - storageCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.StorageCacheHit) - storageCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.StorageCacheMiss) + accountCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.AccountCacheHit) + accountCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.AccountCacheMiss) + storageCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.StorageCacheHit) + storageCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.StorageCacheMiss) - accountCacheHitMeter.Mark(s.StateReadCacheStats.AccountCacheHit) - accountCacheMissMeter.Mark(s.StateReadCacheStats.AccountCacheMiss) - storageCacheHitMeter.Mark(s.StateReadCacheStats.StorageCacheHit) - storageCacheMissMeter.Mark(s.StateReadCacheStats.StorageCacheMiss) + accountCacheHitMeter.Mark(s.StateReadCacheStats.StateStats.AccountCacheHit) + accountCacheMissMeter.Mark(s.StateReadCacheStats.StateStats.AccountCacheMiss) + storageCacheHitMeter.Mark(s.StateReadCacheStats.StateStats.StorageCacheHit) + storageCacheMissMeter.Mark(s.StateReadCacheStats.StateStats.StorageCacheMiss) } // slowBlockLog represents the JSON structure for slow block logging. @@ -177,14 +175,6 @@ type slowBlockCodeCacheEntry struct { MissBytes int64 `json:"miss_bytes"` } -// calculateHitRate computes the cache hit rate as a percentage (0-100). -func calculateHitRate(hits, misses int64) float64 { - if total := hits + misses; total > 0 { - return float64(hits) / float64(total) * 100.0 - } - return 0.0 -} - // durationToMs converts a time.Duration to milliseconds as a float64 // with sub-millisecond precision for accurate cross-client metrics. func durationToMs(d time.Duration) float64 { @@ -216,7 +206,7 @@ func (s *ExecuteStats) logSlow(block *types.Block, slowBlockThreshold time.Durat ExecutionMs: durationToMs(s.Execution), StateReadMs: durationToMs(s.AccountReads + s.StorageReads + s.CodeReads), StateHashMs: durationToMs(s.AccountHashes + s.AccountUpdates + s.StorageUpdates), - CommitMs: durationToMs(max(s.AccountCommits, s.StorageCommits) + s.TrieDBCommit + s.SnapshotCommit + s.BlockWrite), + CommitMs: durationToMs(max(s.AccountCommits, s.StorageCommits) + s.DatabaseCommit + s.BlockWrite), TotalMs: durationToMs(s.TotalTime), }, Throughput: slowBlockThru{ @@ -238,19 +228,19 @@ func (s *ExecuteStats) logSlow(block *types.Block, slowBlockThreshold time.Durat }, Cache: slowBlockCache{ Account: slowBlockCacheEntry{ - Hits: s.StateReadCacheStats.AccountCacheHit, - Misses: s.StateReadCacheStats.AccountCacheMiss, - HitRate: calculateHitRate(s.StateReadCacheStats.AccountCacheHit, s.StateReadCacheStats.AccountCacheMiss), + Hits: s.StateReadCacheStats.StateStats.AccountCacheHit, + Misses: s.StateReadCacheStats.StateStats.AccountCacheMiss, + HitRate: s.StateReadCacheStats.StateStats.AccountCacheHitRate(), }, Storage: slowBlockCacheEntry{ - Hits: s.StateReadCacheStats.StorageCacheHit, - Misses: s.StateReadCacheStats.StorageCacheMiss, - HitRate: calculateHitRate(s.StateReadCacheStats.StorageCacheHit, s.StateReadCacheStats.StorageCacheMiss), + Hits: s.StateReadCacheStats.StateStats.StorageCacheHit, + Misses: s.StateReadCacheStats.StateStats.StorageCacheMiss, + HitRate: s.StateReadCacheStats.StateStats.StorageCacheHitRate(), }, Code: slowBlockCodeCacheEntry{ Hits: s.StateReadCacheStats.CodeStats.CacheHit, Misses: s.StateReadCacheStats.CodeStats.CacheMiss, - HitRate: calculateHitRate(s.StateReadCacheStats.CodeStats.CacheHit, s.StateReadCacheStats.CodeStats.CacheMiss), + HitRate: s.StateReadCacheStats.CodeStats.HitRate(), HitBytes: s.StateReadCacheStats.CodeStats.CacheHitBytes, MissBytes: s.StateReadCacheStats.CodeStats.CacheMissBytes, }, diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 73ffce93fb..ce592f0267 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -18,6 +18,7 @@ package core import ( "bytes" + "context" "errors" "fmt" gomath "math" @@ -156,11 +157,11 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { } return err } - statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), blockchain.statedb) + statedb, err := state.New(blockchain.GetBlockByHash(block.ParentHash()).Root(), state.NewDatabase(blockchain.triedb, blockchain.codedb)) if err != nil { return err } - res, err := blockchain.processor.Process(block, statedb, vm.Config{}) + res, err := blockchain.processor.Process(context.Background(), block, statedb, vm.Config{}) if err != nil { blockchain.reportBadBlock(block, res, err) return err @@ -3456,7 +3457,7 @@ func testSetCanonical(t *testing.T, scheme string) { gen.AddTx(tx) }) for _, block := range side { - _, err := chain.InsertBlockWithoutSetHead(block, false) + _, err := chain.InsertBlockWithoutSetHead(context.Background(), block, false) if err != nil { t.Fatalf("Failed to insert into chain: %v", err) } diff --git a/core/chain_makers.go b/core/chain_makers.go index 7ce86b14e9..e4b5cf964f 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -63,7 +63,7 @@ func (b *BlockGen) SetCoinbase(addr common.Address) { panic("coinbase can only be set once") } b.header.Coinbase = addr - b.gasPool = new(GasPool).AddGas(b.header.GasLimit) + b.gasPool = NewGasPool(b.header.GasLimit) } // SetExtra sets the extra data field of the generated block. @@ -117,10 +117,12 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti evm = vm.NewEVM(blockContext, b.statedb, b.cm.config, vmConfig) ) b.statedb.SetTxContext(tx.Hash(), len(b.txs)) - receipt, err := ApplyTransaction(evm, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed) + receipt, err := ApplyTransaction(evm, b.gasPool, b.statedb, b.header, tx) if err != nil { panic(err) } + b.header.GasUsed = b.gasPool.Used() + // Merge the tx-local access event into the "block-local" one, in order to collect // all values, so that the witness can be built. if b.statedb.Database().TrieDB().IsVerkle() { @@ -479,13 +481,14 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, if genesis.Config != nil && genesis.Config.IsVerkle(genesis.Config.ChainID, 0) { triedbConfig = triedb.VerkleDefaults } - triedb := triedb.NewDatabase(db, triedbConfig) - defer triedb.Close() - _, err := genesis.Commit(db, triedb, nil) + genesisTriedb := triedb.NewDatabase(db, triedbConfig) + block, err := genesis.Commit(db, genesisTriedb, nil) if err != nil { + genesisTriedb.Close() panic(err) } - blocks, receipts := GenerateChain(genesis.Config, genesis.ToBlock(), engine, db, n, gen) + genesisTriedb.Close() + blocks, receipts := GenerateChain(genesis.Config, block, engine, db, n, gen) return db, blocks, receipts } diff --git a/core/error.go b/core/error.go index 635d802863..4610842cee 100644 --- a/core/error.go +++ b/core/error.go @@ -58,6 +58,10 @@ var ( // by a transaction is higher than what's left in the block. ErrGasLimitReached = errors.New("gas limit reached") + // ErrGasLimitOverflow is returned by the gas pool if the remaining gas + // exceeds the maximum value of uint64. + ErrGasLimitOverflow = errors.New("gas limit overflow") + // ErrInsufficientFundsForTransfer is returned if the transaction sender doesn't // have enough funds for transfer(topmost call only). ErrInsufficientFundsForTransfer = errors.New("insufficient funds for transfer") diff --git a/core/evm.go b/core/evm.go index 2e7e138b59..7430c0e21f 100644 --- a/core/evm.go +++ b/core/evm.go @@ -44,6 +44,7 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common baseFee *big.Int blobBaseFee *big.Int random *common.Hash + slotNum uint64 ) // If we don't have an explicit author (i.e. not mining), extract from the header @@ -61,6 +62,10 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common if header.Difficulty.Sign() == 0 { random = &header.MixDigest } + if header.SlotNumber != nil { + slotNum = *header.SlotNumber + } + return vm.BlockContext{ CanTransfer: CanTransfer, Transfer: Transfer, @@ -73,6 +78,7 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common BlobBaseFee: blobBaseFee, GasLimit: header.GasLimit, Random: random, + SlotNum: slotNum, } } diff --git a/core/gaspool.go b/core/gaspool.go index 767222674f..14f5abd93c 100644 --- a/core/gaspool.go +++ b/core/gaspool.go @@ -21,39 +21,87 @@ import ( "math" ) -// GasPool tracks the amount of gas available during execution of the transactions -// in a block. The zero value is a pool with zero gas available. -type GasPool uint64 +// GasPool tracks the amount of gas available for transaction execution +// within a block, along with the cumulative gas consumed. +type GasPool struct { + remaining uint64 + initial uint64 + cumulativeUsed uint64 +} -// AddGas makes gas available for execution. -func (gp *GasPool) AddGas(amount uint64) *GasPool { - if uint64(*gp) > math.MaxUint64-amount { - panic("gas pool pushed above uint64") +// NewGasPool initializes the gasPool with the given amount. +func NewGasPool(amount uint64) *GasPool { + return &GasPool{ + remaining: amount, + initial: amount, } - *(*uint64)(gp) += amount - return gp } // SubGas deducts the given amount from the pool if enough gas is // available and returns an error otherwise. func (gp *GasPool) SubGas(amount uint64) error { - if uint64(*gp) < amount { + if gp.remaining < amount { return ErrGasLimitReached } - *(*uint64)(gp) -= amount + gp.remaining -= amount + return nil +} + +// ReturnGas adds the refunded gas back to the pool and updates +// the cumulative gas usage accordingly. +func (gp *GasPool) ReturnGas(returned uint64, gasUsed uint64) error { + if gp.remaining > math.MaxUint64-returned { + return fmt.Errorf("%w: remaining: %d, returned: %d", ErrGasLimitOverflow, gp.remaining, returned) + } + // The returned gas calculation differs across forks. + // + // - Pre-Amsterdam: + // returned = purchased - remaining (refund included) + // + // - Post-Amsterdam: + // returned = purchased - gasUsed (refund excluded) + gp.remaining += returned + + // gasUsed = max(txGasUsed - gasRefund, calldataFloorGasCost) + // regardless of Amsterdam is activated or not. + gp.cumulativeUsed += gasUsed return nil } // Gas returns the amount of gas remaining in the pool. func (gp *GasPool) Gas() uint64 { - return uint64(*gp) + return gp.remaining } -// SetGas sets the amount of gas with the provided number. -func (gp *GasPool) SetGas(gas uint64) { - *(*uint64)(gp) = gas +// CumulativeUsed returns the amount of cumulative consumed gas (refunded included). +func (gp *GasPool) CumulativeUsed() uint64 { + return gp.cumulativeUsed +} + +// Used returns the amount of consumed gas. +func (gp *GasPool) Used() uint64 { + if gp.initial < gp.remaining { + panic("gas used underflow") + } + return gp.initial - gp.remaining +} + +// Snapshot returns the deep-copied object as the snapshot. +func (gp *GasPool) Snapshot() *GasPool { + return &GasPool{ + initial: gp.initial, + remaining: gp.remaining, + cumulativeUsed: gp.cumulativeUsed, + } +} + +// Set sets the content of gasPool with the provided one. +func (gp *GasPool) Set(other *GasPool) { + gp.initial = other.initial + gp.remaining = other.remaining + gp.cumulativeUsed = other.cumulativeUsed } func (gp *GasPool) String() string { - return fmt.Sprintf("%d", *gp) + return fmt.Sprintf("initial: %d, remaining: %d, cumulative used: %d", gp.initial, gp.remaining, gp.cumulativeUsed) } diff --git a/core/gen_genesis.go b/core/gen_genesis.go index 2028f98edc..b94825b185 100644 --- a/core/gen_genesis.go +++ b/core/gen_genesis.go @@ -34,6 +34,7 @@ func (g Genesis) MarshalJSON() ([]byte, error) { BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas"` BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed"` + SlotNumber *uint64 `json:"slotNumber"` } var enc Genesis enc.Config = g.Config @@ -56,6 +57,7 @@ func (g Genesis) MarshalJSON() ([]byte, error) { enc.BaseFee = (*math.HexOrDecimal256)(g.BaseFee) enc.ExcessBlobGas = (*math.HexOrDecimal64)(g.ExcessBlobGas) enc.BlobGasUsed = (*math.HexOrDecimal64)(g.BlobGasUsed) + enc.SlotNumber = g.SlotNumber return json.Marshal(&enc) } @@ -77,6 +79,7 @@ func (g *Genesis) UnmarshalJSON(input []byte) error { BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` ExcessBlobGas *math.HexOrDecimal64 `json:"excessBlobGas"` BlobGasUsed *math.HexOrDecimal64 `json:"blobGasUsed"` + SlotNumber *uint64 `json:"slotNumber"` } var dec Genesis if err := json.Unmarshal(input, &dec); err != nil { @@ -133,5 +136,8 @@ func (g *Genesis) UnmarshalJSON(input []byte) error { if dec.BlobGasUsed != nil { g.BlobGasUsed = (*uint64)(dec.BlobGasUsed) } + if dec.SlotNumber != nil { + g.SlotNumber = dec.SlotNumber + } return nil } diff --git a/core/genesis.go b/core/genesis.go index 983ad4c3cb..6edc6e6779 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -73,6 +73,7 @@ type Genesis struct { BaseFee *big.Int `json:"baseFeePerGas"` // EIP-1559 ExcessBlobGas *uint64 `json:"excessBlobGas"` // EIP-4844 BlobGasUsed *uint64 `json:"blobGasUsed"` // EIP-4844 + SlotNumber *uint64 `json:"slotNumber"` // EIP-7843 } // copy copies the genesis. @@ -122,6 +123,7 @@ func ReadGenesis(db ethdb.Database) (*Genesis, error) { genesis.BaseFee = genesisHeader.BaseFee genesis.ExcessBlobGas = genesisHeader.ExcessBlobGas genesis.BlobGasUsed = genesisHeader.BlobGasUsed + genesis.SlotNumber = genesisHeader.SlotNumber return &genesis, nil } @@ -547,6 +549,12 @@ func (g *Genesis) toBlockWithRoot(root common.Hash) *types.Block { if conf.IsPrague(num, g.Timestamp) { head.RequestsHash = &types.EmptyRequestsHash } + if conf.IsAmsterdam(num, g.Timestamp) { + head.SlotNumber = g.SlotNumber + if head.SlotNumber == nil { + head.SlotNumber = new(uint64) + } + } } return types.NewBlock(head, &types.Body{Withdrawals: withdrawals}, nil, trie.NewStackTrie(nil)) } diff --git a/core/genesis_test.go b/core/genesis_test.go index ba00581b06..2b08b36690 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -308,7 +308,7 @@ func TestVerkleGenesisCommit(t *testing.T) { }, } - expected := common.FromHex("b94812c1674dcf4f2bc98f4503d15f4cc674265135bcf3be6e4417b60881042a") + expected := common.FromHex("1fd154971d9a386c4ec75fe7138c17efb569bfc2962e46e94a376ba997e3fadc") got := genesis.ToBlock().Root().Bytes() if !bytes.Equal(got, expected) { t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) diff --git a/core/history/historymode.go b/core/history/historymode.go index e735222d37..bdaf07826d 100644 --- a/core/history/historymode.go +++ b/core/history/historymode.go @@ -32,10 +32,13 @@ const ( // KeepPostMerge sets the history pruning point to the merge activation block. KeepPostMerge + + // KeepPostPrague sets the history pruning point to the Prague (Pectra) activation block. + KeepPostPrague ) func (m HistoryMode) IsValid() bool { - return m <= KeepPostMerge + return m <= KeepPostPrague } func (m HistoryMode) String() string { @@ -44,6 +47,8 @@ func (m HistoryMode) String() string { return "all" case KeepPostMerge: return "postmerge" + case KeepPostPrague: + return "postprague" default: return fmt.Sprintf("invalid HistoryMode(%d)", m) } @@ -64,8 +69,10 @@ func (m *HistoryMode) UnmarshalText(text []byte) error { *m = KeepAll case "postmerge": *m = KeepPostMerge + case "postprague": + *m = KeepPostPrague default: - return fmt.Errorf(`unknown sync mode %q, want "all" or "postmerge"`, text) + return fmt.Errorf(`unknown history mode %q, want "all", "postmerge", or "postprague"`, text) } return nil } @@ -75,10 +82,10 @@ type PrunePoint struct { BlockHash common.Hash } -// PrunePoints the pre-defined history pruning cutoff blocks for known networks. +// MergePrunePoints contains the pre-defined history pruning cutoff blocks for known networks. // They point to the first post-merge block. Any pruning should truncate *up to* but excluding -// given block. -var PrunePoints = map[common.Hash]*PrunePoint{ +// the given block. +var MergePrunePoints = map[common.Hash]*PrunePoint{ // mainnet params.MainnetGenesisHash: { BlockNumber: 15537393, @@ -91,6 +98,39 @@ var PrunePoints = map[common.Hash]*PrunePoint{ }, } +// PraguePrunePoints contains the pre-defined history pruning cutoff blocks for the Prague +// (Pectra) upgrade. They point to the first post-Prague block. Any pruning should truncate +// *up to* but excluding the given block. +var PraguePrunePoints = map[common.Hash]*PrunePoint{ + // mainnet - first Prague block (May 7, 2025) + params.MainnetGenesisHash: { + BlockNumber: 22431084, + BlockHash: common.HexToHash("0x50c8cab760b2948349c590461b166773c45d8f4858cccf5a43025ab2960152e8"), + }, + // sepolia - first Prague block (March 5, 2025) + params.SepoliaGenesisHash: { + BlockNumber: 7836331, + BlockHash: common.HexToHash("0xe6571beb68bf24dbd8a6ba354518996920c55a3f8d8fdca423e391b8ad071f22"), + }, +} + +// PrunePoints is an alias for MergePrunePoints for backward compatibility. +// Deprecated: Use GetPrunePoint or MergePrunePoints directly. +var PrunePoints = MergePrunePoints + +// GetPrunePoint returns the prune point for the given genesis hash and history mode. +// Returns nil if no prune point is defined for the given combination. +func GetPrunePoint(genesisHash common.Hash, mode HistoryMode) *PrunePoint { + switch mode { + case KeepPostMerge: + return MergePrunePoints[genesisHash] + case KeepPostPrague: + return PraguePrunePoints[genesisHash] + default: + return nil + } +} + // PrunedHistoryError is returned by APIs when the requested history is pruned. type PrunedHistoryError struct{} diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 14308dd698..6ae64fb2fd 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -424,13 +424,7 @@ func WriteBodyRLP(db ethdb.KeyValueWriter, hash common.Hash, number uint64, rlp // HasBody verifies the existence of a block body corresponding to the hash. func HasBody(db ethdb.Reader, hash common.Hash, number uint64) bool { if isCanon(db, number, hash) { - // Block is in ancient store, but bodies can be pruned. - // Check if the block number is above the pruning tail. - tail, _ := db.Tail() - if number >= tail { - return true - } - return false + return true } if has, err := db.Has(blockBodyKey(number, hash)); !has || err != nil { return false @@ -472,13 +466,7 @@ func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { // to a block. func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool { if isCanon(db, number, hash) { - // Block is in ancient store, but receipts can be pruned. - // Check if the block number is above the pruning tail. - tail, _ := db.Tail() - if number >= tail { - return true - } - return false + return true } if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil { return false diff --git a/core/rawdb/ancienttest/testsuite.go b/core/rawdb/ancienttest/testsuite.go index 7512c1f44b..eb66645a3a 100644 --- a/core/rawdb/ancienttest/testsuite.go +++ b/core/rawdb/ancienttest/testsuite.go @@ -260,6 +260,46 @@ func basicWrite(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) { if err != nil { t.Fatalf("Failed to write ancient data %v", err) } + + // Write should work after truncating from tail but over the head + db.TruncateTail(200) + head, err := db.Ancients() + if err != nil { + t.Fatalf("Failed to retrieve head ancients %v", err) + } + tail, err := db.Tail() + if err != nil { + t.Fatalf("Failed to retrieve tail ancients %v", err) + } + if head != 200 || tail != 200 { + t.Fatalf("Ancient head and tail are not expected") + } + _, err = db.ModifyAncients(func(op ethdb.AncientWriteOp) error { + offset := uint64(200) + for i := 0; i < 100; i++ { + if err := op.AppendRaw("a", offset+uint64(i), dataA[i]); err != nil { + return err + } + if err := op.AppendRaw("b", offset+uint64(i), dataB[i]); err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatalf("Failed to write ancient data %v", err) + } + head, err = db.Ancients() + if err != nil { + t.Fatalf("Failed to retrieve head ancients %v", err) + } + tail, err = db.Tail() + if err != nil { + t.Fatalf("Failed to retrieve tail ancients %v", err) + } + if head != 300 || tail != 200 { + t.Fatalf("Ancient head and tail are not expected") + } } func nonMutable(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) { diff --git a/core/rawdb/database.go b/core/rawdb/database.go index a5335ea56b..576e32b961 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/internal/tablewriter" "github.com/ethereum/go-ethereum/log" "golang.org/x/sync/errgroup" ) @@ -663,7 +664,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { total.Add(uint64(ancient.size())) } - table := NewTableWriter(os.Stdout) + table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Database", "Category", "Size", "Items"}) table.SetFooter([]string{"", "Total", common.StorageSize(total.Load()).String(), fmt.Sprintf("%d", count.Load())}) table.AppendBulk(stats) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 42cd2a7999..0e2f86d6ed 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -59,7 +59,7 @@ const freezerTableSize = 2 * 1000 * 1000 * 1000 // - The in-order data ensures that disk reads are always optimized. type Freezer struct { datadir string - frozen atomic.Uint64 // Number of items already frozen + head atomic.Uint64 // Number of items stored (including items removed from tail) tail atomic.Uint64 // Number of the first stored item in the freezer // This lock synchronizes writers and the truncate operation, as well as @@ -97,12 +97,12 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui return nil, errSymlinkDatadir } } + // Leveldb/Pebble uses LOCK as the filelock filename. To prevent the + // name collision, we use FLOCK as the lock name. flockFile := filepath.Join(datadir, "FLOCK") if err := os.MkdirAll(filepath.Dir(flockFile), 0755); err != nil { return nil, err } - // Leveldb uses LOCK as the filelock filename. To prevent the - // name collision, we use FLOCK as the lock name. lock := flock.New(flockFile) tryLock := lock.TryLock if readonly { @@ -213,7 +213,7 @@ func (f *Freezer) AncientBytes(kind string, id, offset, length uint64) ([]byte, // Ancients returns the length of the frozen items. func (f *Freezer) Ancients() (uint64, error) { - return f.frozen.Load(), nil + return f.head.Load(), nil } // Tail returns the number of first stored item in the freezer. @@ -252,7 +252,7 @@ func (f *Freezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize defer f.writeLock.Unlock() // Roll back all tables to the starting position in case of error. - prevItem := f.frozen.Load() + prevItem := f.head.Load() defer func() { if err != nil { // The write operation has failed. Go back to the previous item position. @@ -273,7 +273,7 @@ func (f *Freezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize if err != nil { return 0, err } - f.frozen.Store(item) + f.head.Store(item) return writeSize, nil } @@ -286,7 +286,7 @@ func (f *Freezer) TruncateHead(items uint64) (uint64, error) { f.writeLock.Lock() defer f.writeLock.Unlock() - oitems := f.frozen.Load() + oitems := f.head.Load() if oitems <= items { return oitems, nil } @@ -295,7 +295,7 @@ func (f *Freezer) TruncateHead(items uint64) (uint64, error) { return 0, err } } - f.frozen.Store(items) + f.head.Store(items) return oitems, nil } @@ -320,6 +320,11 @@ func (f *Freezer) TruncateTail(tail uint64) (uint64, error) { } } f.tail.Store(tail) + + // Update the head if the requested tail exceeds the current head + if f.head.Load() < tail { + f.head.Store(tail) + } return old, nil } @@ -379,7 +384,7 @@ func (f *Freezer) validate() error { prunedTail = &tmp } - f.frozen.Store(head) + f.head.Store(head) f.tail.Store(*prunedTail) return nil } @@ -414,7 +419,7 @@ func (f *Freezer) repair() error { } } - f.frozen.Store(head) + f.head.Store(head) f.tail.Store(prunedTail) return nil } diff --git a/core/rawdb/freezer_memory.go b/core/rawdb/freezer_memory.go index a0d308f896..ec6d4b22e2 100644 --- a/core/rawdb/freezer_memory.go +++ b/core/rawdb/freezer_memory.go @@ -113,7 +113,7 @@ func (t *memoryTable) truncateTail(items uint64) error { return nil } if t.items < items { - return errors.New("truncation above head") + return t.reset(items) } for i := uint64(0); i < items-t.offset; i++ { if t.size > uint64(len(t.data[i])) { @@ -127,6 +127,16 @@ func (t *memoryTable) truncateTail(items uint64) error { return nil } +// reset clears the entire table and sets both the head and tail to the given +// value. It assumes the caller holds the lock and that tail > t.items. +func (t *memoryTable) reset(offset uint64) error { + t.size = 0 + t.data = nil + t.items = offset + t.offset = offset + return nil +} + // commit merges the given item batch into table. It's presumed that the // batch is ordered and continuous with table. func (t *memoryTable) commit(batch [][]byte) error { @@ -387,6 +397,9 @@ func (f *MemoryFreezer) TruncateTail(tail uint64) (uint64, error) { } } f.tail = tail + if f.items < tail { + f.items = tail + } return old, nil } diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index aedb2d8eed..280f6e1aaa 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -707,12 +707,13 @@ func (t *freezerTable) truncateTail(items uint64) error { t.lock.Lock() defer t.lock.Unlock() - // Ensure the given truncate target falls in the correct range + // Short-circuit if the requested tail deletion points to a stale position if t.itemHidden.Load() >= items { return nil } + // If the requested tail exceeds the current head, reset the entire table if t.items.Load() < items { - return errors.New("truncation above head") + return t.resetTo(items) } // Load the new tail index by the given new tail position var ( @@ -822,10 +823,9 @@ func (t *freezerTable) truncateTail(items uint64) error { shorten := indexEntrySize * int64(newDeleted-deleted) if t.metadata.flushOffset <= shorten { return fmt.Errorf("invalid index flush offset: %d, shorten: %d", t.metadata.flushOffset, shorten) - } else { - if err := t.metadata.setFlushOffset(t.metadata.flushOffset-shorten, true); err != nil { - return err - } + } + if err := t.metadata.setFlushOffset(t.metadata.flushOffset-shorten, true); err != nil { + return err } // Retrieve the new size and update the total size counter newSize, err := t.sizeNolock() @@ -836,6 +836,59 @@ func (t *freezerTable) truncateTail(items uint64) error { return nil } +// resetTo clears the entire table and sets both the head and tail to the given +// value. It assumes the caller holds the lock and that tail > t.items. +func (t *freezerTable) resetTo(tail uint64) error { + // Sync the entire table before resetting, eliminating the potential + // data corruption. + err := t.doSync() + if err != nil { + return err + } + // Update the index file to reflect the new offset + if err := t.index.Close(); err != nil { + return err + } + entry := &indexEntry{ + filenum: t.headId + 1, + offset: uint32(tail), + } + if err := reset(t.index.Name(), entry.append(nil)); err != nil { + return err + } + if err := t.metadata.setVirtualTail(tail, true); err != nil { + return err + } + if err := t.metadata.setFlushOffset(indexEntrySize, true); err != nil { + return err + } + t.index, err = openFreezerFileForAppend(t.index.Name()) + if err != nil { + return err + } + + // Purge all the existing data file + if err := t.head.Close(); err != nil { + return err + } + t.headId = t.headId + 1 + t.tailId = t.headId + t.headBytes = 0 + + t.head, err = t.openFile(t.headId, openFreezerFileTruncated) + if err != nil { + return err + } + t.releaseFilesBefore(t.headId, true) + + t.items.Store(tail) + t.itemOffset.Store(tail) + t.itemHidden.Store(tail) + t.sizeGauge.Update(0) + + return nil +} + // Close closes all opened files and finalizes the freezer table for use. // This operation must be completed before shutdown to prevent the loss of // recent writes. @@ -1247,25 +1300,20 @@ func (t *freezerTable) doSync() error { if t.index == nil || t.head == nil || t.metadata.file == nil { return errClosed } - var err error - trackError := func(e error) { - if e != nil && err == nil { - err = e - } + if err := t.index.Sync(); err != nil { + return err + } + if err := t.head.Sync(); err != nil { + return err } - trackError(t.index.Sync()) - trackError(t.head.Sync()) - // A crash may occur before the offset is updated, leaving the offset - // points to a old position. If so, the extra items above the offset + // points to an old position. If so, the extra items above the offset // will be truncated during the next run. stat, err := t.index.Stat() if err != nil { return err } - offset := stat.Size() - trackError(t.metadata.setFlushOffset(offset, true)) - return err + return t.metadata.setFlushOffset(stat.Size(), true) } func (t *freezerTable) dumpIndexStdout(start, stop int64) { diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index fc21ea6c63..3393f88e1a 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -1139,6 +1139,7 @@ const ( opTruncateHeadAll opTruncateTail opTruncateTailAll + opTruncateTailOverHead opCheckAll opMax // boundary value, not an actual op ) @@ -1226,6 +1227,11 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { step.target = deleted + uint64(len(items)) items = items[:0] deleted = step.target + case opTruncateTailOverHead: + newDeleted := deleted + uint64(len(items)) + 10 + step.target = newDeleted + deleted = newDeleted + items = items[:0] } steps = append(steps, step) } @@ -1268,7 +1274,7 @@ func runRandTest(rt randTest) bool { for i := 0; i < len(step.items); i++ { batch.AppendRaw(step.items[i], step.blobs[i]) } - batch.commit() + rt[i].err = batch.commit() values = append(values, step.blobs...) case opRetrieve: @@ -1290,24 +1296,28 @@ func runRandTest(rt randTest) bool { } case opTruncateHead: - f.truncateHead(step.target) + rt[i].err = f.truncateHead(step.target) length := f.items.Load() - f.itemHidden.Load() values = values[:length] case opTruncateHeadAll: - f.truncateHead(step.target) + rt[i].err = f.truncateHead(step.target) values = nil case opTruncateTail: prev := f.itemHidden.Load() - f.truncateTail(step.target) + rt[i].err = f.truncateTail(step.target) truncated := f.itemHidden.Load() - prev values = values[truncated:] case opTruncateTailAll: - f.truncateTail(step.target) + rt[i].err = f.truncateTail(step.target) + values = nil + + case opTruncateTailOverHead: + rt[i].err = f.truncateTail(step.target) values = nil } // Abort the test on error. @@ -1633,3 +1643,43 @@ func TestFreezerAncientBytes(t *testing.T) { }) } } + +func TestTruncateOverHead(t *testing.T) { + t.Parallel() + + fn := fmt.Sprintf("t-%d", rand.Uint64()) + f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 100, freezerTableConfig{noSnappy: true}, false) + if err != nil { + t.Fatal(err) + } + + // Tail truncation on an empty table + if err := f.truncateTail(10); err != nil { + t.Fatal(err) + } + batch := f.newBatch() + data := getChunk(10, 1) + require.NoError(t, batch.AppendRaw(uint64(10), data)) + require.NoError(t, batch.commit()) + + got, err := f.RetrieveItems(uint64(10), 1, 0) + require.NoError(t, err) + if !bytes.Equal(got[0], data) { + t.Fatalf("Unexpected bytes, want: %v, got: %v", data, got[0]) + } + + // Tail truncation on the non-empty table + if err := f.truncateTail(20); err != nil { + t.Fatal(err) + } + batch = f.newBatch() + data = getChunk(10, 1) + require.NoError(t, batch.AppendRaw(uint64(20), data)) + require.NoError(t, batch.commit()) + + got, err = f.RetrieveItems(uint64(20), 1, 0) + require.NoError(t, err) + if !bytes.Equal(got[0], data) { + t.Fatalf("Unexpected bytes, want: %v, got: %v", data, got[0]) + } +} diff --git a/core/rawdb/freezer_utils.go b/core/rawdb/freezer_utils.go index 752e95ba6a..7786b7a990 100644 --- a/core/rawdb/freezer_utils.go +++ b/core/rawdb/freezer_utils.go @@ -22,6 +22,19 @@ import ( "path/filepath" ) +func atomicRename(src, dest string) error { + if err := os.Rename(src, dest); err != nil { + return err + } + dir, err := os.Open(filepath.Dir(src)) + if err != nil { + return err + } + defer dir.Close() + + return dir.Sync() +} + // copyFrom copies data from 'srcPath' at offset 'offset' into 'destPath'. // The 'destPath' is created if it doesn't exist, otherwise it is overwritten. // Before the copy is executed, there is a callback can be registered to @@ -73,13 +86,48 @@ func copyFrom(srcPath, destPath string, offset uint64, before func(f *os.File) e return err } f = nil - return os.Rename(fname, destPath) + + return atomicRename(fname, destPath) +} + +// reset atomically replaces the file at the given path with the provided content. +func reset(path string, content []byte) error { + // Create a temp file in the same dir where we want it to wind up + f, err := os.CreateTemp(filepath.Dir(path), "*") + if err != nil { + return err + } + fname := f.Name() + + // Clean up the leftover file + defer func() { + if f != nil { + f.Close() + } + os.Remove(fname) + }() + + // Write the content into the temp file + _, err = f.Write(content) + if err != nil { + return err + } + // Permanently persist the content into disk + if err := f.Sync(); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + f = nil + + return atomicRename(fname, path) } // openFreezerFileForAppend opens a freezer table file and seeks to the end func openFreezerFileForAppend(filename string) (*os.File, error) { // Open the file without the O_APPEND flag - // because it has differing behaviour during Truncate operations + // because it has differing behavior during Truncate operations // on different OS's file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644) if err != nil { diff --git a/core/rawdb/table.go b/core/rawdb/table.go index d38afdaa35..407a619c9f 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -253,6 +253,11 @@ func (b *tableBatch) Reset() { b.batch.Reset() } +// Close closes the batch and releases all associated resources. +func (b *tableBatch) Close() { + b.batch.Close() +} + // tableReplayer is a wrapper around a batch replayer which truncates // the added prefix. type tableReplayer struct { diff --git a/core/state/database.go b/core/state/database.go index 4a5547d075..002ce57fbc 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -20,13 +20,13 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/overlay" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/bintrie" "github.com/ethereum/go-ethereum/trie/transitiontrie" @@ -34,14 +34,6 @@ import ( "github.com/ethereum/go-ethereum/triedb" ) -const ( - // Number of codehash->size associations to keep. - codeSizeCacheSize = 1_000_000 // 4 megabytes in total - - // Cache size granted for caching clean code. - codeCacheSize = 256 * 1024 * 1024 -) - // Database wraps access to tries and contract code. type Database interface { // Reader returns a state reader associated with the specified state root. @@ -58,6 +50,11 @@ type Database interface { // Snapshot returns the underlying state snapshot. Snapshot() *snapshot.Tree + + // Commit flushes all pending writes and finalizes the state transition, + // committing the changes to the underlying storage. It returns an error + // if the commit fails. + Commit(update *stateUpdate) error } // Trie is a Ethereum Merkle Patricia trie. @@ -149,32 +146,34 @@ type Trie interface { // state snapshot to provide functionalities for state access. It's meant to be a // long-live object and has a few caches inside for sharing between blocks. type CachingDB struct { - disk ethdb.KeyValueStore - triedb *triedb.Database - snap *snapshot.Tree - codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - codeSizeCache *lru.Cache[common.Hash, int] - - // Transition-specific fields - TransitionStatePerRoot *lru.Cache[common.Hash, *overlay.TransitionState] + triedb *triedb.Database + codedb *CodeDB + snap *snapshot.Tree } // NewDatabase creates a state database with the provided data sources. -func NewDatabase(triedb *triedb.Database, snap *snapshot.Tree) *CachingDB { +func NewDatabase(triedb *triedb.Database, codedb *CodeDB) *CachingDB { + if codedb == nil { + codedb = NewCodeDB(triedb.Disk()) + } return &CachingDB{ - disk: triedb.Disk(), - triedb: triedb, - snap: snap, - codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), - TransitionStatePerRoot: lru.NewCache[common.Hash, *overlay.TransitionState](1000), + triedb: triedb, + codedb: codedb, } } // NewDatabaseForTesting is similar to NewDatabase, but it initializes the caching // db by using an ephemeral memory db with default config for testing. func NewDatabaseForTesting() *CachingDB { - return NewDatabase(triedb.NewDatabase(rawdb.NewMemoryDatabase(), nil), nil) + db := rawdb.NewMemoryDatabase() + return NewDatabase(triedb.NewDatabase(db, nil), NewCodeDB(db)) +} + +// WithSnapshot configures the provided contract code cache. Note that this +// registration must be performed before the cachingDB is used. +func (db *CachingDB) WithSnapshot(snapshot *snapshot.Tree) *CachingDB { + db.snap = snapshot + return db } // StateReader returns a state reader associated with the specified state root. @@ -218,21 +217,20 @@ func (db *CachingDB) Reader(stateRoot common.Hash) (Reader, error) { if err != nil { return nil, err } - return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), sr), nil + return newReader(db.codedb.Reader(), sr), nil } // ReadersWithCacheStats creates a pair of state readers that share the same // underlying state reader and internal state cache, while maintaining separate // statistics respectively. -func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (ReaderWithStats, ReaderWithStats, error) { +func (db *CachingDB) ReadersWithCacheStats(stateRoot common.Hash) (Reader, Reader, error) { r, err := db.StateReader(stateRoot) if err != nil { return nil, nil, err } sr := newStateReaderWithCache(r) - - ra := newReaderWithStats(sr, newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache)) - rb := newReaderWithStats(sr, newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache)) + ra := newReader(db.codedb.Reader(), newStateReaderWithStats(sr)) + rb := newReader(db.codedb.Reader(), newStateReaderWithStats(sr)) return ra, rb, nil } @@ -268,22 +266,6 @@ func (db *CachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Addre return tr, nil } -// ContractCodeWithPrefix retrieves a particular contract's code. If the -// code can't be found in the cache, then check the existence with **new** -// db scheme. -func (db *CachingDB) ContractCodeWithPrefix(address common.Address, codeHash common.Hash) []byte { - code, _ := db.codeCache.Get(codeHash) - if len(code) > 0 { - return code - } - code = rawdb.ReadCodeWithPrefix(db.disk, codeHash) - if len(code) > 0 { - db.codeCache.Add(codeHash, code) - db.codeSizeCache.Add(codeHash, len(code)) - } - return code -} - // TrieDB retrieves any intermediate trie-node caching layer. func (db *CachingDB) TrieDB() *triedb.Database { return db.triedb @@ -294,6 +276,40 @@ func (db *CachingDB) Snapshot() *snapshot.Tree { return db.snap } +// Commit flushes all pending writes and finalizes the state transition, +// committing the changes to the underlying storage. It returns an error +// if the commit fails. +func (db *CachingDB) Commit(update *stateUpdate) error { + // Short circuit if nothing to commit + if update.empty() { + return nil + } + // Commit dirty contract code if any exists + if len(update.codes) > 0 { + batch := db.codedb.NewBatchWithSize(len(update.codes)) + for _, code := range update.codes { + batch.Put(code.hash, code.blob) + } + if err := batch.Commit(); err != nil { + return err + } + } + // If snapshotting is enabled, update the snapshot tree with this new version + if db.snap != nil && db.snap.Snapshot(update.originRoot) != nil { + if err := db.snap.Update(update.root, update.originRoot, update.accounts, update.storages); err != nil { + log.Warn("Failed to update snapshot tree", "from", update.originRoot, "to", update.root, "err", err) + } + // Keep 128 diff layers in the memory, persistent layer is 129th. + // - head layer is paired with HEAD state + // - head-1 layer is paired with HEAD-1 state + // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state + if err := db.snap.Cap(update.root, TriesInMemory); err != nil { + log.Warn("Failed to cap snapshot tree", "root", update.root, "layers", TriesInMemory, "err", err) + } + } + return db.triedb.Update(update.root, update.originRoot, update.blockNumber, update.nodes, update.stateSet()) +} + // mustCopyTrie returns a deep-copied trie. func mustCopyTrie(t Trie) Trie { switch t := t.(type) { diff --git a/core/state/database_code.go b/core/state/database_code.go new file mode 100644 index 0000000000..820c9c1168 --- /dev/null +++ b/core/state/database_code.go @@ -0,0 +1,231 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import ( + "sync/atomic" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" +) + +const ( + // Number of codeHash->size associations to keep. + codeSizeCacheSize = 1_000_000 + + // Cache size granted for caching clean code. + codeCacheSize = 256 * 1024 * 1024 +) + +// CodeCache maintains cached contract code that is shared across blocks, enabling +// fast access for external calls such as RPCs and state transitions. +// +// It is thread-safe and has a bounded size. +type codeCache struct { + codeCache *lru.SizeConstrainedCache[common.Hash, []byte] + codeSizeCache *lru.Cache[common.Hash, int] +} + +// newCodeCache initializes the contract code cache with the predefined capacity. +func newCodeCache() *codeCache { + return &codeCache{ + codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), + codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), + } +} + +// Get returns the contract code associated with the provided code hash. +func (c *codeCache) Get(hash common.Hash) ([]byte, bool) { + return c.codeCache.Get(hash) +} + +// GetSize returns the contract code size associated with the provided code hash. +func (c *codeCache) GetSize(hash common.Hash) (int, bool) { + return c.codeSizeCache.Get(hash) +} + +// Put adds the provided contract code along with its size information into the cache. +func (c *codeCache) Put(hash common.Hash, code []byte) { + c.codeCache.Add(hash, code) + c.codeSizeCache.Add(hash, len(code)) +} + +// CodeReader implements state.ContractCodeReader, accessing contract code either in +// local key-value store or the shared code cache. +// +// Reader is safe for concurrent access. +type CodeReader struct { + db ethdb.KeyValueReader + cache *codeCache + + // Cache statistics + hit atomic.Int64 // Number of code lookups found in the cache + miss atomic.Int64 // Number of code lookups not found in the cache + hitBytes atomic.Int64 // Total number of bytes read from cache + missBytes atomic.Int64 // Total number of bytes read from database +} + +// newCodeReader constructs the code reader with provided key value store and the cache. +func newCodeReader(db ethdb.KeyValueReader, cache *codeCache) *CodeReader { + return &CodeReader{ + db: db, + cache: cache, + } +} + +// Has returns the flag indicating whether the contract code with +// specified address and hash exists or not. +func (r *CodeReader) Has(addr common.Address, codeHash common.Hash) bool { + return len(r.Code(addr, codeHash)) > 0 +} + +// Code implements state.ContractCodeReader, retrieving a particular contract's code. +// Null is returned if the contract code is not present. +func (r *CodeReader) Code(addr common.Address, codeHash common.Hash) []byte { + code, _ := r.cache.Get(codeHash) + if len(code) > 0 { + r.hit.Add(1) + r.hitBytes.Add(int64(len(code))) + return code + } + r.miss.Add(1) + + code = rawdb.ReadCode(r.db, codeHash) + if len(code) > 0 { + r.cache.Put(codeHash, code) + r.missBytes.Add(int64(len(code))) + } + return code +} + +// CodeSize implements state.ContractCodeReader, retrieving a particular contract +// code's size. Zero is returned if the contract code is not present. +func (r *CodeReader) CodeSize(addr common.Address, codeHash common.Hash) int { + if cached, ok := r.cache.GetSize(codeHash); ok { + r.hit.Add(1) + return cached + } + return len(r.Code(addr, codeHash)) +} + +// CodeWithPrefix retrieves the contract code for the specified account address +// and code hash. It is almost identical to Code, but uses rawdb.ReadCodeWithPrefix +// for database lookups. The intention is to gradually deprecate the old +// contract code scheme. +func (r *CodeReader) CodeWithPrefix(addr common.Address, codeHash common.Hash) []byte { + code, _ := r.cache.Get(codeHash) + if len(code) > 0 { + r.hit.Add(1) + r.hitBytes.Add(int64(len(code))) + return code + } + r.miss.Add(1) + + code = rawdb.ReadCodeWithPrefix(r.db, codeHash) + if len(code) > 0 { + r.cache.Put(codeHash, code) + r.missBytes.Add(int64(len(code))) + } + return code +} + +// GetCodeStats implements ContractCodeReaderStater, returning the statistics +// of the code reader. +func (r *CodeReader) GetCodeStats() ContractCodeReaderStats { + return ContractCodeReaderStats{ + CacheHit: r.hit.Load(), + CacheMiss: r.miss.Load(), + CacheHitBytes: r.hitBytes.Load(), + CacheMissBytes: r.missBytes.Load(), + } +} + +type CodeBatch struct { + db *CodeDB + codes [][]byte + codeHashes []common.Hash +} + +// newCodeBatch constructs the batch for writing contract code. +func newCodeBatch(db *CodeDB) *CodeBatch { + return &CodeBatch{ + db: db, + } +} + +// newCodeBatchWithSize constructs the batch with a pre-allocated capacity. +func newCodeBatchWithSize(db *CodeDB, size int) *CodeBatch { + return &CodeBatch{ + db: db, + codes: make([][]byte, 0, size), + codeHashes: make([]common.Hash, 0, size), + } +} + +// Put inserts the given contract code into the writer, waiting for commit. +func (b *CodeBatch) Put(codeHash common.Hash, code []byte) { + b.codes = append(b.codes, code) + b.codeHashes = append(b.codeHashes, codeHash) +} + +// Commit flushes the accumulated dirty contract code into the database and +// also place them in the cache. +func (b *CodeBatch) Commit() error { + batch := b.db.db.NewBatch() + for i, code := range b.codes { + rawdb.WriteCode(batch, b.codeHashes[i], code) + b.db.cache.Put(b.codeHashes[i], code) + } + if err := batch.Write(); err != nil { + return err + } + b.codes = b.codes[:0] + b.codeHashes = b.codeHashes[:0] + return nil +} + +// CodeDB is responsible for managing the contract code and provides the access +// to it. It can be used as a global object, sharing it between multiple entities. +type CodeDB struct { + db ethdb.KeyValueStore + cache *codeCache +} + +// NewCodeDB constructs the contract code database with the provided key value store. +func NewCodeDB(db ethdb.KeyValueStore) *CodeDB { + return &CodeDB{ + db: db, + cache: newCodeCache(), + } +} + +// Reader returns the contract code reader. +func (d *CodeDB) Reader() *CodeReader { + return newCodeReader(d.db, d.cache) +} + +// NewBatch returns the batch for flushing contract codes. +func (d *CodeDB) NewBatch() *CodeBatch { + return newCodeBatch(d) +} + +// NewBatchWithSize returns the batch with pre-allocated capacity. +func (d *CodeDB) NewBatchWithSize(size int) *CodeBatch { + return newCodeBatchWithSize(d, size) +} diff --git a/core/state/database_history.go b/core/state/database_history.go index 7a2be8fe4f..c25c4eae4b 100644 --- a/core/state/database_history.go +++ b/core/state/database_history.go @@ -17,15 +17,14 @@ package state import ( + "errors" "fmt" "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" @@ -221,19 +220,15 @@ func (r *historicalTrieReader) Storage(addr common.Address, key common.Hash) (co // HistoricDB is the implementation of Database interface, with the ability to // access historical state. type HistoricDB struct { - disk ethdb.KeyValueStore - triedb *triedb.Database - codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - codeSizeCache *lru.Cache[common.Hash, int] + triedb *triedb.Database + codedb *CodeDB } // NewHistoricDatabase creates a historic state database. -func NewHistoricDatabase(disk ethdb.KeyValueStore, triedb *triedb.Database) *HistoricDB { +func NewHistoricDatabase(triedb *triedb.Database, codedb *CodeDB) *HistoricDB { return &HistoricDB{ - disk: disk, - triedb: triedb, - codeCache: lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize), - codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize), + triedb: triedb, + codedb: codedb, } } @@ -258,7 +253,7 @@ func (db *HistoricDB) Reader(stateRoot common.Hash) (Reader, error) { if err != nil { return nil, err } - return newReader(newCachingCodeReader(db.disk, db.codeCache, db.codeSizeCache), combined), nil + return newReader(db.codedb.Reader(), combined), nil } // OpenTrie opens the main account trie. It's not supported by historic database. @@ -298,3 +293,10 @@ func (db *HistoricDB) TrieDB() *triedb.Database { func (db *HistoricDB) Snapshot() *snapshot.Tree { return nil } + +// Commit flushes all pending writes and finalizes the state transition, +// committing the changes to the underlying storage. It returns an error +// if the commit fails. +func (db *HistoricDB) Commit(update *stateUpdate) error { + return errors.New("not implemented") +} diff --git a/core/state/iterator.go b/core/state/iterator.go index 0abae091d9..0050a840d8 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -144,10 +144,7 @@ func (it *nodeIterator) step() error { } if !bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) { it.codeHash = common.BytesToHash(account.CodeHash) - it.code, err = it.state.reader.Code(address, common.BytesToHash(account.CodeHash)) - if err != nil { - return fmt.Errorf("code %x: %v", account.CodeHash, err) - } + it.code = it.state.reader.Code(address, common.BytesToHash(account.CodeHash)) if len(it.code) == 0 { return fmt.Errorf("code is not found: %x", account.CodeHash) } diff --git a/core/state/reader.go b/core/state/reader.go index 35b732173b..49375c467c 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -18,17 +18,13 @@ package state import ( "errors" - "fmt" "sync" "sync/atomic" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/overlay" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/bintrie" @@ -38,55 +34,26 @@ import ( ) // ContractCodeReader defines the interface for accessing contract code. +// +// ContractCodeReader is supposed to be thread-safe. type ContractCodeReader interface { // Has returns the flag indicating whether the contract code with // specified address and hash exists or not. Has(addr common.Address, codeHash common.Hash) bool - // Code retrieves a particular contract's code. - // - // - Returns nil code along with nil error if the requested contract code - // doesn't exist - // - Returns an error only if an unexpected issue occurs - Code(addr common.Address, codeHash common.Hash) ([]byte, error) + // Code retrieves a particular contract's code. Returns nil code if the + // requested contract code doesn't exist. + Code(addr common.Address, codeHash common.Hash) []byte - // CodeSize retrieves a particular contracts code's size. - // - // - Returns zero code size along with nil error if the requested contract code - // doesn't exist - // - Returns an error only if an unexpected issue occurs - CodeSize(addr common.Address, codeHash common.Hash) (int, error) -} - -// ContractCodeReaderStats aggregates statistics for the contract code reader. -type ContractCodeReaderStats struct { - CacheHit int64 // Number of cache hits - CacheMiss int64 // Number of cache misses - CacheHitBytes int64 // Total bytes served from cache - CacheMissBytes int64 // Total bytes read on cache misses -} - -// HitRate returns the cache hit rate. -func (s ContractCodeReaderStats) HitRate() float64 { - if s.CacheHit == 0 { - return 0 - } - return float64(s.CacheHit) / float64(s.CacheHit+s.CacheMiss) -} - -// ContractCodeReaderWithStats extends ContractCodeReader by adding GetStats to -// expose statistics of code reader. -type ContractCodeReaderWithStats interface { - ContractCodeReader - - GetStats() ContractCodeReaderStats + // CodeSize retrieves a particular contracts code's size. Returns zero code + // size if the requested contract code doesn't exist. + CodeSize(addr common.Address, codeHash common.Hash) int } // StateReader defines the interface for accessing accounts and storage slots // associated with a specific state. // -// StateReader is assumed to be thread-safe and implementation must take care -// of the concurrency issue by themselves. +// StateReader is supposed to be thread-safe. type StateReader interface { // Account retrieves the account associated with a particular address. // @@ -114,119 +81,6 @@ type Reader interface { StateReader } -// ReaderStats wraps the statistics of reader. -type ReaderStats struct { - AccountCacheHit int64 - AccountCacheMiss int64 - StorageCacheHit int64 - StorageCacheMiss int64 - CodeStats ContractCodeReaderStats -} - -// String implements fmt.Stringer, returning string format statistics. -func (s ReaderStats) String() string { - var ( - accountCacheHitRate float64 - storageCacheHitRate float64 - ) - if s.AccountCacheHit > 0 { - accountCacheHitRate = float64(s.AccountCacheHit) / float64(s.AccountCacheHit+s.AccountCacheMiss) * 100 - } - if s.StorageCacheHit > 0 { - storageCacheHitRate = float64(s.StorageCacheHit) / float64(s.StorageCacheHit+s.StorageCacheMiss) * 100 - } - msg := fmt.Sprintf("Reader statistics\n") - msg += fmt.Sprintf("account: hit: %d, miss: %d, rate: %.2f\n", s.AccountCacheHit, s.AccountCacheMiss, accountCacheHitRate) - msg += fmt.Sprintf("storage: hit: %d, miss: %d, rate: %.2f\n", s.StorageCacheHit, s.StorageCacheMiss, storageCacheHitRate) - msg += fmt.Sprintf("code: hit: %d(%v), miss: %d(%v), rate: %.2f\n", s.CodeStats.CacheHit, common.StorageSize(s.CodeStats.CacheHitBytes), s.CodeStats.CacheMiss, common.StorageSize(s.CodeStats.CacheMissBytes), s.CodeStats.HitRate()) - return msg -} - -// ReaderWithStats wraps the additional method to retrieve the reader statistics from. -type ReaderWithStats interface { - Reader - GetStats() ReaderStats -} - -// cachingCodeReader implements ContractCodeReader, accessing contract code either in -// local key-value store or the shared code cache. -// -// cachingCodeReader is safe for concurrent access. -type cachingCodeReader struct { - db ethdb.KeyValueReader - - // These caches could be shared by multiple code reader instances, - // they are natively thread-safe. - codeCache *lru.SizeConstrainedCache[common.Hash, []byte] - codeSizeCache *lru.Cache[common.Hash, int] - - // Cache statistics - hit atomic.Int64 // Number of code lookups found in the cache - miss atomic.Int64 // Number of code lookups not found in the cache - hitBytes atomic.Int64 // Total number of bytes read from cache - missBytes atomic.Int64 // Total number of bytes read from database -} - -// newCachingCodeReader constructs the code reader. -func newCachingCodeReader(db ethdb.KeyValueReader, codeCache *lru.SizeConstrainedCache[common.Hash, []byte], codeSizeCache *lru.Cache[common.Hash, int]) *cachingCodeReader { - return &cachingCodeReader{ - db: db, - codeCache: codeCache, - codeSizeCache: codeSizeCache, - } -} - -// Code implements ContractCodeReader, retrieving a particular contract's code. -// If the contract code doesn't exist, no error will be returned. -func (r *cachingCodeReader) Code(addr common.Address, codeHash common.Hash) ([]byte, error) { - code, _ := r.codeCache.Get(codeHash) - if len(code) > 0 { - r.hit.Add(1) - r.hitBytes.Add(int64(len(code))) - return code, nil - } - r.miss.Add(1) - - code = rawdb.ReadCode(r.db, codeHash) - if len(code) > 0 { - r.codeCache.Add(codeHash, code) - r.codeSizeCache.Add(codeHash, len(code)) - r.missBytes.Add(int64(len(code))) - } - return code, nil -} - -// CodeSize implements ContractCodeReader, retrieving a particular contracts code's size. -// If the contract code doesn't exist, no error will be returned. -func (r *cachingCodeReader) CodeSize(addr common.Address, codeHash common.Hash) (int, error) { - if cached, ok := r.codeSizeCache.Get(codeHash); ok { - r.hit.Add(1) - return cached, nil - } - code, err := r.Code(addr, codeHash) - if err != nil { - return 0, err - } - return len(code), nil -} - -// Has returns the flag indicating whether the contract code with -// specified address and hash exists or not. -func (r *cachingCodeReader) Has(addr common.Address, codeHash common.Hash) bool { - code, _ := r.Code(addr, codeHash) - return len(code) > 0 -} - -// GetStats returns the statistics of the code reader. -func (r *cachingCodeReader) GetStats() ContractCodeReaderStats { - return ContractCodeReaderStats{ - CacheHit: r.hit.Load(), - CacheMiss: r.miss.Load(), - CacheHitBytes: r.hitBytes.Load(), - CacheMissBytes: r.missBytes.Load(), - } -} - // flatReader wraps a database state reader and is safe for concurrent access. type flatReader struct { reader database.StateReader @@ -495,20 +349,6 @@ func (r *multiStateReader) Storage(addr common.Address, slot common.Hash) (commo return common.Hash{}, errors.Join(errs...) } -// reader is the wrapper of ContractCodeReader and StateReader interface. -type reader struct { - ContractCodeReader - StateReader -} - -// newReader constructs a reader with the supplied code reader and state reader. -func newReader(codeReader ContractCodeReader, stateReader StateReader) *reader { - return &reader{ - ContractCodeReader: codeReader, - StateReader: stateReader, - } -} - // stateReaderWithCache is a wrapper around StateReader that maintains additional // state caches to support concurrent state access. type stateReaderWithCache struct { @@ -619,9 +459,10 @@ func (r *stateReaderWithCache) Storage(addr common.Address, slot common.Hash) (c return value, err } -type readerWithStats struct { +// stateReaderWithStats is a wrapper over the stateReaderWithCache, tracking +// the cache hit statistics of the reader. +type stateReaderWithStats struct { *stateReaderWithCache - ContractCodeReaderWithStats accountCacheHit atomic.Int64 accountCacheMiss atomic.Int64 @@ -629,11 +470,10 @@ type readerWithStats struct { storageCacheMiss atomic.Int64 } -// newReaderWithStats constructs the reader with additional statistics tracked. -func newReaderWithStats(sr *stateReaderWithCache, cr ContractCodeReaderWithStats) *readerWithStats { - return &readerWithStats{ - stateReaderWithCache: sr, - ContractCodeReaderWithStats: cr, +// newReaderWithStats constructs the state reader with additional statistics tracked. +func newStateReaderWithStats(sr *stateReaderWithCache) *stateReaderWithStats { + return &stateReaderWithStats{ + stateReaderWithCache: sr, } } @@ -641,7 +481,7 @@ func newReaderWithStats(sr *stateReaderWithCache, cr ContractCodeReaderWithStats // The returned account might be nil if it's not existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithStats) Account(addr common.Address) (*types.StateAccount, error) { +func (r *stateReaderWithStats) Account(addr common.Address) (*types.StateAccount, error) { account, incache, err := r.stateReaderWithCache.account(addr) if err != nil { return nil, err @@ -659,7 +499,7 @@ func (r *readerWithStats) Account(addr common.Address) (*types.StateAccount, err // existent. // // An error will be returned if the state is corrupted in the underlying reader. -func (r *readerWithStats) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { +func (r *stateReaderWithStats) Storage(addr common.Address, slot common.Hash) (common.Hash, error) { value, incache, err := r.stateReaderWithCache.storage(addr, slot) if err != nil { return common.Hash{}, err @@ -672,13 +512,51 @@ func (r *readerWithStats) Storage(addr common.Address, slot common.Hash) (common return value, nil } -// GetStats implements ReaderWithStats, returning the statistics of state reader. -func (r *readerWithStats) GetStats() ReaderStats { - return ReaderStats{ +// GetStateStats implements StateReaderStater, returning the statistics of the +// state reader. +func (r *stateReaderWithStats) GetStateStats() StateReaderStats { + return StateReaderStats{ AccountCacheHit: r.accountCacheHit.Load(), AccountCacheMiss: r.accountCacheMiss.Load(), StorageCacheHit: r.storageCacheHit.Load(), StorageCacheMiss: r.storageCacheMiss.Load(), - CodeStats: r.ContractCodeReaderWithStats.GetStats(), + } +} + +// reader aggregates a code reader and a state reader into a single object. +type reader struct { + ContractCodeReader + StateReader +} + +// newReader constructs a reader with the supplied code reader and state reader. +func newReader(codeReader ContractCodeReader, stateReader StateReader) *reader { + return &reader{ + ContractCodeReader: codeReader, + StateReader: stateReader, + } +} + +// GetCodeStats returns the statistics of code access. +func (r *reader) GetCodeStats() ContractCodeReaderStats { + if stater, ok := r.ContractCodeReader.(ContractCodeReaderStater); ok { + return stater.GetCodeStats() + } + return ContractCodeReaderStats{} +} + +// GetStateStats returns the statistics of state access. +func (r *reader) GetStateStats() StateReaderStats { + if stater, ok := r.StateReader.(StateReaderStater); ok { + return stater.GetStateStats() + } + return StateReaderStats{} +} + +// GetStats returns the aggregated statistics for both state and code access. +func (r *reader) GetStats() ReaderStats { + return ReaderStats{ + CodeStats: r.GetCodeStats(), + StateStats: r.GetStateStats(), } } diff --git a/core/state/reader_stater.go b/core/state/reader_stater.go new file mode 100644 index 0000000000..5294275953 --- /dev/null +++ b/core/state/reader_stater.go @@ -0,0 +1,82 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +// ContractCodeReaderStats aggregates statistics for the contract code reader. +type ContractCodeReaderStats struct { + CacheHit int64 // Number of cache hits + CacheMiss int64 // Number of cache misses + CacheHitBytes int64 // Total bytes served from cache + CacheMissBytes int64 // Total bytes read on cache misses +} + +// HitRate returns the cache hit rate in percentage. +func (s ContractCodeReaderStats) HitRate() float64 { + total := s.CacheHit + s.CacheMiss + if total == 0 { + return 0 + } + return float64(s.CacheHit) / float64(total) * 100 +} + +// ContractCodeReaderStater wraps the method to retrieve the statistics of +// contract code reader. +type ContractCodeReaderStater interface { + GetCodeStats() ContractCodeReaderStats +} + +// StateReaderStats aggregates statistics for the state reader. +type StateReaderStats struct { + AccountCacheHit int64 // Number of account cache hits + AccountCacheMiss int64 // Number of account cache misses + StorageCacheHit int64 // Number of storage cache hits + StorageCacheMiss int64 // Number of storage cache misses +} + +// AccountCacheHitRate returns the cache hit rate of account requests in percentage. +func (s StateReaderStats) AccountCacheHitRate() float64 { + total := s.AccountCacheHit + s.AccountCacheMiss + if total == 0 { + return 0 + } + return float64(s.AccountCacheHit) / float64(total) * 100 +} + +// StorageCacheHitRate returns the cache hit rate of storage requests in percentage. +func (s StateReaderStats) StorageCacheHitRate() float64 { + total := s.StorageCacheHit + s.StorageCacheMiss + if total == 0 { + return 0 + } + return float64(s.StorageCacheHit) / float64(total) * 100 +} + +// StateReaderStater wraps the method to retrieve the statistics of state reader. +type StateReaderStater interface { + GetStateStats() StateReaderStats +} + +// ReaderStats wraps the statistics of reader. +type ReaderStats struct { + CodeStats ContractCodeReaderStats + StateStats StateReaderStats +} + +// ReaderStater defines the capability to retrieve aggregated statistics. +type ReaderStater interface { + GetStats() ReaderStats +} diff --git a/core/state/state_object.go b/core/state/state_object.go index eecb72ce5d..dd30bb64a5 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -195,6 +195,19 @@ func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { // have been handles via pendingStorage above. // 2) we don't have new values, and can deliver empty response back if _, destructed := s.db.stateObjectsDestruct[s.address]; destructed { + // Invoke the reader regardless and discard the returned value. + // The returned value may not be empty, as it could belong to a + // self-destructed contract. + // + // The read operation is still essential for correctly building + // the block-level access list. + // + // TODO(rjl493456442) the reader interface can be extended with + // Touch, recording the read access without the actual disk load. + _, err := s.db.reader.Storage(s.address, key) + if err != nil { + s.db.setError(err) + } s.originStorage[key] = common.Hash{} // track the empty slot as origin value return common.Hash{} } @@ -551,10 +564,7 @@ func (s *stateObject) Code() []byte { s.db.CodeLoadBytes += len(s.code) }(time.Now()) - code, err := s.db.reader.Code(s.address, common.BytesToHash(s.CodeHash())) - if err != nil { - s.db.setError(fmt.Errorf("can't load code hash %x: %v", s.CodeHash(), err)) - } + code := s.db.reader.Code(s.address, common.BytesToHash(s.CodeHash())) if len(code) == 0 { s.db.setError(fmt.Errorf("code is not found %x", s.CodeHash())) } @@ -577,10 +587,7 @@ func (s *stateObject) CodeSize() int { s.db.CodeReads += time.Since(start) }(time.Now()) - size, err := s.db.reader.CodeSize(s.address, common.BytesToHash(s.CodeHash())) - if err != nil { - s.db.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err)) - } + size := s.db.reader.CodeSize(s.address, common.BytesToHash(s.CodeHash())) if size == 0 { s.db.setError(fmt.Errorf("code is not found %x", s.CodeHash())) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 3a2d9c9ac2..2477242eb5 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -28,7 +28,6 @@ import ( "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/stateless" "github.com/ethereum/go-ethereum/core/tracing" @@ -148,8 +147,7 @@ type StateDB struct { StorageReads time.Duration StorageUpdates time.Duration StorageCommits time.Duration - SnapshotCommits time.Duration - TrieDBCommits time.Duration + DatabaseCommits time.Duration CodeReads time.Duration AccountLoaded int // Number of accounts retrieved from the database during the state transition @@ -1333,41 +1331,14 @@ func (s *StateDB) commitAndFlush(block uint64, deleteEmptyObjects bool, noStorag return nil, err } } - // Commit dirty contract code if any exists - if db := s.db.TrieDB().Disk(); db != nil && len(ret.codes) > 0 { - batch := db.NewBatch() - for _, code := range ret.codes { - rawdb.WriteCode(batch, code.hash, code.blob) - } - if err := batch.Write(); err != nil { - return nil, err - } - } - if !ret.empty() { - // If snapshotting is enabled, update the snapshot tree with this new version - if snap := s.db.Snapshot(); snap != nil && snap.Snapshot(ret.originRoot) != nil { - start := time.Now() - if err := snap.Update(ret.root, ret.originRoot, ret.accounts, ret.storages); err != nil { - log.Warn("Failed to update snapshot tree", "from", ret.originRoot, "to", ret.root, "err", err) - } - // Keep 128 diff layers in the memory, persistent layer is 129th. - // - head layer is paired with HEAD state - // - head-1 layer is paired with HEAD-1 state - // - head-127 layer(bottom-most diff layer) is paired with HEAD-127 state - if err := snap.Cap(ret.root, TriesInMemory); err != nil { - log.Warn("Failed to cap snapshot tree", "root", ret.root, "layers", TriesInMemory, "err", err) - } - s.SnapshotCommits += time.Since(start) - } - // If trie database is enabled, commit the state update as a new layer - if db := s.db.TrieDB(); db != nil { - start := time.Now() - if err := db.Update(ret.root, ret.originRoot, block, ret.nodes, ret.stateSet()); err != nil { - return nil, err - } - s.TrieDBCommits += time.Since(start) - } + start := time.Now() + if err := s.db.Commit(ret); err != nil { + return nil, err } + s.DatabaseCommits = time.Since(start) + + // The reader update must be performed as the final step, otherwise, + // the new state would not be visible before db.commit. s.reader, _ = s.db.Reader(s.originalRoot) return ret, err } diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index 8b6ac0ba64..3582185344 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -209,7 +209,7 @@ func (test *stateTest) run() bool { if i != 0 { root = roots[len(roots)-1] } - state, err := New(root, NewDatabase(tdb, snaps)) + state, err := New(root, NewDatabase(tdb, nil).WithSnapshot(snaps)) if err != nil { panic(err) } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 661d17bb7b..8d1f93ca1b 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -1276,7 +1276,7 @@ func TestDeleteStorage(t *testing.T) { disk = rawdb.NewMemoryDatabase() tdb = triedb.NewDatabase(disk, nil) snaps, _ = snapshot.New(snapshot.Config{CacheSize: 10}, disk, tdb, types.EmptyRootHash) - db = NewDatabase(tdb, snaps) + db = NewDatabase(tdb, nil).WithSnapshot(snaps) state, _ = New(types.EmptyRootHash, db) addr = common.HexToAddress("0x1") ) @@ -1290,7 +1290,7 @@ func TestDeleteStorage(t *testing.T) { } root, _ := state.Commit(0, true, false) // Init phase done, create two states, one with snap and one without - fastState, _ := New(root, NewDatabase(tdb, snaps)) + fastState, _ := New(root, NewDatabase(tdb, nil).WithSnapshot(snaps)) slowState, _ := New(root, NewDatabase(tdb, nil)) obj := fastState.getOrNewStateObject(addr) diff --git a/core/state/stateupdate.go b/core/state/stateupdate.go index 0c1b76b4f8..1c171cbd5e 100644 --- a/core/state/stateupdate.go +++ b/core/state/stateupdate.go @@ -211,9 +211,9 @@ func (sc *stateUpdate) deriveCodeFields(reader ContractCodeReader) error { cache := make(map[common.Hash]bool) for addr, code := range sc.codes { if code.originHash != types.EmptyCodeHash { - blob, err := reader.Code(addr, code.originHash) - if err != nil { - return err + blob := reader.Code(addr, code.originHash) + if len(blob) == 0 { + return fmt.Errorf("original code of %x is empty", addr) } code.originBlob = blob } diff --git a/core/state/sync_test.go b/core/state/sync_test.go index cae0e0a936..e5e22deae5 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -222,8 +222,8 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool, s codeResults = make([]trie.CodeSyncResult, len(codeElements)) ) for i, element := range codeElements { - data, err := cReader.Code(common.Address{}, element.code) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, element.code) + if len(data) == 0 { t.Fatalf("failed to retrieve contract bytecode for hash %x", element.code) } codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} @@ -346,8 +346,8 @@ func testIterativeDelayedStateSync(t *testing.T, scheme string) { if len(codeElements) > 0 { codeResults := make([]trie.CodeSyncResult, len(codeElements)/2+1) for i, element := range codeElements[:len(codeResults)] { - data, err := cReader.Code(common.Address{}, element.code) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, element.code) + if len(data) == 0 { t.Fatalf("failed to retrieve contract bytecode for %x", element.code) } codeResults[i] = trie.CodeSyncResult{Hash: element.code, Data: data} @@ -452,8 +452,8 @@ func testIterativeRandomStateSync(t *testing.T, count int, scheme string) { if len(codeQueue) > 0 { results := make([]trie.CodeSyncResult, 0, len(codeQueue)) for hash := range codeQueue { - data, err := cReader.Code(common.Address{}, hash) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, hash) + if len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) @@ -551,8 +551,8 @@ func testIterativeRandomDelayedStateSync(t *testing.T, scheme string) { for hash := range codeQueue { delete(codeQueue, hash) - data, err := cReader.Code(common.Address{}, hash) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, hash) + if len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) @@ -671,8 +671,8 @@ func testIncompleteStateSync(t *testing.T, scheme string) { if len(codeQueue) > 0 { results := make([]trie.CodeSyncResult, 0, len(codeQueue)) for hash := range codeQueue { - data, err := cReader.Code(common.Address{}, hash) - if err != nil || len(data) == 0 { + data := cReader.Code(common.Address{}, hash) + if len(data) == 0 { t.Fatalf("failed to retrieve node data for %x", hash) } results = append(results, trie.CodeSyncResult{Hash: hash, Data: data}) diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index 1c738c1e38..c91d40d94f 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -107,7 +107,7 @@ func (p *statePrefetcher) Prefetch(block *types.Block, statedb *state.StateDB, c // We attempt to apply a transaction. The goal is not to execute // the transaction successfully, rather to warm up touched data slots. - if _, err := ApplyMessage(evm, msg, new(GasPool).AddGas(block.GasLimit())); err != nil { + if _, err := ApplyMessage(evm, msg, nil); err != nil { fails.Add(1) return nil // Ugh, something went horribly wrong, bail out } diff --git a/core/state_processor.go b/core/state_processor.go index b4b22e4318..85f106d58c 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -17,6 +17,7 @@ package core import ( + "context" "fmt" "math/big" @@ -27,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/params" ) @@ -57,18 +59,16 @@ func (p *StateProcessor) chainConfig() *params.ChainConfig { // Process returns the receipts and logs accumulated during the process and // returns the amount of gas that was used in the process. If any of the // transactions failed to execute due to insufficient gas it will return an error. -func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) { +func (p *StateProcessor) Process(ctx context.Context, block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) { var ( config = p.chainConfig() receipts types.Receipts - usedGas = new(uint64) header = block.Header() blockHash = block.Hash() blockNumber = block.Number() allLogs []*types.Log - gp = new(GasPool).AddGas(block.GasLimit()) + gp = NewGasPool(block.GasLimit()) ) - var tracingStateDB = vm.StateDB(statedb) if hooks := cfg.Tracer; hooks != nil { tracingStateDB = state.NewHookedState(statedb, hooks) @@ -101,30 +101,23 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } statedb.SetTxContext(tx.Hash(), i) + _, _, spanEnd := telemetry.StartSpan(ctx, "core.ApplyTransactionWithEVM", + telemetry.StringAttribute("tx.hash", tx.Hash().Hex()), + telemetry.Int64Attribute("tx.index", int64(i)), + ) - receipt, err := ApplyTransactionWithEVM(msg, gp, statedb, blockNumber, blockHash, context.Time, tx, usedGas, evm) + receipt, err := ApplyTransactionWithEVM(msg, gp, statedb, blockNumber, blockHash, context.Time, tx, evm) if err != nil { + spanEnd(&err) return nil, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } receipts = append(receipts, receipt) allLogs = append(allLogs, receipt.Logs...) + spanEnd(nil) } - // Read requests if Prague is enabled. - var requests [][]byte - if config.IsPrague(block.Number(), block.Time()) { - requests = [][]byte{} - // EIP-6110 - if err := ParseDepositLogs(&requests, allLogs, config); err != nil { - return nil, fmt.Errorf("failed to parse deposit logs: %w", err) - } - // EIP-7002 - if err := ProcessWithdrawalQueue(&requests, evm); err != nil { - return nil, fmt.Errorf("failed to process withdrawal queue: %w", err) - } - // EIP-7251 - if err := ProcessConsolidationQueue(&requests, evm); err != nil { - return nil, fmt.Errorf("failed to process consolidation queue: %w", err) - } + requests, err := postExecution(ctx, config, block, allLogs, evm) + if err != nil { + return nil, err } // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) @@ -134,14 +127,39 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg Receipts: receipts, Requests: requests, Logs: allLogs, - GasUsed: *usedGas, + GasUsed: gp.Used(), }, nil } +// postExecution processes the post-execution system calls if Prague is enabled. +func postExecution(ctx context.Context, config *params.ChainConfig, block *types.Block, allLogs []*types.Log, evm *vm.EVM) (requests [][]byte, err error) { + _, _, spanEnd := telemetry.StartSpan(ctx, "core.postExecution") + defer spanEnd(&err) + + // Read requests if Prague is enabled. + if config.IsPrague(block.Number(), block.Time()) { + requests = [][]byte{} + // EIP-6110 + if err := ParseDepositLogs(&requests, allLogs, config); err != nil { + return requests, fmt.Errorf("failed to parse deposit logs: %w", err) + } + // EIP-7002 + if err := ProcessWithdrawalQueue(&requests, evm); err != nil { + return requests, fmt.Errorf("failed to process withdrawal queue: %w", err) + } + // EIP-7251 + if err := ProcessConsolidationQueue(&requests, evm); err != nil { + return requests, fmt.Errorf("failed to process consolidation queue: %w", err) + } + } + + return requests, nil +} + // ApplyTransactionWithEVM attempts to apply a transaction to the given state database // and uses the input parameters for its environment similar to ApplyTransaction. However, // this method takes an already created EVM instance as input. -func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (receipt *types.Receipt, err error) { +func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, evm *vm.EVM) (receipt *types.Receipt, err error) { if hooks := evm.Config.Tracer; hooks != nil { if hooks.OnTxStart != nil { hooks.OnTxStart(evm.GetVMContext(), tx, msg.From) @@ -162,27 +180,31 @@ func ApplyTransactionWithEVM(msg *Message, gp *GasPool, statedb *state.StateDB, } else { root = statedb.IntermediateRoot(evm.ChainConfig().IsEIP158(blockNumber)).Bytes() } - *usedGas += result.UsedGas - // Merge the tx-local access event into the "block-local" one, in order to collect // all values, so that the witness can be built. if statedb.Database().TrieDB().IsVerkle() { statedb.AccessEvents().Merge(evm.AccessEvents) } - return MakeReceipt(evm, result, statedb, blockNumber, blockHash, blockTime, tx, *usedGas, root), nil + return MakeReceipt(evm, result, statedb, blockNumber, blockHash, blockTime, tx, gp.CumulativeUsed(), root), nil } // MakeReceipt generates the receipt object for a transaction given its execution result. -func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, usedGas uint64, root []byte) *types.Receipt { - // Create a new receipt for the transaction, storing the intermediate root and gas used - // by the tx. - receipt := &types.Receipt{Type: tx.Type(), PostState: root, CumulativeGasUsed: usedGas} +func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, blockTime uint64, tx *types.Transaction, cumulativeGas uint64, root []byte) *types.Receipt { + // Create a new receipt for the transaction, storing the intermediate root + // and gas used by the tx. + // + // The cumulative gas used equals the sum of gasUsed across all preceding + // txs with refunded gas deducted. + receipt := &types.Receipt{Type: tx.Type(), PostState: root, CumulativeGasUsed: cumulativeGas} if result.Failed() { receipt.Status = types.ReceiptStatusFailed } else { receipt.Status = types.ReceiptStatusSuccessful } receipt.TxHash = tx.Hash() + + // GasUsed = max(tx_gas_used - gas_refund, calldata_floor_gas_cost), unchanged + // in the Amsterdam fork. receipt.GasUsed = result.UsedGas if tx.Type() == types.BlobTxType { @@ -206,15 +228,15 @@ func MakeReceipt(evm *vm.EVM, result *ExecutionResult, statedb *state.StateDB, b // ApplyTransaction attempts to apply a transaction to the given state database // and uses the input parameters for its environment. It returns the receipt -// for the transaction, gas used and an error if the transaction failed, +// for the transaction and an error if the transaction failed, // indicating the block was invalid. -func ApplyTransaction(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction, usedGas *uint64) (*types.Receipt, error) { +func ApplyTransaction(evm *vm.EVM, gp *GasPool, statedb *state.StateDB, header *types.Header, tx *types.Transaction) (*types.Receipt, error) { msg, err := TransactionToMessage(tx, types.MakeSigner(evm.ChainConfig(), header.Number, header.Time), header.BaseFee) if err != nil { return nil, err } // Create a new context to be used in the EVM environment - return ApplyTransactionWithEVM(msg, gp, statedb, header.Number, header.Hash(), header.Time, tx, usedGas, evm) + return ApplyTransactionWithEVM(msg, gp, statedb, header.Number, header.Hash(), header.Time, tx, evm) } // ProcessBeaconBlockRoot applies the EIP-4788 system call to the beacon block root diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 3bf372800b..2700aed505 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -422,6 +422,9 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr beaconRoot := common.HexToHash("0xbeac00") header.ParentBeaconRoot = &beaconRoot } + if config.IsAmsterdam(header.Number, header.Time) { + header.SlotNumber = new(uint64) + } // Assemble and return the final block for sealing body := &types.Body{Transactions: txs} if config.IsShanghai(header.Number, header.Time) { diff --git a/core/state_transition.go b/core/state_transition.go index bf5ac07636..6a40b4f7ab 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -34,7 +34,7 @@ import ( // ExecutionResult includes all output after executing given evm // message no matter the execution itself is successful or not. type ExecutionResult struct { - UsedGas uint64 // Total used gas, not including the refunded gas + UsedGas uint64 // Total used gas, refunded gas is deducted MaxUsedGas uint64 // Maximum gas consumed during execution, excluding gas refunds. Err error // Any error encountered during the execution(listed in core/vm/errors.go) ReturnData []byte // Returned data from evm(function result or data supplied with revert opcode) @@ -177,9 +177,9 @@ func TransactionToMessage(tx *types.Transaction, s types.Signer, baseFee *big.In msg := &Message{ Nonce: tx.Nonce(), GasLimit: tx.Gas(), - GasPrice: new(big.Int).Set(tx.GasPrice()), - GasFeeCap: new(big.Int).Set(tx.GasFeeCap()), - GasTipCap: new(big.Int).Set(tx.GasTipCap()), + GasPrice: tx.GasPrice(), + GasFeeCap: tx.GasFeeCap(), + GasTipCap: tx.GasTipCap(), To: tx.To(), Value: tx.Value(), Data: tx.Data(), @@ -210,6 +210,11 @@ func TransactionToMessage(tx *types.Transaction, s types.Signer, baseFee *big.In // indicates a core error meaning that the message would always fail for that particular // state and would never be accepted within a block. func ApplyMessage(evm *vm.EVM, msg *Message, gp *GasPool) (*ExecutionResult, error) { + // Do not panic if the gas pool is nil. This is allowed when executing + // a single message via RPC invocation. + if gp == nil { + gp = NewGasPool(msg.GasLimit) + } evm.SetTxContext(NewEVMTxContext(msg)) return newStateTransition(evm, msg, gp).execute() } @@ -300,8 +305,8 @@ func (st *stateTransition) buyGas() error { st.evm.Config.Tracer.OnGasChange(0, st.msg.GasLimit, tracing.GasChangeTxInitialBalance) } st.gasRemaining = st.msg.GasLimit - st.initialGas = st.msg.GasLimit + mgvalU256, _ := uint256.FromBig(mgval) st.state.SubBalance(st.msg.From, mgvalU256, tracing.BalanceDecreaseGasBuy) return nil @@ -483,8 +488,10 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { } // Check whether the init code size has been exceeded. - if rules.IsShanghai && contractCreation && len(msg.Data) > params.MaxInitCodeSize { - return nil, fmt.Errorf("%w: code size %v limit %v", ErrMaxInitCodeSizeExceeded, len(msg.Data), params.MaxInitCodeSize) + if contractCreation { + if err := vm.CheckMaxInitCodeSize(&rules, uint64(len(msg.Data))); err != nil { + return nil, err + } } // Execute the preparatory steps for state transition which includes: @@ -542,8 +549,20 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { peakGasUsed = floorDataGas } } + // Return gas to the user st.returnGas() + // Return gas to the gas pool + if rules.IsAmsterdam { + // Refund is excluded for returning + err = st.gp.ReturnGas(st.initialGas-peakGasUsed, st.gasUsed()) + } else { + // Refund is included for returning + err = st.gp.ReturnGas(st.gasRemaining, st.gasUsed()) + } + if err != nil { + return nil, err + } effectiveTip := msg.GasPrice if rules.IsLondon { effectiveTip = new(big.Int).Sub(msg.GasPrice, st.evm.Context.BaseFee) @@ -564,7 +583,6 @@ func (st *stateTransition) execute() (*ExecutionResult, error) { st.evm.AccessEvents.AddAccount(st.evm.Context.Coinbase, true, math.MaxUint64) } } - return &ExecutionResult{ UsedGas: st.gasUsed(), MaxUsedGas: peakGasUsed, @@ -660,10 +678,6 @@ func (st *stateTransition) returnGas() { if st.evm.Config.Tracer != nil && st.evm.Config.Tracer.OnGasChange != nil && st.gasRemaining > 0 { st.evm.Config.Tracer.OnGasChange(st.gasRemaining, 0, tracing.GasChangeTxLeftOverReturned) } - - // Also return remaining gas to the block gas counter so it is - // available for the next transaction. - st.gp.AddGas(st.gasRemaining) } // gasUsed returns the amount of gas used up by the state transition. diff --git a/core/stateless.go b/core/stateless.go index b20c909da6..88d8ed8138 100644 --- a/core/stateless.go +++ b/core/stateless.go @@ -17,6 +17,8 @@ package core import ( + "context" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/consensus/beacon" @@ -40,7 +42,7 @@ import ( // - It cannot be placed outside of core, because it needs to construct a dud headerchain // // TODO(karalabe): Would be nice to resolve both issues above somehow and move it. -func ExecuteStateless(config *params.ChainConfig, vmconfig vm.Config, block *types.Block, witness *stateless.Witness) (common.Hash, common.Hash, error) { +func ExecuteStateless(ctx context.Context, config *params.ChainConfig, vmconfig vm.Config, block *types.Block, witness *stateless.Witness) (common.Hash, common.Hash, error) { // Sanity check if the supplied block accidentally contains a set root or // receipt hash. If so, be very loud, but still continue. if block.Root() != (common.Hash{}) { @@ -66,7 +68,7 @@ func ExecuteStateless(config *params.ChainConfig, vmconfig vm.Config, block *typ validator := NewBlockValidator(config, nil) // No chain, we only validate the state, not the block // Run the stateless blocks processing and self-validate certain fields - res, err := processor.Process(block, db, vmconfig) + res, err := processor.Process(ctx, block, db, vmconfig) if err != nil { return common.Hash{}, common.Hash{}, err } diff --git a/core/stateless/encoding.go b/core/stateless/encoding.go index 5c43159e66..d559178892 100644 --- a/core/stateless/encoding.go +++ b/core/stateless/encoding.go @@ -40,8 +40,8 @@ func (w *Witness) ToExtWitness() *ExtWitness { return ext } -// fromExtWitness converts the consensus witness format into our internal one. -func (w *Witness) fromExtWitness(ext *ExtWitness) error { +// FromExtWitness converts the consensus witness format into our internal one. +func (w *Witness) FromExtWitness(ext *ExtWitness) error { w.Headers = ext.Headers w.Codes = make(map[string]struct{}, len(ext.Codes)) @@ -66,7 +66,7 @@ func (w *Witness) DecodeRLP(s *rlp.Stream) error { if err := s.Decode(&ext); err != nil { return err } - return w.fromExtWitness(&ext) + return w.FromExtWitness(&ext) } // ExtWitness is a witness RLP encoding for transferring across clients. diff --git a/core/stateless/stats.go b/core/stateless/stats.go index 73ce031bff..7f4473a67c 100644 --- a/core/stateless/stats.go +++ b/core/stateless/stats.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/trie" ) var accountTrieLeavesAtDepth [16]*metrics.Counter @@ -41,59 +42,68 @@ func init() { // WitnessStats aggregates statistics for account and storage trie accesses. type WitnessStats struct { - accountTrieLeaves [16]int64 - storageTrieLeaves [16]int64 + accountTrie *trie.LevelStats + storageTrie *trie.LevelStats } // NewWitnessStats creates a new WitnessStats collector. func NewWitnessStats() *WitnessStats { - return &WitnessStats{} + return &WitnessStats{ + accountTrie: trie.NewLevelStats(), + storageTrie: trie.NewLevelStats(), + } +} + +func (s *WitnessStats) init() { + if s.accountTrie == nil { + s.accountTrie = trie.NewLevelStats() + } + if s.storageTrie == nil { + s.storageTrie = trie.NewLevelStats() + } } // Add records trie access depths from the given node paths. // If `owner` is the zero hash, accesses are attributed to the account trie; // otherwise, they are attributed to the storage trie of that account. func (s *WitnessStats) Add(nodes map[string][]byte, owner common.Hash) { - // Extract paths from the nodes map + s.init() + + // Extract paths from the nodes map. paths := slices.Collect(maps.Keys(nodes)) sort.Strings(paths) + ownerStat := s.accountTrie + if owner != (common.Hash{}) { + ownerStat = s.storageTrie + } + for i, path := range paths { // If current path is a prefix of the next path, it's not a leaf. // The last path is always a leaf. if i == len(paths)-1 || !strings.HasPrefix(paths[i+1], paths[i]) { - depth := len(path) - if owner == (common.Hash{}) { - if depth >= len(s.accountTrieLeaves) { - depth = len(s.accountTrieLeaves) - 1 - } - s.accountTrieLeaves[depth] += 1 - } else { - if depth >= len(s.storageTrieLeaves) { - depth = len(s.storageTrieLeaves) - 1 - } - s.storageTrieLeaves[depth] += 1 - } + ownerStat.AddLeaf(len(path)) } } } // ReportMetrics reports the collected statistics to the global metrics registry. func (s *WitnessStats) ReportMetrics(blockNumber uint64) { - // Encode the metrics as JSON for easier consumption - accountLeavesJson, _ := json.Marshal(s.accountTrieLeaves) - storageLeavesJson, _ := json.Marshal(s.storageTrieLeaves) + s.init() - // Log account trie depth statistics - log.Info("Account trie depth stats", - "block", blockNumber, - "leavesAtDepth", string(accountLeavesJson)) - log.Info("Storage trie depth stats", - "block", blockNumber, - "leavesAtDepth", string(storageLeavesJson)) + accountTrieLeaves := s.accountTrie.LeafDepths() + storageTrieLeaves := s.storageTrie.LeafDepths() - for i := 0; i < 16; i++ { - accountTrieLeavesAtDepth[i].Inc(s.accountTrieLeaves[i]) - storageTrieLeavesAtDepth[i].Inc(s.storageTrieLeaves[i]) + // Encode the metrics as JSON for easier consumption. + accountLeavesJSON, _ := json.Marshal(accountTrieLeaves) + storageLeavesJSON, _ := json.Marshal(storageTrieLeaves) + + // Log account trie depth statistics. + log.Info("Account trie depth stats", "block", blockNumber, "leavesAtDepth", string(accountLeavesJSON)) + log.Info("Storage trie depth stats", "block", blockNumber, "leavesAtDepth", string(storageLeavesJSON)) + + for i := 0; i < len(accountTrieLeavesAtDepth); i++ { + accountTrieLeavesAtDepth[i].Inc(accountTrieLeaves[i]) + storageTrieLeavesAtDepth[i].Inc(storageTrieLeaves[i]) } } diff --git a/core/stateless/stats_test.go b/core/stateless/stats_test.go index e77084df5d..6dc1fa0844 100644 --- a/core/stateless/stats_test.go +++ b/core/stateless/stats_test.go @@ -17,18 +17,27 @@ package stateless import ( + "strings" "testing" "github.com/ethereum/go-ethereum/common" ) +func expectedLeaves(counts map[int]int64) [16]int64 { + var leaves [16]int64 + for depth, count := range counts { + leaves[depth] = count + } + return leaves +} + func TestWitnessStatsAdd(t *testing.T) { tests := []struct { name string nodes map[string][]byte owner common.Hash - expectedAccountLeaves map[int64]int64 - expectedStorageLeaves map[int64]int64 + expectedAccountLeaves map[int]int64 + expectedStorageLeaves map[int]int64 }{ { name: "empty nodes", @@ -41,7 +50,7 @@ func TestWitnessStatsAdd(t *testing.T) { "": []byte("data"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{0: 1}, + expectedAccountLeaves: map[int]int64{0: 1}, }, { name: "single account trie leaf", @@ -49,7 +58,7 @@ func TestWitnessStatsAdd(t *testing.T) { "abc": []byte("data"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{3: 1}, + expectedAccountLeaves: map[int]int64{3: 1}, }, { name: "account trie with internal nodes", @@ -59,7 +68,7 @@ func TestWitnessStatsAdd(t *testing.T) { "abc": []byte("data3"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{3: 1}, // Only "abc" is a leaf + expectedAccountLeaves: map[int]int64{3: 1}, // Only "abc" is a leaf }, { name: "multiple account trie branches", @@ -72,7 +81,7 @@ func TestWitnessStatsAdd(t *testing.T) { "bcd": []byte("data6"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{3: 2}, // "abc" (3) + "bcd" (3) + expectedAccountLeaves: map[int]int64{3: 2}, // "abc" (3) + "bcd" (3) }, { name: "siblings are all leaves", @@ -82,7 +91,7 @@ func TestWitnessStatsAdd(t *testing.T) { "ac": []byte("data3"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{2: 3}, + expectedAccountLeaves: map[int]int64{2: 3}, }, { name: "storage trie leaves", @@ -93,7 +102,7 @@ func TestWitnessStatsAdd(t *testing.T) { "124": []byte("data4"), }, owner: common.HexToHash("0x1234"), - expectedStorageLeaves: map[int64]int64{3: 2}, // "123" (3) + "124" (3) + expectedStorageLeaves: map[int]int64{3: 2}, // "123" (3) + "124" (3) }, { name: "complex trie structure", @@ -109,7 +118,7 @@ func TestWitnessStatsAdd(t *testing.T) { "3": []byte("data9"), }, owner: common.Hash{}, - expectedAccountLeaves: map[int64]int64{1: 1, 3: 4}, // "123"(3) + "124"(3) + "234"(3) + "235"(3) + "3"(1) + expectedAccountLeaves: map[int]int64{1: 1, 3: 4}, // "123"(3) + "124"(3) + "234"(3) + "235"(3) + "3"(1) }, } @@ -118,32 +127,59 @@ func TestWitnessStatsAdd(t *testing.T) { stats := NewWitnessStats() stats.Add(tt.nodes, tt.owner) - var expectedAccountTrieLeaves [16]int64 - for depth, count := range tt.expectedAccountLeaves { - expectedAccountTrieLeaves[depth] = count + if got, want := stats.accountTrie.LeafDepths(), expectedLeaves(tt.expectedAccountLeaves); got != want { + t.Errorf("account trie leaves = %v, want %v", got, want) } - var expectedStorageTrieLeaves [16]int64 - for depth, count := range tt.expectedStorageLeaves { - expectedStorageTrieLeaves[depth] = count - } - - // Check account trie depth - if stats.accountTrieLeaves != expectedAccountTrieLeaves { - t.Errorf("Account trie total depth = %v, want %v", stats.accountTrieLeaves, expectedAccountTrieLeaves) - } - - // Check storage trie depth - if stats.storageTrieLeaves != expectedStorageTrieLeaves { - t.Errorf("Storage trie total depth = %v, want %v", stats.storageTrieLeaves, expectedStorageTrieLeaves) + if got, want := stats.storageTrie.LeafDepths(), expectedLeaves(tt.expectedStorageLeaves); got != want { + t.Errorf("storage trie leaves = %v, want %v", got, want) } }) } } +func TestWitnessStatsStorageTrieAggregation(t *testing.T) { + stats := NewWitnessStats() + ownerA := common.HexToHash("0xa") + ownerB := common.HexToHash("0xb") + + stats.Add(map[string][]byte{ + "a": []byte("data1"), + "ab": []byte("data2"), + "abc": []byte("data3"), + }, ownerA) + stats.Add(map[string][]byte{ + "xy": []byte("data4"), + }, ownerA) + stats.Add(map[string][]byte{ + "1": []byte("data5"), + "12": []byte("data6"), + "123": []byte("data7"), + "124": []byte("data8"), + }, ownerB) + + if got, want := stats.storageTrie.LeafDepths(), expectedLeaves(map[int]int64{2: 1, 3: 3}); got != want { + t.Errorf("storage leaves = %v, want %v", got, want) + } + if got, want := stats.accountTrie.LeafDepths(), expectedLeaves(nil); got != want { + t.Errorf("account leaves = %v, want %v", got, want) + } +} + +func TestWitnessStatsPanicsOnDeepLeaf(t *testing.T) { + stats := NewWitnessStats() + + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for depth >= 16") + } + }() + stats.Add(map[string][]byte{strings.Repeat("a", 16): []byte("data")}, common.Hash{}) +} + func TestWitnessStatsMinMax(t *testing.T) { stats := NewWitnessStats() - // Add some account trie nodes with varying depths + // Add some account trie nodes with varying depths. stats.Add(map[string][]byte{ "a": []byte("data1"), "ab": []byte("data2"), @@ -152,21 +188,21 @@ func TestWitnessStatsMinMax(t *testing.T) { "abcde": []byte("data5"), }, common.Hash{}) - // Only "abcde" is a leaf (depth 5) - for i, v := range stats.accountTrieLeaves { + // Only "abcde" is a leaf (depth 5). + for i, v := range stats.accountTrie.LeafDepths() { if v != 0 && i != 5 { t.Errorf("leaf found at invalid depth %d", i) } } - // Add more leaves with different depths + // Add more leaves with different depths. stats.Add(map[string][]byte{ "x": []byte("data6"), "yz": []byte("data7"), }, common.Hash{}) - // Now we have leaves at depths 1, 2, and 5 - for i, v := range stats.accountTrieLeaves { + // Now we have leaves at depths 1, 2, and 5. + for i, v := range stats.accountTrie.LeafDepths() { if v != 0 && (i != 5 && i != 2 && i != 1) { t.Errorf("leaf found at invalid depth %d", i) } @@ -176,7 +212,7 @@ func TestWitnessStatsMinMax(t *testing.T) { func TestWitnessStatsAverage(t *testing.T) { stats := NewWitnessStats() - // Add nodes that will create leaves at depths 2, 3, and 4 + // Add nodes that will create leaves at depths 2, 3, and 4. stats.Add(map[string][]byte{ "aa": []byte("data1"), "bb": []byte("data2"), @@ -184,22 +220,22 @@ func TestWitnessStatsAverage(t *testing.T) { "dddd": []byte("data4"), }, common.Hash{}) - // All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples + // All are leaves: 2 + 2 + 3 + 4 = 11 total, 4 samples. expectedAvg := int64(11) / int64(4) var actualAvg, totalSamples int64 - for i, c := range stats.accountTrieLeaves { + for i, c := range stats.accountTrie.LeafDepths() { actualAvg += c * int64(i) totalSamples += c } actualAvg = actualAvg / totalSamples if actualAvg != expectedAvg { - t.Errorf("Account trie average depth = %d, want %d", actualAvg, expectedAvg) + t.Errorf("account trie average depth = %d, want %d", actualAvg, expectedAvg) } } func BenchmarkWitnessStatsAdd(b *testing.B) { - // Create a realistic trie node structure + // Create a realistic trie node structure. nodes := make(map[string][]byte) for i := 0; i < 100; i++ { base := string(rune('a' + i%26)) diff --git a/core/tracing/hooks.go b/core/tracing/hooks.go index c85abe6482..6d0131ce70 100644 --- a/core/tracing/hooks.go +++ b/core/tracing/hooks.go @@ -358,7 +358,7 @@ const ( // this generates an increase in gas. There is at most one of such gas change per transaction. GasChangeTxRefunds GasChangeReason = 3 // GasChangeTxLeftOverReturned is the amount of gas left over at the end of transaction's execution that will be returned - // to the chain. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas + // to the account. This change will always be a negative change as we "drain" left over gas towards 0. If there was no gas // left at the end of execution, no such even will be emitted. The returned gas's value in Wei is returned to caller. // There is at most one of such gas change per transaction. GasChangeTxLeftOverReturned GasChangeReason = 4 diff --git a/core/tracing/journal.go b/core/tracing/journal.go index 62a70d6c27..560c937115 100644 --- a/core/tracing/journal.go +++ b/core/tracing/journal.go @@ -155,10 +155,18 @@ func (j *journal) OnBalanceChange(addr common.Address, prev, new *big.Int, reaso } func (j *journal) OnNonceChangeV2(addr common.Address, prev, new uint64, reason NonceChangeReason) { - // When a contract is created, the nonce of the creator is incremented. - // This change is not reverted when the creation fails. - if reason != NonceChangeContractCreator { - j.entries = append(j.entries, nonceChange{addr: addr, prev: prev, new: new}) + j.entries = append(j.entries, nonceChange{addr: addr, prev: prev, new: new}) + if reason == NonceChangeContractCreator { + // When a contract is created via CREATE/CREATE2, the creator's nonce is + // incremented. The EVM does not revert this when the CREATE frame itself + // fails (the nonce change happens before the EVM snapshot). However, if + // a parent frame reverts, the nonce must be reverted along with everything + // else. + // + // To achieve this, advance the current frame's revision point past this + // entry. The CREATE frame's revert won't touch it (it's below the revision), + // but a parent frame's revert will (it's above the parent's revision). + j.revisions[len(j.revisions)-1] = len(j.entries) } if j.hooks.OnNonceChangeV2 != nil { j.hooks.OnNonceChangeV2(addr, prev, new, reason) diff --git a/core/tracing/journal_test.go b/core/tracing/journal_test.go index e00447f5f3..488d192502 100644 --- a/core/tracing/journal_test.go +++ b/core/tracing/journal_test.go @@ -219,6 +219,42 @@ func TestNonceIncOnCreate(t *testing.T) { } } +// TestNonceIncOnCreateParentReverts checks that the creator's nonce increment +// from CREATE survives the CREATE frame's own revert but is properly reverted +// when the parent call frame reverts. +func TestNonceIncOnCreateParentReverts(t *testing.T) { + const opCREATE = 0xf0 + + tr := &testTracer{t: t} + wr, err := WrapWithJournal(&Hooks{OnNonceChange: tr.OnNonceChange}) + if err != nil { + t.Fatalf("failed to wrap test tracer: %v", err) + } + + addr := common.HexToAddress("0x1234") + { + // Parent call frame + wr.OnEnter(0, 0, addr, addr, nil, 1000, big.NewInt(0)) + { + // CREATE frame — creator nonce incremented, then CREATE reverts + wr.OnEnter(1, opCREATE, addr, addr, nil, 1000, big.NewInt(0)) + wr.OnNonceChangeV2(addr, 0, 1, NonceChangeContractCreator) + wr.OnExit(1, nil, 100, errors.New("revert"), true) + } + // After CREATE reverts, nonce should still be 1 + if tr.nonce != 1 { + t.Fatalf("nonce after CREATE revert: got %v, want 1", tr.nonce) + } + // Parent frame also reverts + wr.OnExit(0, nil, 150, errors.New("revert"), true) + } + + // After parent reverts, nonce should be back to 0 + if tr.nonce != 0 { + t.Fatalf("nonce after parent revert: got %v, want 0", tr.nonce) + } +} + func TestOnNonceChangeV2(t *testing.T) { tr := &testTracer{t: t} wr, err := WrapWithJournal(&Hooks{OnNonceChangeV2: tr.OnNonceChangeV2}) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 4144c3a1b2..415f5bc48f 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -104,6 +104,16 @@ const ( // maxGappedTxs is the maximum number of gapped transactions kept overall. // This is a safety limit to avoid DoS vectors. maxGapped = 128 + + // notifyThreshold is the eviction priority threshold above which a transaction + // is considered close enough to being includable to be announced to peers. + // Setting this to zero will disable announcements for anyting not immediately + // includable. Setting it to -1 allows transactions that are close to being + // includable, maybe already in the next block if fees go down, to be announced. + + // Note, this threshold is in the abstract eviction priority space, so its + // meaning depends on the current basefee/blobfee and the transaction's fees. + announceThreshold = -1 ) // blobTxMeta is the minimal subset of types.BlobTx necessary to validate and @@ -115,6 +125,8 @@ type blobTxMeta struct { vhashes []common.Hash // Blob versioned hashes to maintain the lookup table version byte // Blob transaction version to determine proof type + announced bool // Whether the tx has been announced to listeners + id uint64 // Storage ID in the pool's persistent store storageSize uint32 // Byte size in the pool's persistent store size uint64 // RLP-encoded size of transaction including the attached blob @@ -159,7 +171,7 @@ func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transac blobGas: tx.BlobGas(), } meta.basefeeJumps = dynamicFeeJumps(meta.execFeeCap) - meta.blobfeeJumps = dynamicFeeJumps(meta.blobFeeCap) + meta.blobfeeJumps = dynamicBlobFeeJumps(meta.blobFeeCap) return meta } @@ -210,6 +222,14 @@ func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transac // via a normal transaction. It should nonetheless be high enough to support // resurrecting reorged transactions. Perhaps 4-16. // +// - It is not the role of the blobpool to serve as a storage for limit orders +// below market: blob transactions with fee caps way below base fee or blob fee. +// Therefore, the propagation of blob transactions that are far from being +// includable is suppressed. The pool will only announce blob transactions that +// are close to being includable (based on the current fees and the transaction's +// fee caps), and will delay the announcement of blob transactions that are far +// from being includable until base fee and/or blob fee is reduced. +// // - Local txs are meaningless. Mining pools historically used local transactions // for payouts or for backdoor deals. With 1559 in place, the basefee usually // dominates the final price, so 0 or non-0 tip doesn't change much. Blob txs @@ -281,47 +301,54 @@ func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transac // solve after every block. // // - The first observation is that comparing 1559 base fees or 4844 blob fees -// needs to happen in the context of their dynamism. Since these fees jump -// up or down in ~1.125 multipliers (at max) across blocks, comparing fees -// in two transactions should be based on log1.125(fee) to eliminate noise. +// needs to happen in the context of their dynamism. Since base fees are +// adjusted continuously and fluctuate, and we want to optimize for effective +// miner fees, it is better to disregard small base fee cap differences. +// Instead of considering the exact fee cap values, we should group +// transactions into buckets based on fee cap values, allowing us to use +// the miner tip meaningfully as a splitter inside a bucket. // -// - The second observation is that the basefee and blobfee move independently, -// so there's no way to split mixed txs on their own (A has higher base fee, -// B has higher blob fee). Rather than look at the absolute fees, the useful -// metric is the max time it can take to exceed the transaction's fee caps. +// To create these buckets, rather than looking at the absolute fee +// differences, the useful metric is the max time it can take to exceed the +// transaction's fee caps. Base fee changes are multiplicative, so we use a +// logarithmic scale. Fees jumps up or down in ~1.125 multipliers at max +// across blocks, so we use log1.125(fee) and rounding to eliminate noise. // Specifically, we're interested in the number of jumps needed to go from // the current fee to the transaction's cap: // -// jumps = log1.125(txfee) - log1.125(basefee) +// jumps = floor(log1.125(txfee) - log1.125(basefee)) // -// - The third observation is that the base fee tends to hover around rather -// than swing wildly. The number of jumps needed from the current fee starts -// to get less relevant the higher it is. To remove the noise here too, the -// pool will use log(jumps) as the delta for comparing transactions. +// For blob fees, EIP-7892 changed the ratio of target to max blobs, and +// with that also the maximum blob fee decrease in a slot from 1.125 to +// approx 1.17. therefore, we use: // -// delta = sign(jumps) * log(abs(jumps)) +// blobfeeJumps = floor(log1.17(txBlobfee) - log1.17(blobfee)) // -// - To establish a total order, we need to reduce the dimensionality of the +// - The second observation is that when ranking executable blob txs, it +// does not make sense to grant a later eviction priority to txs with high +// fee caps since it could enable pool wars. As such, any positive priority +// will be grouped together. +// +// priority = min(jumps, 0) +// +// - The third observation is that the basefee and blobfee move independently, +// so there's no way to split mixed txs on their own (A has higher base fee, +// B has higher blob fee). +// +// To establish a total order, we need to reduce the dimensionality of the // two base fees (log jumps) to a single value. The interesting aspect from // the pool's perspective is how fast will a tx get executable (fees going // down, crossing the smaller negative jump counter) or non-executable (fees // going up, crossing the smaller positive jump counter). As such, the pool // cares only about the min of the two delta values for eviction priority. // -// priority = min(deltaBasefee, deltaBlobfee) +// priority = min(deltaBasefee, deltaBlobfee, 0) // // - The above very aggressive dimensionality and noise reduction should result // in transaction being grouped into a small number of buckets, the further // the fees the larger the buckets. This is good because it allows us to use // the miner tip meaningfully as a splitter. // -// - For the scenario where the pool does not contain non-executable blob txs -// anymore, it does not make sense to grant a later eviction priority to txs -// with high fee caps since it could enable pool wars. As such, any positive -// priority will be grouped together. -// -// priority = min(deltaBasefee, deltaBlobfee, 0) -// // Optimisation tradeoffs: // // - Eviction relies on 3 fee minimums per account (exec tip, exec cap and blob @@ -470,6 +497,20 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser } p.evict = newPriceHeap(basefee, blobfee, p.index) + // Guess what was announced. This is needed because we don't want to + // participate in the diffusion of transactions where inclusion is blocked by + // a low base fee transaction. Since we don't persist that info, the best + // we can do is to assume that anything that could have been announced + // at current prices, actually was. + for addr := range p.index { + for _, tx := range p.index[addr] { + tx.announced = p.isAnnouncable(tx) + if !tx.announced { + break + } + } + } + // Pool initialized, attach the blob limbo to it to track blobs included // recently but not yet finalized p.limbo, err = newLimbo(p.chain.Config(), limbodir) @@ -517,6 +558,7 @@ func (p *BlobPool) Close() error { // parseTransaction is a callback method on pool creation that gets called for // each transaction on disk to create the in-memory metadata index. +// Announced state is not initialized here, it needs to be iniitalized seprately. func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { tx := new(types.Transaction) if err := rlp.DecodeBytes(blob, tx); err != nil { @@ -893,6 +935,37 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) { } p.evict.reinit(basefee, blobfee, false) + // Announce transactions that became announcable due to fee changes + var announcable []*types.Transaction + for addr, txs := range p.index { + for i, meta := range txs { + if !meta.announced && (i == 0 || txs[i-1].announced) && p.isAnnouncable(meta) { + // Load the full transaction and strip the sidecar before announcing + // TODO: this is a bit ugly, as we have everything needed in meta already + data, err := p.store.Get(meta.id) + // Technically, we are supposed to set announced only if Get is successful. + // However, Get failing here indicates a more serious issue (data loss), + // so we set announced anyway to avoid repeated attempts. + meta.announced = true + if err != nil { + log.Error("Blobs missing for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err) + continue + } + var tx types.Transaction + if err = rlp.DecodeBytes(data, &tx); err != nil { + log.Error("Blobs corrupted for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err) + continue + } + announcable = append(announcable, tx.WithoutBlobTxSidecar()) + log.Trace("Blob transaction now announcable", "from", addr, "nonce", meta.nonce, "id", meta.id, "hash", tx.Hash()) + } + } + } + if len(announcable) > 0 { + p.discoverFeed.Send(core.NewTxsEvent{Txs: announcable}) + } + + // Update the basefee and blobfee metrics basefeeGauge.Update(int64(basefee.Uint64())) blobfeeGauge.Update(int64(blobfee.Uint64())) p.updateStorageMetrics() @@ -1495,17 +1568,12 @@ func (p *BlobPool) AvailableBlobs(vhashes []common.Hash) int { // Add inserts a set of blob transactions into the pool if they pass validation (both // consensus validity and pool restrictions). func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error { - var ( - errs = make([]error, len(txs)) - adds = make([]*types.Transaction, 0, len(txs)) - ) + errs := make([]error, len(txs)) for i, tx := range txs { if errs[i] = p.ValidateTxBasics(tx); errs[i] != nil { continue } - if errs[i] = p.add(tx); errs[i] == nil { - adds = append(adds, tx.WithoutBlobTxSidecar()) - } + errs[i] = p.add(tx) } return errs } @@ -1697,9 +1765,15 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error addValidMeter.Mark(1) - // Notify all listeners of the new arrival - p.discoverFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}}) - p.insertFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}}) + // Transaction was addded successfully, but we only announce if it is (close to being) + // includable and the previous one was already announced. + if p.isAnnouncable(meta) && (meta.nonce == next || (len(txs) > 1 && txs[offset-1].announced)) { + meta.announced = true + p.discoverFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}}) + p.insertFeed.Send(core.NewTxsEvent{Txs: []*types.Transaction{tx.WithoutBlobTxSidecar()}}) + } else { + log.Trace("Blob transaction not announcable yet", "hash", tx.Hash(), "nonce", tx.Nonce()) + } //check the gapped queue for this account and try to promote if gtxs, ok := p.gapped[from]; checkGapped && ok && len(gtxs) > 0 { @@ -2023,6 +2097,12 @@ func (p *BlobPool) evictGapped() { } } +// isAnnouncable checks whether a transaction is announcable based on its +// fee parameters and announceThreshold. +func (p *BlobPool) isAnnouncable(meta *blobTxMeta) bool { + return evictionPriority(p.evict.basefeeJumps, meta.basefeeJumps, p.evict.blobfeeJumps, meta.blobfeeJumps) >= announceThreshold +} + // Stats retrieves the current pool stats, namely the number of pending and the // number of queued (non-executable) transactions. func (p *BlobPool) Stats() (int, int) { diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 4bb3567b69..6580a339e3 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -830,8 +830,8 @@ func TestOpenIndex(t *testing.T) { //blobfeeJumps = []float64{34.023, 35.570, 36.879, 29.686, 26.243, 20.358} // log 1.125 (blob fee cap) evictExecTipCaps = []uint64{10, 10, 5, 5, 1, 1} - evictExecFeeJumps = []float64{39.098, 38.204, 38.204, 19.549, 19.549, 19.549} // min(log 1.125 (exec fee cap)) - evictBlobFeeJumps = []float64{34.023, 34.023, 34.023, 29.686, 26.243, 20.358} // min(log 1.125 (blob fee cap)) + evictExecFeeJumps = []float64{39.098, 38.204, 38.204, 19.549, 19.549, 19.549} // min(log 1.125 (exec fee cap)) + evictBlobFeeJumps = []float64{25.517256, 25.517256, 25.517256, 22.264502, 19.682646, 15.268934} // min(log 1.17 (blob fee cap)) totalSpent = uint256.NewInt(21000*(100+90+200+10+80+300) + blobSize*(55+66+77+33+22+11) + 100*6) // 21000 gas x price + 128KB x blobprice + value ) @@ -1751,8 +1751,8 @@ func TestAdd(t *testing.T) { // Create a blob pool out of the pre-seeded dats chain := &testBlockChain{ config: params.MainnetChainConfig, - basefee: uint256.NewInt(1050), - blobfee: uint256.NewInt(105), + basefee: uint256.NewInt(1), + blobfee: uint256.NewInt(1), statedb: statedb, } pool := New(Config{Datadir: storage}, chain, nil) diff --git a/core/txpool/blobpool/evictheap.go b/core/txpool/blobpool/evictheap.go index 722a71bc9b..a46b8e9a6f 100644 --- a/core/txpool/blobpool/evictheap.go +++ b/core/txpool/blobpool/evictheap.go @@ -67,7 +67,7 @@ func newPriceHeap(basefee *uint256.Int, blobfee *uint256.Int, index map[common.A func (h *evictHeap) reinit(basefee *uint256.Int, blobfee *uint256.Int, force bool) { // If the update is mostly the same as the old, don't sort pointlessly basefeeJumps := dynamicFeeJumps(basefee) - blobfeeJumps := dynamicFeeJumps(blobfee) + blobfeeJumps := dynamicBlobFeeJumps(blobfee) if !force && math.Abs(h.basefeeJumps-basefeeJumps) < 0.01 && math.Abs(h.blobfeeJumps-blobfeeJumps) < 0.01 { // TODO(karalabe): 0.01 enough, maybe should be smaller? Maybe this optimization is moot? return @@ -95,13 +95,7 @@ func (h *evictHeap) Less(i, j int) bool { lastJ := txsJ[len(txsJ)-1] prioI := evictionPriority(h.basefeeJumps, lastI.evictionExecFeeJumps, h.blobfeeJumps, lastI.evictionBlobFeeJumps) - if prioI > 0 { - prioI = 0 - } prioJ := evictionPriority(h.basefeeJumps, lastJ.evictionExecFeeJumps, h.blobfeeJumps, lastJ.evictionBlobFeeJumps) - if prioJ > 0 { - prioJ = 0 - } if prioI == prioJ { return lastI.evictionExecTip.Lt(lastJ.evictionExecTip) } diff --git a/core/txpool/blobpool/evictheap_test.go b/core/txpool/blobpool/evictheap_test.go index de4076e298..112fa77a01 100644 --- a/core/txpool/blobpool/evictheap_test.go +++ b/core/txpool/blobpool/evictheap_test.go @@ -109,22 +109,22 @@ func TestPriceHeapSorting(t *testing.T) { order: []int{3, 2, 1, 0, 4, 5, 6}, }, // If both basefee and blobfee is specified, sort by the larger distance - // of the two from the current network conditions, splitting same (loglog) + // of the two from the current network conditions, splitting same // ones via the tip. // - // Basefee: 1000 - // Blobfee: 100 + // Basefee: 1000 , jumps: 888, 790, 702, 624 + // Blobfee: 100 , jumps: 85, 73, 62, 53 // - // Tx #0: (800, 80) - 2 jumps below both => priority -1 - // Tx #1: (630, 63) - 4 jumps below both => priority -2 - // Tx #2: (800, 63) - 2 jumps below basefee, 4 jumps below blobfee => priority -2 (blob penalty dominates) - // Tx #3: (630, 80) - 4 jumps below basefee, 2 jumps below blobfee => priority -2 (base penalty dominates) + // Tx #0: (800, 80) - 2 jumps below both => priority -2 + // Tx #1: (630, 55) - 4 jumps below both => priority -4 + // Tx #2: (800, 55) - 2 jumps below basefee, 4 jumps below blobfee => priority -4 (blob penalty dominates) + // Tx #3: (630, 80) - 4 jumps below basefee, 2 jumps below blobfee => priority -4 (base penalty dominates) // // Txs 1, 2, 3 share the same priority, split via tip, prefer 0 as the best { execTips: []uint64{1, 2, 3, 4}, execFees: []uint64{800, 630, 800, 630}, - blobFees: []uint64{80, 63, 63, 80}, + blobFees: []uint64{80, 55, 55, 80}, basefee: 1000, blobfee: 100, order: []int{1, 2, 3, 0}, @@ -142,7 +142,7 @@ func TestPriceHeapSorting(t *testing.T) { blobFee = uint256.NewInt(tt.blobFees[j]) basefeeJumps = dynamicFeeJumps(execFee) - blobfeeJumps = dynamicFeeJumps(blobFee) + blobfeeJumps = dynamicBlobFeeJumps(blobFee) ) index[addr] = []*blobTxMeta{{ id: uint64(j), @@ -201,7 +201,7 @@ func benchmarkPriceHeapReinit(b *testing.B, datacap uint64) { blobFee = uint256.NewInt(rnd.Uint64()) basefeeJumps = dynamicFeeJumps(execFee) - blobfeeJumps = dynamicFeeJumps(blobFee) + blobfeeJumps = dynamicBlobFeeJumps(blobFee) ) index[addr] = []*blobTxMeta{{ id: uint64(i), @@ -277,7 +277,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) { blobFee = uint256.NewInt(rnd.Uint64()) basefeeJumps = dynamicFeeJumps(execFee) - blobfeeJumps = dynamicFeeJumps(blobFee) + blobfeeJumps = dynamicBlobFeeJumps(blobFee) ) index[addr] = []*blobTxMeta{{ id: uint64(i), @@ -308,7 +308,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) { blobFee = uint256.NewInt(rnd.Uint64()) basefeeJumps = dynamicFeeJumps(execFee) - blobfeeJumps = dynamicFeeJumps(blobFee) + blobfeeJumps = dynamicBlobFeeJumps(blobFee) ) metas[i] = &blobTxMeta{ id: uint64(int(blobs) + i), diff --git a/core/txpool/blobpool/priority.go b/core/txpool/blobpool/priority.go index 7ae7f92def..d7e8789ce7 100644 --- a/core/txpool/blobpool/priority.go +++ b/core/txpool/blobpool/priority.go @@ -18,7 +18,6 @@ package blobpool import ( "math" - "math/bits" "github.com/holiman/uint256" ) @@ -26,6 +25,13 @@ import ( // log1_125 is used in the eviction priority calculation. var log1_125 = math.Log(1.125) +// log1_17 is used in the eviction priority calculation for blob fees. +// EIP-7892 (BPO) changed the ratio of target to max blobs, and with that +// also the maximum blob fee decrease in a slot from 1.125 to approx 1.17 . +// Since we want priorities to approximate time, we should change our log +// calculation for blob fees. +var log1_17 = log1_125 * 4 / 3 + // evictionPriority calculates the eviction priority based on the algorithm // described in the BlobPool docs for both fee components. // @@ -36,23 +42,20 @@ func evictionPriority(basefeeJumps float64, txBasefeeJumps, blobfeeJumps, txBlob basefeePriority = evictionPriority1D(basefeeJumps, txBasefeeJumps) blobfeePriority = evictionPriority1D(blobfeeJumps, txBlobfeeJumps) ) - if basefeePriority < blobfeePriority { - return basefeePriority - } - return blobfeePriority + return min(0, basefeePriority, blobfeePriority) } // evictionPriority1D calculates the eviction priority based on the algorithm // described in the BlobPool docs for a single fee component. func evictionPriority1D(basefeeJumps float64, txfeeJumps float64) int { jumps := txfeeJumps - basefeeJumps - if int(jumps) == 0 { - return 0 // can't log2 0 + if jumps <= 0 { + return int(math.Floor(jumps)) } - if jumps < 0 { - return -intLog2(uint(-math.Floor(jumps))) - } - return intLog2(uint(math.Ceil(jumps))) + // We only use the negative part for ordering. The positive part is only used + // for threshold comparison (with a negative threshold), so the value is almost + // irrelevant, as long as it's positive. + return int((math.Ceil(jumps))) } // dynamicFeeJumps calculates the log1.125(fee), namely the number of fee jumps @@ -70,21 +73,9 @@ func dynamicFeeJumps(fee *uint256.Int) float64 { return math.Log(fee.Float64()) / log1_125 } -// intLog2 is a helper to calculate the integral part of a log2 of an unsigned -// integer. It is a very specific calculation that's not particularly useful in -// general, but it's what we need here (it's fast). -func intLog2(n uint) int { - switch { - case n == 0: - panic("log2(0) is undefined") - - case n < 2048: - return bits.UintSize - bits.LeadingZeros(n) - 1 - - default: - // The input is log1.125(uint256) = log2(uint256) / log2(1.125). At the - // most extreme, log2(uint256) will be a bit below 257, and the constant - // log2(1.125) ~= 0.17. The larges input thus is ~257 / ~0.17 ~= ~1511. - panic("dynamic fee jump diffs cannot reach this") +func dynamicBlobFeeJumps(fee *uint256.Int) float64 { + if fee.IsZero() { + return 0 // can't log2 zero, should never happen outside tests, but don't choke } + return math.Log(fee.Float64()) / log1_17 } diff --git a/core/txpool/blobpool/priority_test.go b/core/txpool/blobpool/priority_test.go index 1eaee6d7df..47b3a5375f 100644 --- a/core/txpool/blobpool/priority_test.go +++ b/core/txpool/blobpool/priority_test.go @@ -30,12 +30,12 @@ func TestPriorityCalculation(t *testing.T) { txfee uint64 result int }{ - {basefee: 7, txfee: 10, result: 2}, // 3.02 jumps, 4 ceil, 2 log2 - {basefee: 17_200_000_000, txfee: 17_200_000_000, result: 0}, // 0 jumps, special case 0 log2 - {basefee: 9_853_941_692, txfee: 11_085_092_510, result: 0}, // 0.99 jumps, 1 ceil, 0 log2 - {basefee: 11_544_106_391, txfee: 10_356_781_100, result: 0}, // -0.92 jumps, -1 floor, 0 log2 - {basefee: 17_200_000_000, txfee: 7, result: -7}, // -183.57 jumps, -184 floor, -7 log2 - {basefee: 7, txfee: 17_200_000_000, result: 7}, // 183.57 jumps, 184 ceil, 7 log2 + {basefee: 7, txfee: 10, result: 4}, // 3.02 jumps, 4 ceil + {basefee: 17_200_000_000, txfee: 17_200_000_000, result: 0}, // 0 jumps, special case 0 + {basefee: 9_853_941_692, txfee: 11_085_092_510, result: 1}, // 0.99 jumps, 1 ceil + {basefee: 11_544_106_391, txfee: 10_356_781_100, result: -1}, // -0.92 jumps, -1 floor + {basefee: 17_200_000_000, txfee: 7, result: -184}, // -183.57 jumps, -184 floor + {basefee: 7, txfee: 17_200_000_000, result: 184}, // 183.57 jumps, 184 ceil } for i, tt := range tests { var ( @@ -69,7 +69,7 @@ func BenchmarkPriorityCalculation(b *testing.B) { blobfee := uint256.NewInt(123_456_789_000) // Completely random, no idea what this will be basefeeJumps := dynamicFeeJumps(basefee) - blobfeeJumps := dynamicFeeJumps(blobfee) + blobfeeJumps := dynamicBlobFeeJumps(blobfee) // The transaction's fee cap and blob fee cap are constant across the life // of the transaction, so we can pre-calculate and cache them. @@ -77,7 +77,7 @@ func BenchmarkPriorityCalculation(b *testing.B) { txBlobfeeJumps := make([]float64, b.N) for i := 0; i < b.N; i++ { txBasefeeJumps[i] = dynamicFeeJumps(uint256.NewInt(rnd.Uint64())) - txBlobfeeJumps[i] = dynamicFeeJumps(uint256.NewInt(rnd.Uint64())) + txBlobfeeJumps[i] = dynamicBlobFeeJumps(uint256.NewInt(rnd.Uint64())) } b.ResetTimer() b.ReportAllocs() diff --git a/core/txpool/legacypool/list_test.go b/core/txpool/legacypool/list_test.go index 8587c66f7d..dc03def26c 100644 --- a/core/txpool/legacypool/list_test.go +++ b/core/txpool/legacypool/list_test.go @@ -68,6 +68,43 @@ func TestListAddVeryExpensive(t *testing.T) { } } +// TestPriceHeapCmp tests that the price heap comparison function works as intended. +// It also tests combinations where the basefee is higher than the gas fee cap, which +// are useful to sort in the mempool to support basefee changes. +func TestPriceHeapCmp(t *testing.T) { + key, _ := crypto.GenerateKey() + txs := []*types.Transaction{ + // nonce, gaslimit, gasfee, gastip + dynamicFeeTx(0, 1000, big.NewInt(2), big.NewInt(1), key), + dynamicFeeTx(0, 1000, big.NewInt(1), big.NewInt(2), key), + dynamicFeeTx(0, 1000, big.NewInt(1), big.NewInt(1), key), + dynamicFeeTx(0, 1000, big.NewInt(1), big.NewInt(0), key), + } + + // create priceHeap + ph := &priceHeap{} + + // now set the basefee on the heap + for _, basefee := range []uint64{0, 1, 2, 3} { + ph.baseFee = uint256.NewInt(basefee) + + for i := 0; i < len(txs); i++ { + for j := 0; j < len(txs); j++ { + switch { + case i == j: + if c := ph.cmp(txs[i], txs[j]); c != 0 { + t.Errorf("tx %d should be equal priority to tx %d with basefee %d (cmp=%d)", i, j, basefee, c) + } + case i < j: + if c := ph.cmp(txs[i], txs[j]); c != 1 { + t.Errorf("tx %d vs tx %d comparison inconsistent with basefee %d (cmp=%d)", i, j, basefee, c) + } + } + } + } + } +} + func BenchmarkListAdd(b *testing.B) { // Generate a list of transactions to insert key, _ := crypto.GenerateKey() diff --git a/core/txpool/validation.go b/core/txpool/validation.go index e0a333dfa5..13b1bfa312 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -86,8 +87,10 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types return fmt.Errorf("%w: type %d rejected, pool not yet in Prague", core.ErrTxTypeNotSupported, tx.Type()) } // Check whether the init code size has been exceeded - if rules.IsShanghai && tx.To() == nil && len(tx.Data()) > params.MaxInitCodeSize { - return fmt.Errorf("%w: code size %v, limit %v", core.ErrMaxInitCodeSizeExceeded, len(tx.Data()), params.MaxInitCodeSize) + if tx.To() == nil { + if err := vm.CheckMaxInitCodeSize(&rules, uint64(len(tx.Data()))); err != nil { + return err + } } if rules.IsOsaka && tx.Gas() > params.MaxTxGas { return fmt.Errorf("%w (cap: %d, tx: %d)", core.ErrGasLimitTooHigh, params.MaxTxGas, tx.Gas()) diff --git a/core/types.go b/core/types.go index bed20802ab..87bbfcff58 100644 --- a/core/types.go +++ b/core/types.go @@ -17,6 +17,7 @@ package core import ( + "context" "sync/atomic" "github.com/ethereum/go-ethereum/core/state" @@ -48,7 +49,7 @@ type Processor interface { // Process processes the state changes according to the Ethereum rules by running // the transaction messages using the statedb and applying any rewards to both // the processor (coinbase) and any included uncles. - Process(block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) + Process(ctx context.Context, block *types.Block, statedb *state.StateDB, cfg vm.Config) (*ProcessResult, error) } // ProcessResult contains the values computed by Process. diff --git a/core/types/bal/bal.go b/core/types/bal/bal.go index fca54f7681..86dc8e5426 100644 --- a/core/types/bal/bal.go +++ b/core/types/bal/bal.go @@ -24,13 +24,6 @@ import ( "github.com/holiman/uint256" ) -// CodeChange contains the runtime bytecode deployed at an address and the -// transaction index where the deployment took place. -type CodeChange struct { - TxIndex uint16 - Code []byte `json:"code,omitempty"` -} - // ConstructionAccountAccess contains post-block account state for mutations as well as // all storage keys that were read during execution. It is used when building block // access list during execution. @@ -55,9 +48,9 @@ type ConstructionAccountAccess struct { // by tx index. NonceChanges map[uint16]uint64 `json:"nonceChanges,omitempty"` - // CodeChange is only set for contract accounts which were deployed in - // the block. - CodeChange *CodeChange `json:"codeChange,omitempty"` + // CodeChange contains the post-state contract code of an account keyed + // by tx index. + CodeChange map[uint16][]byte `json:"codeChange,omitempty"` } // NewConstructionAccountAccess initializes the account access object. @@ -67,6 +60,7 @@ func NewConstructionAccountAccess() *ConstructionAccountAccess { StorageReads: make(map[common.Hash]struct{}), BalanceChanges: make(map[uint16]*uint256.Int), NonceChanges: make(map[uint16]uint64), + CodeChange: make(map[uint16][]byte), } } @@ -120,10 +114,8 @@ func (b *ConstructionBlockAccessList) CodeChange(address common.Address, txIndex if _, ok := b.Accounts[address]; !ok { b.Accounts[address] = NewConstructionAccountAccess() } - b.Accounts[address].CodeChange = &CodeChange{ - TxIndex: txIndex, - Code: bytes.Clone(code), - } + // TODO(rjl493456442) is it essential to deep-copy the code? + b.Accounts[address].CodeChange[txIndex] = bytes.Clone(code) } // NonceChange records tx post-state nonce of any contract-like accounts whose @@ -170,12 +162,11 @@ func (b *ConstructionBlockAccessList) Copy() *ConstructionBlockAccessList { aaCopy.BalanceChanges = balances aaCopy.NonceChanges = maps.Clone(aa.NonceChanges) - if aa.CodeChange != nil { - aaCopy.CodeChange = &CodeChange{ - TxIndex: aa.CodeChange.TxIndex, - Code: bytes.Clone(aa.CodeChange.Code), - } + codes := make(map[uint16][]byte, len(aa.CodeChange)) + for index, code := range aa.CodeChange { + codes[index] = bytes.Clone(code) } + aaCopy.CodeChange = codes res.Accounts[addr] = &aaCopy } return &res diff --git a/core/types/bal/bal_encoding.go b/core/types/bal/bal_encoding.go index 24dfafa083..1b1406ea32 100644 --- a/core/types/bal/bal_encoding.go +++ b/core/types/bal/bal_encoding.go @@ -119,6 +119,13 @@ func (e *encodingSlotWrites) validate() error { return errors.New("storage write tx indices not in order") } +// encodingCodeChange contains the runtime bytecode deployed at an address +// and the transaction index where the deployment took place. +type encodingCodeChange struct { + TxIndex uint16 `ssz-size:"2"` + Code []byte `ssz-max:"300000"` // TODO(rjl493456442) shall we put the limit here? The limit will be increased gradually +} + // AccountAccess is the encoding format of ConstructionAccountAccess. type AccountAccess struct { Address [20]byte `ssz-size:"20"` // 20-byte Ethereum address @@ -126,7 +133,7 @@ type AccountAccess struct { StorageReads [][32]byte `ssz-max:"300000"` // Read-only storage keys BalanceChanges []encodingBalanceChange `ssz-max:"300000"` // Balance changes ([tx_index -> post_balance]) NonceChanges []encodingAccountNonce `ssz-max:"300000"` // Nonce changes ([tx_index -> new_nonce]) - Code []CodeChange `ssz-max:"1"` // Code changes ([tx_index -> new_code]) + CodeChanges []encodingCodeChange `ssz-max:"300000"` // Code changes ([tx_index -> new_code]) } // validate converts the account accesses out of encoding format. @@ -166,9 +173,16 @@ func (e *AccountAccess) validate() error { return errors.New("nonce changes not in ascending order by tx index") } - // Convert code change - if len(e.Code) == 1 { - if len(e.Code[0].Code) > params.MaxCodeSize { + // Check the code changes are sorted in order + if !slices.IsSortedFunc(e.CodeChanges, func(a, b encodingCodeChange) int { + return cmp.Compare[uint16](a.TxIndex, b.TxIndex) + }) { + return errors.New("code changes not in ascending order by tx index") + } + for _, change := range e.CodeChanges { + // TODO(rjl493456442): This check should be fork-aware, since the limit may + // differ across forks. + if len(change.Code) > params.MaxCodeSize { return errors.New("code change contained oversized code") } } @@ -182,6 +196,8 @@ func (e *AccountAccess) Copy() AccountAccess { StorageReads: slices.Clone(e.StorageReads), BalanceChanges: slices.Clone(e.BalanceChanges), NonceChanges: slices.Clone(e.NonceChanges), + StorageWrites: make([]encodingSlotWrites, 0, len(e.StorageWrites)), + CodeChanges: make([]encodingCodeChange, 0, len(e.CodeChanges)), } for _, storageWrite := range e.StorageWrites { res.StorageWrites = append(res.StorageWrites, encodingSlotWrites{ @@ -189,13 +205,11 @@ func (e *AccountAccess) Copy() AccountAccess { Accesses: slices.Clone(storageWrite.Accesses), }) } - if len(e.Code) == 1 { - res.Code = []CodeChange{ - { - e.Code[0].TxIndex, - bytes.Clone(e.Code[0].Code), - }, - } + for _, codeChange := range e.CodeChanges { + res.CodeChanges = append(res.CodeChanges, encodingCodeChange{ + TxIndex: codeChange.TxIndex, + Code: bytes.Clone(codeChange.Code), + }) } return res } @@ -212,11 +226,11 @@ var _ rlp.Encoder = &ConstructionBlockAccessList{} func (a *ConstructionAccountAccess) toEncodingObj(addr common.Address) AccountAccess { res := AccountAccess{ Address: addr, - StorageWrites: make([]encodingSlotWrites, 0), - StorageReads: make([][32]byte, 0), - BalanceChanges: make([]encodingBalanceChange, 0), - NonceChanges: make([]encodingAccountNonce, 0), - Code: nil, + StorageWrites: make([]encodingSlotWrites, 0, len(a.StorageWrites)), + StorageReads: make([][32]byte, 0, len(a.StorageReads)), + BalanceChanges: make([]encodingBalanceChange, 0, len(a.BalanceChanges)), + NonceChanges: make([]encodingAccountNonce, 0, len(a.NonceChanges)), + CodeChanges: make([]encodingCodeChange, 0, len(a.CodeChange)), } // Convert write slots @@ -268,13 +282,13 @@ func (a *ConstructionAccountAccess) toEncodingObj(addr common.Address) AccountAc } // Convert code change - if a.CodeChange != nil { - res.Code = []CodeChange{ - { - a.CodeChange.TxIndex, - bytes.Clone(a.CodeChange.Code), - }, - } + codeIndices := slices.Collect(maps.Keys(a.CodeChange)) + slices.SortFunc(codeIndices, cmp.Compare[uint16]) + for _, idx := range codeIndices { + res.CodeChanges = append(res.CodeChanges, encodingCodeChange{ + TxIndex: idx, + Code: a.CodeChange[idx], + }) } return res } @@ -327,9 +341,9 @@ func (e *BlockAccessList) PrettyPrint() string { printWithIndent(2, fmt.Sprintf("%d: %d", change.TxIdx, change.Nonce)) } - if len(accountDiff.Code) > 0 { - printWithIndent(1, "code:") - printWithIndent(2, fmt.Sprintf("%d: %x", accountDiff.Code[0].TxIndex, accountDiff.Code[0].Code)) + printWithIndent(1, "code changes:") + for _, change := range accountDiff.CodeChanges { + printWithIndent(2, fmt.Sprintf("%d: %x", change.TxIndex, change.Code)) } } return res.String() diff --git a/core/types/bal/bal_encoding_rlp_generated.go b/core/types/bal/bal_encoding_rlp_generated.go index 0d52395329..640035e30e 100644 --- a/core/types/bal/bal_encoding_rlp_generated.go +++ b/core/types/bal/bal_encoding_rlp_generated.go @@ -49,7 +49,7 @@ func (obj *BlockAccessList) EncodeRLP(_w io.Writer) error { } w.ListEnd(_tmp15) _tmp18 := w.List() - for _, _tmp19 := range _tmp2.Code { + for _, _tmp19 := range _tmp2.CodeChanges { _tmp20 := w.List() w.WriteUint64(uint64(_tmp19.TxIndex)) w.WriteBytes(_tmp19.Code) @@ -228,13 +228,13 @@ func (obj *BlockAccessList) DecodeRLP(dec *rlp.Stream) error { return err } _tmp2.NonceChanges = _tmp17 - // Code: - var _tmp21 []CodeChange + // CodeChanges: + var _tmp21 []encodingCodeChange if _, err := dec.List(); err != nil { return err } for dec.MoreDataInList() { - var _tmp22 CodeChange + var _tmp22 encodingCodeChange { if _, err := dec.List(); err != nil { return err @@ -260,7 +260,7 @@ func (obj *BlockAccessList) DecodeRLP(dec *rlp.Stream) error { if err := dec.ListEnd(); err != nil { return err } - _tmp2.Code = _tmp21 + _tmp2.CodeChanges = _tmp21 if err := dec.ListEnd(); err != nil { return err } diff --git a/core/types/bal/bal_test.go b/core/types/bal/bal_test.go index 29414e414e..52c0de825e 100644 --- a/core/types/bal/bal_test.go +++ b/core/types/bal/bal_test.go @@ -60,9 +60,8 @@ func makeTestConstructionBAL() *ConstructionBlockAccessList { 1: 2, 2: 6, }, - CodeChange: &CodeChange{ - TxIndex: 0, - Code: common.Hex2Bytes("deadbeef"), + CodeChange: map[uint16][]byte{ + 0: common.Hex2Bytes("deadbeef"), }, }, common.BytesToAddress([]byte{0xff, 0xff, 0xff}): { @@ -85,6 +84,9 @@ func makeTestConstructionBAL() *ConstructionBlockAccessList { NonceChanges: map[uint16]uint64{ 1: 2, }, + CodeChange: map[uint16][]byte{ + 0: common.Hex2Bytes("deadbeef"), + }, }, }, } @@ -179,7 +181,7 @@ func makeTestAccountAccess(sort bool) AccountAccess { StorageReads: storageReads, BalanceChanges: balances, NonceChanges: nonces, - Code: []CodeChange{ + CodeChanges: []encodingCodeChange{ { TxIndex: 100, Code: testrand.Bytes(256), diff --git a/core/types/block.go b/core/types/block.go index c52c05a4c7..d092351b58 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -98,6 +98,9 @@ type Header struct { // RequestsHash was added by EIP-7685 and is ignored in legacy headers. RequestsHash *common.Hash `json:"requestsHash" rlp:"optional"` + + // SlotNumber was added by EIP-7843 and is ignored in legacy headers. + SlotNumber *uint64 `json:"slotNumber" rlp:"optional"` } // field type overrides for gencodec @@ -112,6 +115,7 @@ type headerMarshaling struct { Hash common.Hash `json:"hash"` // adds call to Hash() in MarshalJSON BlobGasUsed *hexutil.Uint64 ExcessBlobGas *hexutil.Uint64 + SlotNumber *hexutil.Uint64 } // Hash returns the block hash of the header, which is simply the keccak256 hash of its @@ -316,6 +320,10 @@ func CopyHeader(h *Header) *Header { cpy.RequestsHash = new(common.Hash) *cpy.RequestsHash = *h.RequestsHash } + if h.SlotNumber != nil { + cpy.SlotNumber = new(uint64) + *cpy.SlotNumber = *h.SlotNumber + } return &cpy } @@ -416,6 +424,15 @@ func (b *Block) BlobGasUsed() *uint64 { return blobGasUsed } +func (b *Block) SlotNumber() *uint64 { + var slotNum *uint64 + if b.header.SlotNumber != nil { + slotNum = new(uint64) + *slotNum = *b.header.SlotNumber + } + return slotNum +} + // Size returns the true RLP encoded storage size of the block, either by encoding // and returning it, or returning a previously cached value. func (b *Block) Size() uint64 { diff --git a/core/types/gen_header_json.go b/core/types/gen_header_json.go index 0af12500bd..16fb03f612 100644 --- a/core/types/gen_header_json.go +++ b/core/types/gen_header_json.go @@ -37,6 +37,7 @@ func (h Header) MarshalJSON() ([]byte, error) { ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` RequestsHash *common.Hash `json:"requestsHash" rlp:"optional"` + SlotNumber *hexutil.Uint64 `json:"slotNumber" rlp:"optional"` Hash common.Hash `json:"hash"` } var enc Header @@ -61,6 +62,7 @@ func (h Header) MarshalJSON() ([]byte, error) { enc.ExcessBlobGas = (*hexutil.Uint64)(h.ExcessBlobGas) enc.ParentBeaconRoot = h.ParentBeaconRoot enc.RequestsHash = h.RequestsHash + enc.SlotNumber = (*hexutil.Uint64)(h.SlotNumber) enc.Hash = h.Hash() return json.Marshal(&enc) } @@ -89,6 +91,7 @@ func (h *Header) UnmarshalJSON(input []byte) error { ExcessBlobGas *hexutil.Uint64 `json:"excessBlobGas" rlp:"optional"` ParentBeaconRoot *common.Hash `json:"parentBeaconBlockRoot" rlp:"optional"` RequestsHash *common.Hash `json:"requestsHash" rlp:"optional"` + SlotNumber *hexutil.Uint64 `json:"slotNumber" rlp:"optional"` } var dec Header if err := json.Unmarshal(input, &dec); err != nil { @@ -169,5 +172,8 @@ func (h *Header) UnmarshalJSON(input []byte) error { if dec.RequestsHash != nil { h.RequestsHash = dec.RequestsHash } + if dec.SlotNumber != nil { + h.SlotNumber = (*uint64)(dec.SlotNumber) + } return nil } diff --git a/core/types/gen_header_rlp.go b/core/types/gen_header_rlp.go index c79aa8a250..cfbd57ab8a 100644 --- a/core/types/gen_header_rlp.go +++ b/core/types/gen_header_rlp.go @@ -43,7 +43,8 @@ func (obj *Header) EncodeRLP(_w io.Writer) error { _tmp4 := obj.ExcessBlobGas != nil _tmp5 := obj.ParentBeaconRoot != nil _tmp6 := obj.RequestsHash != nil - if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 { + _tmp7 := obj.SlotNumber != nil + if _tmp1 || _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { if obj.BaseFee == nil { w.Write(rlp.EmptyString) } else { @@ -53,41 +54,48 @@ func (obj *Header) EncodeRLP(_w io.Writer) error { w.WriteBigInt(obj.BaseFee) } } - if _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 { + if _tmp2 || _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { if obj.WithdrawalsHash == nil { w.Write([]byte{0x80}) } else { w.WriteBytes(obj.WithdrawalsHash[:]) } } - if _tmp3 || _tmp4 || _tmp5 || _tmp6 { + if _tmp3 || _tmp4 || _tmp5 || _tmp6 || _tmp7 { if obj.BlobGasUsed == nil { w.Write([]byte{0x80}) } else { w.WriteUint64((*obj.BlobGasUsed)) } } - if _tmp4 || _tmp5 || _tmp6 { + if _tmp4 || _tmp5 || _tmp6 || _tmp7 { if obj.ExcessBlobGas == nil { w.Write([]byte{0x80}) } else { w.WriteUint64((*obj.ExcessBlobGas)) } } - if _tmp5 || _tmp6 { + if _tmp5 || _tmp6 || _tmp7 { if obj.ParentBeaconRoot == nil { w.Write([]byte{0x80}) } else { w.WriteBytes(obj.ParentBeaconRoot[:]) } } - if _tmp6 { + if _tmp6 || _tmp7 { if obj.RequestsHash == nil { w.Write([]byte{0x80}) } else { w.WriteBytes(obj.RequestsHash[:]) } } + if _tmp7 { + if obj.SlotNumber == nil { + w.Write([]byte{0x80}) + } else { + w.WriteUint64((*obj.SlotNumber)) + } + } w.ListEnd(_tmp0) return w.Flush() } diff --git a/core/types/transaction.go b/core/types/transaction.go index 6af960b8c3..e9bf08daef 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -317,11 +317,15 @@ func (tx *Transaction) To() *common.Address { // Cost returns (gas * gasPrice) + (blobGas * blobGasPrice) + value. func (tx *Transaction) Cost() *big.Int { - total := new(big.Int).Mul(tx.GasPrice(), new(big.Int).SetUint64(tx.Gas())) - if tx.Type() == BlobTxType { - total.Add(total, new(big.Int).Mul(tx.BlobGasFeeCap(), new(big.Int).SetUint64(tx.BlobGas()))) + // Avoid allocating copies via tx.GasPrice()/tx.Value(); use inner values directly. + total := new(big.Int).SetUint64(tx.inner.gas()) + total.Mul(total, tx.inner.gasPrice()) + if blobtx, ok := tx.inner.(*BlobTx); ok { + tmp := new(big.Int).SetUint64(blobtx.blobGas()) + tmp.Mul(tmp, blobtx.BlobFeeCap.ToBig()) + total.Add(total, tmp) } - total.Add(total, tx.Value()) + total.Add(total, tx.inner.value()) return total } @@ -396,14 +400,37 @@ func (tx *Transaction) calcEffectiveGasTip(dst *uint256.Int, baseFee *uint256.In return err } +// EffectiveGasTipValue returns the effective gasTip value for the given base fee, +// even if it would be negative. This can be used for sorting purposes. +func (tx *Transaction) EffectiveGasTipValue(baseFee *big.Int) *big.Int { + // min(gasTipCap, gasFeeCap - baseFee) + dst := new(big.Int) + if baseFee == nil { + dst.Set(tx.inner.gasTipCap()) + return dst + } + + dst.Sub(tx.inner.gasFeeCap(), baseFee) // gasFeeCap - baseFee + gasTipCap := tx.inner.gasTipCap() + if gasTipCap.Cmp(dst) < 0 { // gasTipCap < (gasFeeCap - baseFee) + dst.Set(gasTipCap) + } + return dst +} + func (tx *Transaction) EffectiveGasTipCmp(other *Transaction, baseFee *uint256.Int) int { if baseFee == nil { return tx.GasTipCapCmp(other) } // Use more efficient internal method. txTip, otherTip := new(uint256.Int), new(uint256.Int) - tx.calcEffectiveGasTip(txTip, baseFee) - other.calcEffectiveGasTip(otherTip, baseFee) + err1 := tx.calcEffectiveGasTip(txTip, baseFee) + err2 := other.calcEffectiveGasTip(otherTip, baseFee) + if err1 != nil || err2 != nil { + // fall back to big int comparison in case of error + base := baseFee.ToBig() + return tx.EffectiveGasTipValue(base).Cmp(other.EffectiveGasTipValue(base)) + } return txTip.Cmp(otherTip) } diff --git a/core/vm/common.go b/core/vm/common.go index 2990f58972..2d631f8a55 100644 --- a/core/vm/common.go +++ b/core/vm/common.go @@ -17,12 +17,43 @@ package vm import ( + "fmt" "math" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" ) +// CheckMaxInitCodeSize checks the size of contract initcode against the protocol-defined limit. +func CheckMaxInitCodeSize(rules *params.Rules, size uint64) error { + if rules.IsAmsterdam { + if size > params.MaxInitCodeSizeAmsterdam { + return fmt.Errorf("%w: code size %v limit %v", ErrMaxInitCodeSizeExceeded, size, params.MaxInitCodeSizeAmsterdam) + } + } else if rules.IsShanghai { + if size > params.MaxInitCodeSize { + return fmt.Errorf("%w: code size %v limit %v", ErrMaxInitCodeSizeExceeded, size, params.MaxInitCodeSize) + } + } + + return nil +} + +// CheckMaxCodeSize checks the size of contract code against the protocol-defined limit. +func CheckMaxCodeSize(rules *params.Rules, size uint64) error { + if rules.IsAmsterdam { + if size > params.MaxCodeSizeAmsterdam { + return fmt.Errorf("%w: code size %v limit %v", ErrMaxCodeSizeExceeded, size, params.MaxCodeSizeAmsterdam) + } + } else if rules.IsEIP158 { + if size > params.MaxCodeSize { + return fmt.Errorf("%w: code size %v limit %v", ErrMaxCodeSizeExceeded, size, params.MaxCodeSize) + } + } + return nil +} + // calcMemSize64 calculates the required memory size, and returns // the size and whether the result overflowed uint64 func calcMemSize64(off, l *uint256.Int) (uint64, bool) { diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 867746acc8..010f477337 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -262,7 +262,7 @@ func ActivePrecompiles(rules params.Rules) []common.Address { // - the returned bytes, // - the _remaining_ gas, // - any error that occurred -func RunPrecompiledContract(p PrecompiledContract, input []byte, suppliedGas uint64, logger *tracing.Hooks) (ret []byte, remainingGas uint64, err error) { +func RunPrecompiledContract(stateDB StateDB, p PrecompiledContract, address common.Address, input []byte, suppliedGas uint64, logger *tracing.Hooks) (ret []byte, remainingGas uint64, err error) { gasCost := p.RequiredGas(input) if suppliedGas < gasCost { return nil, 0, ErrOutOfGas @@ -271,6 +271,12 @@ func RunPrecompiledContract(p PrecompiledContract, input []byte, suppliedGas uin logger.OnGasChange(suppliedGas, suppliedGas-gasCost, tracing.GasChangeCallPrecompiledContract) } suppliedGas -= gasCost + + // Touch the precompile for block-level accessList recording once Amsterdam + // fork is activated. + if stateDB != nil { + stateDB.Exist(address) + } output, err := p.Run(input) return output, suppliedGas, err } diff --git a/core/vm/contracts_fuzz_test.go b/core/vm/contracts_fuzz_test.go index 1e5cc80074..34ed1d87fe 100644 --- a/core/vm/contracts_fuzz_test.go +++ b/core/vm/contracts_fuzz_test.go @@ -36,7 +36,7 @@ func FuzzPrecompiledContracts(f *testing.F) { return } inWant := string(input) - RunPrecompiledContract(p, input, gas, nil) + RunPrecompiledContract(nil, p, a, input, gas, nil) if inHave := string(input); inWant != inHave { t.Errorf("Precompiled %v modified input data", a) } diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index 51bd7101ff..b647dcc62b 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -99,7 +99,7 @@ func testPrecompiled(addr string, test precompiledTest, t *testing.T) { in := common.Hex2Bytes(test.Input) gas := p.RequiredGas(in) t.Run(fmt.Sprintf("%s-Gas=%d", test.Name, gas), func(t *testing.T) { - if res, _, err := RunPrecompiledContract(p, in, gas, nil); err != nil { + if res, _, err := RunPrecompiledContract(nil, p, common.HexToAddress(addr), in, gas, nil); err != nil { t.Error(err) } else if common.Bytes2Hex(res) != test.Expected { t.Errorf("Expected %v, got %v", test.Expected, common.Bytes2Hex(res)) @@ -121,7 +121,7 @@ func testPrecompiledOOG(addr string, test precompiledTest, t *testing.T) { gas := test.Gas - 1 t.Run(fmt.Sprintf("%s-Gas=%d", test.Name, gas), func(t *testing.T) { - _, _, err := RunPrecompiledContract(p, in, gas, nil) + _, _, err := RunPrecompiledContract(nil, p, common.HexToAddress(addr), in, gas, nil) if err.Error() != "out of gas" { t.Errorf("Expected error [out of gas], got [%v]", err) } @@ -138,7 +138,7 @@ func testPrecompiledFailure(addr string, test precompiledFailureTest, t *testing in := common.Hex2Bytes(test.Input) gas := p.RequiredGas(in) t.Run(test.Name, func(t *testing.T) { - _, _, err := RunPrecompiledContract(p, in, gas, nil) + _, _, err := RunPrecompiledContract(nil, p, common.HexToAddress(addr), in, gas, nil) if err.Error() != test.ExpectedError { t.Errorf("Expected error [%v], got [%v]", test.ExpectedError, err) } @@ -169,7 +169,7 @@ func benchmarkPrecompiled(addr string, test precompiledTest, bench *testing.B) { start := time.Now() for bench.Loop() { copy(data, in) - res, _, err = RunPrecompiledContract(p, data, reqGas, nil) + res, _, err = RunPrecompiledContract(nil, p, common.HexToAddress(addr), data, reqGas, nil) } elapsed := uint64(time.Since(start)) if elapsed < 1 { diff --git a/core/vm/eips.go b/core/vm/eips.go index dfcac4b930..3ccd9aaaf0 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -43,6 +43,7 @@ var activators = map[int]func(*JumpTable){ 7702: enable7702, 7939: enable7939, 8024: enable8024, + 7843: enable7843, } // EnableEIP enables the given EIP on the config. @@ -579,3 +580,19 @@ func enable7702(jt *JumpTable) { jt[STATICCALL].dynamicGas = gasStaticCallEIP7702 jt[DELEGATECALL].dynamicGas = gasDelegateCallEIP7702 } + +// opSlotNum enables the SLOTNUM opcode +func opSlotNum(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { + scope.Stack.push(uint256.NewInt(evm.Context.SlotNum)) + return nil, nil +} + +// enable7843 enables the SLOTNUM opcode as specified in EIP-7843. +func enable7843(jt *JumpTable) { + jt[SLOTNUM] = &operation{ + execute: opSlotNum, + constantGas: GasQuickStep, + minStack: minStack(0, 1), + maxStack: maxStack(0, 1), + } +} diff --git a/core/vm/evm.go b/core/vm/evm.go index c365f637d2..5897dbd265 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -66,6 +66,7 @@ type BlockContext struct { BaseFee *big.Int // Provides information for BASEFEE (0 if vm runs with NoBaseFee flag and 0 gas price) BlobBaseFee *big.Int // Provides information for BLOBBASEFEE (0 if vm runs with NoBaseFee flag and 0 blob gas price) Random *common.Hash // Provides information for PREVRANDAO + SlotNum uint64 // Provides information for SLOTNUM } // TxContext provides the EVM with information about a transaction. @@ -144,6 +145,8 @@ func NewEVM(blockCtx BlockContext, statedb StateDB, chainConfig *params.ChainCon evm.precompiles = activePrecompiledContracts(evm.chainRules) switch { + case evm.chainRules.IsAmsterdam: + evm.table = &amsterdamInstructionSet case evm.chainRules.IsOsaka: evm.table = &osakaInstructionSet case evm.chainRules.IsVerkle: @@ -245,13 +248,14 @@ func (evm *EVM) Call(caller common.Address, addr common.Address, input []byte, g if evm.depth > int(params.CallCreateDepth) { return nil, gas, ErrDepth } - // Fail if we're trying to transfer more than the available balance - if !value.IsZero() && !evm.Context.CanTransfer(evm.StateDB, caller, value) { + syscall := isSystemCall(caller) + + // Fail if we're trying to transfer more than the available balance. + if !syscall && !value.IsZero() && !evm.Context.CanTransfer(evm.StateDB, caller, value) { return nil, gas, ErrInsufficientBalance } snapshot := evm.StateDB.Snapshot() p, isPrecompile := evm.precompile(addr) - if !evm.StateDB.Exist(addr) { if !isPrecompile && evm.chainRules.IsEIP4762 && !isSystemCall(caller) { // Add proof of absence to witness @@ -275,10 +279,18 @@ func (evm *EVM) Call(caller common.Address, addr common.Address, input []byte, g } evm.StateDB.CreateAccount(addr) } - evm.Context.Transfer(evm.StateDB, caller, addr, value) - + // Perform the value transfer only in non-syscall mode. + // Calling this is required even for zero-value transfers, + // to ensure the state clearing mechanism is applied. + if !syscall { + evm.Context.Transfer(evm.StateDB, caller, addr, value) + } if isPrecompile { - ret, gas, err = RunPrecompiledContract(p, input, gas, evm.Config.Tracer) + var stateDB StateDB + if evm.chainRules.IsAmsterdam { + stateDB = evm.StateDB + } + ret, gas, err = RunPrecompiledContract(stateDB, p, addr, input, gas, evm.Config.Tracer) } else { // Initialise a new contract and set the code that is to be used by the EVM. code := evm.resolveCode(addr) @@ -302,7 +314,6 @@ func (evm *EVM) Call(caller common.Address, addr common.Address, input []byte, g if evm.Config.Tracer != nil && evm.Config.Tracer.OnGasChange != nil { evm.Config.Tracer.OnGasChange(gas, 0, tracing.GasChangeCallFailedExecution) } - gas = 0 } // TODO: consider clearing up unused snapshots: @@ -342,7 +353,11 @@ func (evm *EVM) CallCode(caller common.Address, addr common.Address, input []byt // It is allowed to call precompiles, even via delegatecall if p, isPrecompile := evm.precompile(addr); isPrecompile { - ret, gas, err = RunPrecompiledContract(p, input, gas, evm.Config.Tracer) + var stateDB StateDB + if evm.chainRules.IsAmsterdam { + stateDB = evm.StateDB + } + ret, gas, err = RunPrecompiledContract(stateDB, p, addr, input, gas, evm.Config.Tracer) } else { // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. @@ -385,7 +400,11 @@ func (evm *EVM) DelegateCall(originCaller common.Address, caller common.Address, // It is allowed to call precompiles, even via delegatecall if p, isPrecompile := evm.precompile(addr); isPrecompile { - ret, gas, err = RunPrecompiledContract(p, input, gas, evm.Config.Tracer) + var stateDB StateDB + if evm.chainRules.IsAmsterdam { + stateDB = evm.StateDB + } + ret, gas, err = RunPrecompiledContract(stateDB, p, addr, input, gas, evm.Config.Tracer) } else { // Initialise a new contract and make initialise the delegate values // @@ -437,7 +456,11 @@ func (evm *EVM) StaticCall(caller common.Address, addr common.Address, input []b evm.StateDB.AddBalance(addr, new(uint256.Int), tracing.BalanceChangeTouchAccount) if p, isPrecompile := evm.precompile(addr); isPrecompile { - ret, gas, err = RunPrecompiledContract(p, input, gas, evm.Config.Tracer) + var stateDB StateDB + if evm.chainRules.IsAmsterdam { + stateDB = evm.StateDB + } + ret, gas, err = RunPrecompiledContract(stateDB, p, addr, input, gas, evm.Config.Tracer) } else { // Initialise a new contract and set the code that is to be used by the EVM. // The contract is a scoped environment for this execution context only. @@ -574,8 +597,8 @@ func (evm *EVM) initNewContract(contract *Contract, address common.Address) ([]b } // Check whether the max code size has been exceeded, assign err if the case. - if evm.chainRules.IsEIP158 && len(ret) > params.MaxCodeSize { - return ret, ErrMaxCodeSizeExceeded + if err := CheckMaxCodeSize(&evm.chainRules, uint64(len(ret))); err != nil { + return ret, err } // Reject code starting with 0xEF if EIP-3541 is enabled. diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 23a2cbbf4d..aa1ad918bb 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -18,7 +18,6 @@ package vm import ( "errors" - "fmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" @@ -318,10 +317,10 @@ func gasCreateEip3860(evm *EVM, contract *Contract, stack *Stack, mem *Memory, m if overflow { return 0, ErrGasUintOverflow } - if size > params.MaxInitCodeSize { - return 0, fmt.Errorf("%w: size %d", ErrMaxInitCodeSizeExceeded, size) + if err := CheckMaxInitCodeSize(&evm.chainRules, size); err != nil { + return 0, err } - // Since size <= params.MaxInitCodeSize, these multiplication cannot overflow + // Since size <= the protocol-defined maximum initcode size limit, these multiplication cannot overflow moreGas := params.InitCodeWordGas * ((size + 31) / 32) if gas, overflow = math.SafeAdd(gas, moreGas); overflow { return 0, ErrGasUintOverflow @@ -337,10 +336,10 @@ func gasCreate2Eip3860(evm *EVM, contract *Contract, stack *Stack, mem *Memory, if overflow { return 0, ErrGasUintOverflow } - if size > params.MaxInitCodeSize { - return 0, fmt.Errorf("%w: size %d", ErrMaxInitCodeSizeExceeded, size) + if err := CheckMaxInitCodeSize(&evm.chainRules, size); err != nil { + return 0, err } - // Since size <= params.MaxInitCodeSize, these multiplication cannot overflow + // Since size <= the protocol-defined maximum initcode size limit, these multiplication cannot overflow moreGas := (params.InitCodeWordGas + params.Keccak256WordGas) * ((size + 31) / 32) if gas, overflow = math.SafeAdd(gas, moreGas); overflow { return 0, ErrGasUintOverflow diff --git a/core/vm/gas_table_test.go b/core/vm/gas_table_test.go index 7fe76b0a63..436cc47f2e 100644 --- a/core/vm/gas_table_test.go +++ b/core/vm/gas_table_test.go @@ -148,11 +148,15 @@ func TestCreateGas(t *testing.T) { BlockNumber: big.NewInt(0), } config := Config{} + chainConfig := params.AllEthashProtocolChanges if tt.eip3860 { config.ExtraEips = []int{3860} + vmctx.Random = new(common.Hash) + + chainConfig = params.MergedTestChainConfig } - evm := NewEVM(vmctx, statedb, params.AllEthashProtocolChanges, config) + evm := NewEVM(vmctx, statedb, chainConfig, config) var startGas = uint64(testGas) ret, gas, err := evm.Call(common.Address{}, address, nil, startGas, new(uint256.Int)) if err != nil { diff --git a/core/vm/instructions.go b/core/vm/instructions.go index a4c4b0703b..4e4a33acda 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -946,24 +946,34 @@ func opSelfdestruct6780(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, erro return nil, errStopToken } +// decodeSingle decodes the immediate operand of a backward-compatible DUPN or SWAPN instruction (EIP-8024) +// https://eips.ethereum.org/EIPS/eip-8024 func decodeSingle(x byte) int { - if x <= 90 { - return int(x) + 17 - } - return int(x) - 20 + // Depths 1-16 are already covered by the legacy opcodes. The forbidden byte range [91, 127] removes + // 37 values from the 256 possible immediates, leaving 219 usable values, so this encoding covers depths + // 17 through 235. The immediate is encoded as (x + 111) % 256, where 111 is chosen so that these values + // avoid the forbidden range. Decoding is simply the modular inverse (i.e. 111+145=256). + return (int(x) + 145) % 256 } +// decodePair decodes the immediate operand of a backward-compatible EXCHANGE +// instruction (EIP-8024) into stack indices (n, m) where 1 <= n < m +// and n + m <= 30. The forbidden byte range [82, 127] removes 46 values from +// the 256 possible immediates, leaving exactly 210 usable bytes. +// https://eips.ethereum.org/EIPS/eip-8024 func decodePair(x byte) (int, int) { - var k int - if x <= 79 { - k = int(x) - } else { - k = int(x) - 48 - } + // XOR with 143 remaps the forbidden bytes [82, 127] to an unused corner + // of the 16x16 grid below. + k := int(x ^ 143) + // Split into row q and column r of a 16x16 grid. The 210 valid pairs + // occupy two triangles within this grid. q, r := k/16, k%16 + // Upper triangle (q < r): pairs where m <= 16, encoded directly as + // (q+1, r+1). if q < r { return q + 1, r + 1 } + // Lower triangle: pairs where m > 16, recovered as (r+1, 29-q). return r + 1, 29 - q } @@ -1034,8 +1044,8 @@ func opExchange(pc *uint64, evm *EVM, scope *ScopeContext) ([]byte, error) { } // This range is excluded both to preserve compatibility with existing opcodes - // and to keep decode_pair’s 16-aligned arithmetic mapping valid (0–79, 128–255). - if x > 79 && x < 128 { + // and to keep decode_pair’s 16-aligned arithmetic mapping valid (0–81, 128–255). + if x > 81 && x < 128 { return nil, &ErrInvalidOpCode{opcode: OpCode(x)} } n, m := decodePair(x) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 3f776146f1..4c6d093d2e 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -1022,16 +1022,7 @@ func TestEIP8024_Execution(t *testing.T) { }{ { name: "DUPN", - codeHex: "60016000808080808080808080808080808080e600", - wantVals: []uint64{ - 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, - }, - }, - { - name: "DUPN_MISSING_IMMEDIATE", - codeHex: "60016000808080808080808080808080808080e6", + codeHex: "60016000808080808080808080808080808080e680", wantVals: []uint64{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1040,7 +1031,7 @@ func TestEIP8024_Execution(t *testing.T) { }, { name: "SWAPN", - codeHex: "600160008080808080808080808080808080806002e700", + codeHex: "600160008080808080808080808080808080806002e780", wantVals: []uint64{ 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1048,22 +1039,23 @@ func TestEIP8024_Execution(t *testing.T) { }, }, { - name: "SWAPN_MISSING_IMMEDIATE", - codeHex: "600160008080808080808080808080808080806002e7", + name: "EXCHANGE_MISSING_IMMEDIATE", + codeHex: "600260008080808080600160008080808080808080e8", wantVals: []uint64{ - 1, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 2, + 0, 0, 0, 0, 0, 0, 0, 0, 0, + 2, // 10th from top + 0, 0, 0, 0, 0, 0, + 1, // bottom }, }, { name: "EXCHANGE", - codeHex: "600060016002e801", + codeHex: "600060016002e88e", wantVals: []uint64{2, 0, 1}, }, { - name: "EXCHANGE_MISSING_IMMEDIATE", - codeHex: "600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060006000600060016002e8", + name: "EXCHANGE", + codeHex: "600080808080808080808080808080808080808080808080808080808060016002e88f", wantVals: []uint64{ 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -1077,68 +1069,31 @@ func TestEIP8024_Execution(t *testing.T) { wantOpcode: SWAPN, }, { - name: "JUMP over INVALID_DUPN", + name: "JUMP_OVER_INVALID_DUPN", codeHex: "600456e65b", wantErr: nil, }, { - name: "UNDERFLOW_DUPN_1", - codeHex: "6000808080808080808080808080808080e600", + name: "EXCHANGE", + codeHex: "60008080e88e15", + wantVals: []uint64{1, 0, 0}, + }, + { + name: "INVALID_EXCHANGE", + codeHex: "e852", + wantErr: &ErrInvalidOpCode{}, + wantOpcode: EXCHANGE, + }, + { + name: "UNDERFLOW_DUPN", + codeHex: "6000808080808080808080808080808080e680", wantErr: &ErrStackUnderflow{}, wantOpcode: DUPN, }, // Additional test cases - { - name: "INVALID_DUPN_LOW", - codeHex: "e65b", - wantErr: &ErrInvalidOpCode{}, - wantOpcode: DUPN, - }, - { - name: "INVALID_EXCHANGE_LOW", - codeHex: "e850", - wantErr: &ErrInvalidOpCode{}, - wantOpcode: EXCHANGE, - }, - { - name: "INVALID_DUPN_HIGH", - codeHex: "e67f", - wantErr: &ErrInvalidOpCode{}, - wantOpcode: DUPN, - }, - { - name: "INVALID_SWAPN_HIGH", - codeHex: "e77f", - wantErr: &ErrInvalidOpCode{}, - wantOpcode: SWAPN, - }, - { - name: "INVALID_EXCHANGE_HIGH", - codeHex: "e87f", - wantErr: &ErrInvalidOpCode{}, - wantOpcode: EXCHANGE, - }, - { - name: "UNDERFLOW_DUPN_2", - codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe600", // (n=17, need 17 items, have 16) - wantErr: &ErrStackUnderflow{}, - wantOpcode: DUPN, - }, - { - name: "UNDERFLOW_SWAPN", - codeHex: "5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5f5fe700", // (n=17, need 18 items, have 17) - wantErr: &ErrStackUnderflow{}, - wantOpcode: SWAPN, - }, - { - name: "UNDERFLOW_EXCHANGE", - codeHex: "60016002e801", // (n,m)=(1,2), need 3 items, have 2 - wantErr: &ErrStackUnderflow{}, - wantOpcode: EXCHANGE, - }, { name: "PC_INCREMENT", - codeHex: "600060006000e80115", + codeHex: "600060006000e88e15", wantVals: []uint64{1, 0, 0}, }, } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 52dbe83d86..620c069fc8 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -27,13 +27,11 @@ import ( // Config are the configuration options for the Interpreter type Config struct { - Tracer *tracing.Hooks + Tracer *tracing.Hooks + NoBaseFee bool // Forces the EIP-1559 baseFee to 0 (needed for 0 price calls) EnablePreimageRecording bool // Enables recording of SHA3/keccak preimages ExtraEips []int // Additional EIPS that are to be enabled - - StatelessSelfValidation bool // Generate execution witnesses and self-check against them (testing purpose) - EnableWitnessStats bool // Whether trie access statistics collection is enabled } // ScopeContext contains the things that are per-call, such as stack and memory, diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index d7a4d9da1d..a2e2c91194 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -63,6 +63,7 @@ var ( verkleInstructionSet = newVerkleInstructionSet() pragueInstructionSet = newPragueInstructionSet() osakaInstructionSet = newOsakaInstructionSet() + amsterdamInstructionSet = newAmsterdamInstructionSet() ) // JumpTable contains the EVM opcodes supported at a given fork. @@ -92,6 +93,13 @@ func newVerkleInstructionSet() JumpTable { return validate(instructionSet) } +func newAmsterdamInstructionSet() JumpTable { + instructionSet := newOsakaInstructionSet() + enable7843(&instructionSet) // EIP-7843 (SLOTNUM opcode) + enable8024(&instructionSet) // EIP-8024 (Backward compatible SWAPN, DUPN, EXCHANGE) + return validate(instructionSet) +} + func newOsakaInstructionSet() JumpTable { instructionSet := newPragueInstructionSet() enable7939(&instructionSet) // EIP-7939 (CLZ opcode) diff --git a/core/vm/jump_table_export.go b/core/vm/jump_table_export.go index 89a2ebf6f4..fdf814d64c 100644 --- a/core/vm/jump_table_export.go +++ b/core/vm/jump_table_export.go @@ -28,6 +28,8 @@ func LookupInstructionSet(rules params.Rules) (JumpTable, error) { switch { case rules.IsVerkle: return newCancunInstructionSet(), errors.New("verkle-fork not defined yet") + case rules.IsAmsterdam: + return newAmsterdamInstructionSet(), nil case rules.IsOsaka: return newOsakaInstructionSet(), nil case rules.IsPrague: diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 9a32126a80..cb871dca6d 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -105,6 +105,7 @@ const ( BASEFEE OpCode = 0x48 BLOBHASH OpCode = 0x49 BLOBBASEFEE OpCode = 0x4a + SLOTNUM OpCode = 0x4b ) // 0x50 range - 'storage' and execution. @@ -320,6 +321,7 @@ var opCodeToString = [256]string{ BASEFEE: "BASEFEE", BLOBHASH: "BLOBHASH", BLOBBASEFEE: "BLOBBASEFEE", + SLOTNUM: "SLOTNUM", // 0x50 range - 'storage' and execution. POP: "POP", @@ -502,6 +504,7 @@ var stringToOp = map[string]OpCode{ "BASEFEE": BASEFEE, "BLOBHASH": BLOBHASH, "BLOBBASEFEE": BLOBBASEFEE, + "SLOTNUM": SLOTNUM, "DELEGATECALL": DELEGATECALL, "STATICCALL": STATICCALL, "CODESIZE": CODESIZE, diff --git a/crypto/secp256k1/curve.go b/crypto/secp256k1/curve.go index b82b147e3c..504602f5be 100644 --- a/crypto/secp256k1/curve.go +++ b/crypto/secp256k1/curve.go @@ -73,6 +73,10 @@ func (bitCurve *BitCurve) Params() *elliptic.CurveParams { // IsOnCurve returns true if the given (x,y) lies on the BitCurve. func (bitCurve *BitCurve) IsOnCurve(x, y *big.Int) bool { + if x.Cmp(bitCurve.P) >= 0 || y.Cmp(bitCurve.P) >= 0 { + return false + } + // y² = x³ + b y2 := new(big.Int).Mul(y, y) //y² y2.Mod(y2, bitCurve.P) //y²%P diff --git a/crypto/secp256k1/ext.h b/crypto/secp256k1/ext.h index 1c485e2603..baafb4404b 100644 --- a/crypto/secp256k1/ext.h +++ b/crypto/secp256k1/ext.h @@ -109,8 +109,10 @@ int secp256k1_ext_scalar_mul(const secp256k1_context* ctx, unsigned char *point, ARG_CHECK(scalar != NULL); (void)ctx; - secp256k1_fe_set_b32_limit(&feX, point); - secp256k1_fe_set_b32_limit(&feY, point+32); + if (!secp256k1_fe_set_b32_limit(&feX, point) || + !secp256k1_fe_set_b32_limit(&feY, point+32)) { + return 0; + } secp256k1_ge_set_xy(&ge, &feX, &feY); secp256k1_scalar_set_b32(&s, scalar, &overflow); if (overflow || secp256k1_scalar_is_zero(&s)) { diff --git a/crypto/signature_nocgo.go b/crypto/signature_nocgo.go index 9dce1057fa..0aab7180d3 100644 --- a/crypto/signature_nocgo.go +++ b/crypto/signature_nocgo.go @@ -167,6 +167,13 @@ type btCurve struct { *secp256k1.KoblitzCurve } +func (curve btCurve) IsOnCurve(x, y *big.Int) bool { + if x.Cmp(secp256k1.Params().P) >= 0 || y.Cmp(secp256k1.Params().P) >= 0 { + return false + } + return curve.KoblitzCurve.IsOnCurve(x, y) +} + // Marshal converts a point given as (x, y) into a byte slice. func (curve btCurve) Marshal(x, y *big.Int) []byte { byteLen := (curve.Params().BitSize + 7) / 8 diff --git a/eth/api_debug.go b/eth/api_debug.go index db1b842e90..b8267902b2 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/stateless" @@ -493,34 +494,22 @@ func (api *DebugAPI) StateSize(blockHashOrNumber *rpc.BlockNumberOrHash) (interf }, nil } -func (api *DebugAPI) ExecutionWitness(bn rpc.BlockNumber) (*stateless.ExtWitness, error) { +func (api *DebugAPI) ExecutionWitness(bn rpc.BlockNumberOrHash) (*stateless.ExtWitness, error) { bc := api.eth.blockchain - block, err := api.eth.APIBackend.BlockByNumber(context.Background(), bn) + block, err := api.eth.APIBackend.BlockByNumberOrHash(context.Background(), bn) if err != nil { - return &stateless.ExtWitness{}, fmt.Errorf("block number %v not found", bn) + return &stateless.ExtWitness{}, fmt.Errorf("block %v not found", bn) } parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) if parent == nil { - return &stateless.ExtWitness{}, fmt.Errorf("block number %v found, but parent missing", bn) + return &stateless.ExtWitness{}, fmt.Errorf("block %v found, but parent missing", bn) } - result, err := bc.ProcessBlock(parent.Root, block, false, true) - if err != nil { - return nil, err - } - return result.Witness().ToExtWitness(), nil -} - -func (api *DebugAPI) ExecutionWitnessByHash(hash common.Hash) (*stateless.ExtWitness, error) { - bc := api.eth.blockchain - block := bc.GetBlockByHash(hash) - if block == nil { - return &stateless.ExtWitness{}, fmt.Errorf("block hash %x not found", hash) - } - parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1) - if parent == nil { - return &stateless.ExtWitness{}, fmt.Errorf("block number %x found, but parent missing", hash) - } - result, err := bc.ProcessBlock(parent.Root, block, false, true) + config := core.ExecuteConfig{ + WriteState: false, + EnableTracer: false, + MakeWitness: true, + } + result, err := bc.ProcessBlock(context.Background(), parent.Root, block, config) if err != nil { return nil, err } diff --git a/eth/backend.go b/eth/backend.go index 5a487ea2ad..03559aa5b0 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -238,8 +238,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TxIndexSender: config.TxIndexSender, VmConfig: vm.Config{ EnablePreimageRecording: config.EnablePreimageRecording, - EnableWitnessStats: config.EnableWitnessStats, - StatelessSelfValidation: config.StatelessSelfValidation, }, // Enables file journaling for the trie database. The journal files will be stored // within the data directory. The corresponding paths will be either: @@ -248,6 +246,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { TrieJournalDirectory: stack.ResolvePath("triedb"), StateSizeTracking: config.EnableStateSizeTracking, SlowBlockThreshold: config.SlowBlockThreshold, + + StatelessSelfValidation: config.StatelessSelfValidation, + EnableWitnessStats: config.EnableWitnessStats, } ) if config.VMTrace != "" { diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index e6ecf4ff6a..96d4570561 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -18,6 +18,7 @@ package catalyst import ( + "context" "errors" "fmt" "reflect" @@ -35,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/miner" @@ -45,9 +47,10 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -// Register adds the engine API to the full node. +// Register adds the engine API and related APIs to the full node. func Register(stack *node.Node, backend *eth.Ethereum) error { stack.RegisterAPIs([]rpc.API{ + newTestingAPI(backend), { Namespace: "engine", Service: NewConsensusAPI(backend), @@ -209,6 +212,28 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, pa return api.forkchoiceUpdated(update, params, engine.PayloadV3, false) } +// ForkchoiceUpdatedV4 is equivalent to V3 with the addition of slot number +// in the payload attributes. It supports only PayloadAttributesV4. +func (api *ConsensusAPI) ForkchoiceUpdatedV4(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { + if params != nil { + switch { + case params.Withdrawals == nil: + return engine.STATUS_INVALID, attributesErr("missing withdrawals") + case params.BeaconRoot == nil: + return engine.STATUS_INVALID, attributesErr("missing beacon root") + case params.SlotNumber == nil: + return engine.STATUS_INVALID, attributesErr("missing slot number") + case !api.checkFork(params.Timestamp, forks.Amsterdam): + return engine.STATUS_INVALID, unsupportedForkErr("fcuV4 must only be called for amsterdam payloads") + } + } + // TODO(matt): the spec requires that fcu is applied when called on a valid + // hash, even if params are wrong. To do this we need to split up + // forkchoiceUpdate into a function that only updates the head and then a + // function that kicks off block construction. + return api.forkchoiceUpdated(update, params, engine.PayloadV4, false) +} + func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes, payloadVersion engine.PayloadVersion, payloadWitness bool) (engine.ForkChoiceResponse, error) { api.forkchoiceLock.Lock() defer api.forkchoiceLock.Unlock() @@ -230,12 +255,9 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl if res := api.checkInvalidAncestor(update.HeadBlockHash, update.HeadBlockHash); res != nil { return engine.ForkChoiceResponse{PayloadStatus: *res, PayloadID: nil}, nil } - // If the head hash is unknown (was not given to us in a newPayload request), - // we cannot resolve the header, so not much to do. This could be extended in - // the future to resolve from the `eth` network, but it's an unexpected case - // that should be fixed, not papered over. header := api.remoteBlocks.get(update.HeadBlockHash) if header == nil { + // The head hash is unknown locally, try to resolve it from the `eth` network log.Warn("Fetching the unknown forkchoice head from network", "hash", update.HeadBlockHash) retrievedHead, err := api.eth.Downloader().GetHeader(update.HeadBlockHash) if err != nil { @@ -248,7 +270,9 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl // If the finalized hash is known, we can direct the downloader to move // potentially more data to the freezer from the get go. finalized := api.remoteBlocks.get(update.FinalizedBlockHash) - + if finalized == nil { + finalized = api.eth.BlockChain().GetHeaderByHash(update.FinalizedBlockHash) + } // Header advertised via a past newPayload request. Start syncing to it. context := []interface{}{"number", header.Number, "hash", header.Hash()} if update.FinalizedBlockHash != (common.Hash{}) { @@ -342,6 +366,7 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl Random: payloadAttributes.Random, Withdrawals: payloadAttributes.Withdrawals, BeaconRoot: payloadAttributes.BeaconRoot, + SlotNum: payloadAttributes.SlotNumber, Version: payloadVersion, } id := args.Id() @@ -409,7 +434,7 @@ func (api *ConsensusAPI) GetPayloadV2(payloadID engine.PayloadID) (*engine.Execu payloadID, false, []engine.PayloadVersion{engine.PayloadV1, engine.PayloadV2}, - []forks.Fork{forks.Shanghai}, + []forks.Fork{forks.Paris, forks.Shanghai}, ) } @@ -455,6 +480,18 @@ func (api *ConsensusAPI) GetPayloadV5(payloadID engine.PayloadID) (*engine.Execu }) } +// GetPayloadV6 returns a cached payload by id. This endpoint should only +// be used after the Amsterdam fork. +func (api *ConsensusAPI) GetPayloadV6(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { + return api.getPayload( + payloadID, + false, + []engine.PayloadVersion{engine.PayloadV4}, + []forks.Fork{ + forks.Amsterdam, + }) +} + // getPayload will retrieve the specified payload and verify it conforms to the // endpoint's allowed payload versions and forks. // @@ -625,15 +662,15 @@ func (api *ConsensusAPI) getBlobs(hashes []common.Hash, v2 bool) ([]*engine.Blob var invalidStatus = engine.PayloadStatusV1{Status: engine.INVALID} // NewPayloadV1 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV1(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { if params.Withdrawals != nil { return invalidStatus, paramsErr("withdrawals not supported in V1") } - return api.newPayload(params, nil, nil, nil, false) + return api.newPayload(ctx, params, nil, nil, nil, false) } // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV2(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { var ( cancun = api.config().IsCancun(api.config().LondonBlock, params.Timestamp) shanghai = api.config().IsShanghai(api.config().LondonBlock, params.Timestamp) @@ -650,11 +687,11 @@ func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.Payl case params.BlobGasUsed != nil: return invalidStatus, paramsErr("non-nil blobGasUsed pre-cancun") } - return api.newPayload(params, nil, nil, nil, false) + return api.newPayload(ctx, params, nil, nil, nil, false) } // NewPayloadV3 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV3(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -669,11 +706,11 @@ func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHas case !api.checkFork(params.Timestamp, forks.Cancun): return invalidStatus, unsupportedForkErr("newPayloadV3 must only be called for cancun payloads") } - return api.newPayload(params, versionedHashes, beaconRoot, nil, false) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, nil, false) } // NewPayloadV4 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) NewPayloadV4(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadV4(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -694,10 +731,37 @@ func (api *ConsensusAPI) NewPayloadV4(params engine.ExecutableData, versionedHas if err := validateRequests(requests); err != nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(err) } - return api.newPayload(params, versionedHashes, beaconRoot, requests, false) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, requests, false) } -func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, requests [][]byte, witness bool) (engine.PayloadStatusV1, error) { +// NewPayloadV5 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. +func (api *ConsensusAPI) NewPayloadV5(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { + switch { + case params.Withdrawals == nil: + return invalidStatus, paramsErr("nil withdrawals post-shanghai") + case params.ExcessBlobGas == nil: + return invalidStatus, paramsErr("nil excessBlobGas post-cancun") + case params.BlobGasUsed == nil: + return invalidStatus, paramsErr("nil blobGasUsed post-cancun") + case versionedHashes == nil: + return invalidStatus, paramsErr("nil versionedHashes post-cancun") + case beaconRoot == nil: + return invalidStatus, paramsErr("nil beaconRoot post-cancun") + case executionRequests == nil: + return invalidStatus, paramsErr("nil executionRequests post-prague") + case params.SlotNumber == nil: + return invalidStatus, paramsErr("nil slotnumber post-amsterdam") + case !api.checkFork(params.Timestamp, forks.Amsterdam): + return invalidStatus, unsupportedForkErr("newPayloadV5 must only be called for amsterdam payloads") + } + requests := convertRequests(executionRequests) + if err := validateRequests(requests); err != nil { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(err) + } + return api.newPayload(ctx, params, versionedHashes, beaconRoot, requests, false) +} + +func (api *ConsensusAPI) newPayload(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, requests [][]byte, witness bool) (result engine.PayloadStatusV1, err error) { // The locking here is, strictly, not required. Without these locks, this can happen: // // 1. NewPayload( execdata-N ) is invoked from the CL. It goes all the way down to @@ -711,6 +775,13 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe // sequentially. // Hence, we use a lock here, to be sure that the previous call has finished before we // check whether we already have the block locally. + var attrs = []telemetry.Attribute{ + telemetry.Int64Attribute("block.number", int64(params.Number)), + telemetry.StringAttribute("block.hash", params.BlockHash.Hex()), + telemetry.Int64Attribute("tx.count", int64(len(params.Transactions))), + } + ctx, _, spanEnd := telemetry.StartSpan(ctx, "engine.newPayload", attrs...) + defer spanEnd(&err) api.newPayloadLock.Lock() defer api.newPayloadLock.Unlock() @@ -725,6 +796,10 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe if params.ExcessBlobGas != nil { ebg = strconv.Itoa(int(*params.ExcessBlobGas)) } + slotNum := "nil" + if params.SlotNumber != nil { + slotNum = strconv.Itoa(int(*params.SlotNumber)) + } log.Warn("Invalid NewPayload params", "params.Number", params.Number, "params.ParentHash", params.ParentHash, @@ -740,6 +815,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe "params.BaseFeePerGas", params.BaseFeePerGas, "params.BlobGasUsed", bgu, "params.ExcessBlobGas", ebg, + "params.SlotNumber", slotNum, "len(params.Transactions)", len(params.Transactions), "len(params.Withdrawals)", len(params.Withdrawals), "beaconRoot", beaconRoot, @@ -789,7 +865,7 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe } log.Trace("Inserting block without sethead", "hash", block.Hash(), "number", block.Number()) start := time.Now() - proofs, err := api.eth.BlockChain().InsertBlockWithoutSetHead(block, witness) + proofs, err := api.eth.BlockChain().InsertBlockWithoutSetHead(ctx, block, witness) processingTime := time.Since(start) if err != nil { log.Warn("NewPayload: inserting block failed", "error", err) diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 4d7246d4ed..db0505101f 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -314,7 +314,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } - newResp, err := api.NewPayloadV1(*execData) + newResp, err := api.NewPayloadV1(context.Background(), *execData) switch { case err != nil: t.Fatalf("Failed to insert block: %v", err) @@ -356,7 +356,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } - newResp, err := api.NewPayloadV1(*execData) + newResp, err := api.NewPayloadV1(context.Background(), *execData) if err != nil || newResp.Status != "VALID" { t.Fatalf("Failed to insert block: %v", err) } @@ -502,7 +502,7 @@ func setupBlocks(t *testing.T, ethservice *eth.Ethereum, n int, parent *types.He } envelope := getNewEnvelope(t, api, parent, w, h) - execResp, err := api.newPayload(*envelope.ExecutionPayload, []common.Hash{}, h, envelope.Requests, false) + execResp, err := api.newPayload(context.Background(), *envelope.ExecutionPayload, []common.Hash{}, h, envelope.Requests, false) if err != nil { t.Fatalf("can't execute payload: %v", err) } @@ -648,7 +648,7 @@ func TestNewPayloadOnInvalidChain(t *testing.T) { t.Fatalf("payload should not be empty") } } - execResp, err := api.NewPayloadV1(*payload.ExecutionPayload) + execResp, err := api.NewPayloadV1(context.Background(), *payload.ExecutionPayload) if err != nil { t.Fatalf("can't execute payload: %v", err) } @@ -708,7 +708,7 @@ func TestEmptyBlocks(t *testing.T) { // (1) check LatestValidHash by sending a normal payload (P1'') payload := getNewPayload(t, api, commonAncestor, nil, nil) - status, err := api.NewPayloadV1(*payload) + status, err := api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -724,7 +724,7 @@ func TestEmptyBlocks(t *testing.T) { payload.GasUsed += 1 payload = setBlockhash(payload) // Now latestValidHash should be the common ancestor - status, err = api.NewPayloadV1(*payload) + status, err = api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -742,7 +742,7 @@ func TestEmptyBlocks(t *testing.T) { payload.ParentHash = common.Hash{1} payload = setBlockhash(payload) // Now latestValidHash should be the common ancestor - status, err = api.NewPayloadV1(*payload) + status, err = api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -859,7 +859,7 @@ func TestTrickRemoteBlockCache(t *testing.T) { // feed the payloads to node B for _, payload := range invalidChain { - status, err := apiB.NewPayloadV1(*payload) + status, err := apiB.NewPayloadV1(context.Background(), *payload) if err != nil { panic(err) } @@ -892,7 +892,7 @@ func TestInvalidBloom(t *testing.T) { // (1) check LatestValidHash by sending a normal payload (P1'') payload := getNewPayload(t, api, commonAncestor, nil, nil) payload.LogsBloom = append(payload.LogsBloom, byte(1)) - status, err := api.NewPayloadV1(*payload) + status, err := api.NewPayloadV1(context.Background(), *payload) if err != nil { t.Fatal(err) } @@ -931,7 +931,7 @@ func TestSimultaneousNewBlock(t *testing.T) { for ii := 0; ii < 10; ii++ { go func() { defer wg.Done() - if newResp, err := api.NewPayloadV1(*execData); err != nil { + if newResp, err := api.NewPayloadV1(context.Background(), *execData); err != nil { errMu.Lock() testErr = fmt.Errorf("failed to insert block: %w", err) errMu.Unlock() @@ -1038,7 +1038,7 @@ func TestWithdrawals(t *testing.T) { } // 10: verify locally built block - if status, err := api.NewPayloadV2(*execData.ExecutionPayload); err != nil { + if status, err := api.NewPayloadV2(context.Background(), *execData.ExecutionPayload); err != nil { t.Fatalf("error validating payload: %v", err) } else if status.Status != engine.VALID { t.Fatalf("invalid payload") @@ -1082,7 +1082,7 @@ func TestWithdrawals(t *testing.T) { if err != nil { t.Fatalf("error getting payload, err=%v", err) } - if status, err := api.NewPayloadV2(*execData.ExecutionPayload); err != nil { + if status, err := api.NewPayloadV2(context.Background(), *execData.ExecutionPayload); err != nil { t.Fatalf("error validating payload: %v", err) } else if status.Status != engine.VALID { t.Fatalf("invalid payload") @@ -1219,15 +1219,20 @@ func TestNilWithdrawals(t *testing.T) { Random: test.blockParams.Random, Version: payloadVersion, }).Id() + if !shanghai { + if _, err := api.GetPayloadV2(payloadID); err != nil { + t.Fatalf("GetPayloadV2 rejected pre-shanghai payload: %v", err) + } + } execData, err := api.getPayload(payloadID, false, nil, nil) if err != nil { t.Fatalf("error getting payload, err=%v", err) } var status engine.PayloadStatusV1 if !shanghai { - status, err = api.NewPayloadV1(*execData.ExecutionPayload) + status, err = api.NewPayloadV1(context.Background(), *execData.ExecutionPayload) } else { - status, err = api.NewPayloadV2(*execData.ExecutionPayload) + status, err = api.NewPayloadV2(context.Background(), *execData.ExecutionPayload) } if err != nil { t.Fatalf("error validating payload: %v", err.(*engine.EngineAPIError).ErrorData()) @@ -1598,7 +1603,7 @@ func TestParentBeaconBlockRoot(t *testing.T) { } // 11: verify locally built block - if status, err := api.NewPayloadV3(*execData.ExecutionPayload, []common.Hash{}, &common.Hash{42}); err != nil { + if status, err := api.NewPayloadV3(context.Background(), *execData.ExecutionPayload, []common.Hash{}, &common.Hash{42}); err != nil { t.Fatalf("error validating payload: %v", err) } else if status.Status != engine.VALID { t.Fatalf("invalid payload") @@ -1705,7 +1710,7 @@ func TestWitnessCreationAndConsumption(t *testing.T) { envelope.ExecutionPayload.StateRoot = wantStateRoot envelope.ExecutionPayload.ReceiptsRoot = wantReceiptRoot - res2, err := api.NewPayloadWithWitnessV3(*envelope.ExecutionPayload, []common.Hash{}, &common.Hash{42}) + res2, err := api.NewPayloadWithWitnessV3(context.Background(), *envelope.ExecutionPayload, []common.Hash{}, &common.Hash{42}) if err != nil { t.Fatalf("error executing stateless payload witness: %v", err) } diff --git a/eth/catalyst/api_testing.go b/eth/catalyst/api_testing.go new file mode 100644 index 0000000000..8586029468 --- /dev/null +++ b/eth/catalyst/api_testing.go @@ -0,0 +1,79 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "errors" + + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/miner" + "github.com/ethereum/go-ethereum/rpc" +) + +// testingAPI implements the testing_ namespace. +// It's an engine-API adjacent namespace for testing purposes. +type testingAPI struct { + eth *eth.Ethereum +} + +func newTestingAPI(backend *eth.Ethereum) rpc.API { + return rpc.API{ + Namespace: "testing", + Service: &testingAPI{backend}, + Version: "1.0", + Authenticated: false, + } +} + +func (api *testingAPI) BuildBlockV1(parentHash common.Hash, payloadAttributes engine.PayloadAttributes, transactions *[]hexutil.Bytes, extraData *hexutil.Bytes) (*engine.ExecutionPayloadEnvelope, error) { + if api.eth.BlockChain().CurrentBlock().Hash() != parentHash { + return nil, errors.New("parentHash is not current head") + } + // If transactions is empty but not nil, build an empty block + // If the transactions is nil, build a block with the current transactions from the txpool + // If the transactions is not nil and not empty, build a block with the transactions + buildEmpty := transactions != nil && len(*transactions) == 0 + var txs []*types.Transaction + if transactions != nil { + dec := make([][]byte, 0, len(*transactions)) + for _, tx := range *transactions { + dec = append(dec, tx) + } + var err error + txs, err = engine.DecodeTransactions(dec) + if err != nil { + return nil, err + } + } + extra := make([]byte, 0) + if extraData != nil { + extra = *extraData + } + args := &miner.BuildPayloadArgs{ + Parent: parentHash, + Timestamp: payloadAttributes.Timestamp, + FeeRecipient: payloadAttributes.SuggestedFeeRecipient, + Random: payloadAttributes.Random, + Withdrawals: payloadAttributes.Withdrawals, + BeaconRoot: payloadAttributes.BeaconRoot, + } + return api.eth.Miner().BuildTestingPayload(args, txs, buildEmpty, extra) +} diff --git a/eth/catalyst/api_testing_test.go b/eth/catalyst/api_testing_test.go new file mode 100644 index 0000000000..fd4d28d26a --- /dev/null +++ b/eth/catalyst/api_testing_test.go @@ -0,0 +1,121 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "context" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" +) + +func TestBuildBlockV1(t *testing.T) { + genesis, blocks := generateMergeChain(5, true) + n, ethservice := startEthService(t, genesis, blocks) + defer n.Close() + + parent := ethservice.BlockChain().CurrentBlock() + attrs := engine.PayloadAttributes{ + Timestamp: parent.Time + 1, + Random: crypto.Keccak256Hash([]byte("test")), + SuggestedFeeRecipient: parent.Coinbase, + Withdrawals: nil, + BeaconRoot: nil, + } + + currentNonce, _ := ethservice.APIBackend.GetPoolNonce(context.Background(), testAddr) + tx, _ := types.SignTx(types.NewTransaction(currentNonce, testAddr, big.NewInt(1), params.TxGas, big.NewInt(params.InitialBaseFee*2), nil), types.LatestSigner(ethservice.BlockChain().Config()), testKey) + + api := &testingAPI{eth: ethservice} + + t.Run("buildOnCurrentHead", func(t *testing.T) { + envelope, err := api.BuildBlockV1(parent.Hash(), attrs, nil, nil) + if err != nil { + t.Fatalf("BuildBlockV1 failed: %v", err) + } + if envelope == nil || envelope.ExecutionPayload == nil { + t.Fatal("expected non-nil envelope and payload") + } + payload := envelope.ExecutionPayload + if payload.ParentHash != parent.Hash() { + t.Errorf("parent hash mismatch: got %x want %x", payload.ParentHash, parent.Hash()) + } + if payload.Number != parent.Number.Uint64()+1 { + t.Errorf("block number mismatch: got %d want %d", payload.Number, parent.Number.Uint64()+1) + } + if payload.Timestamp != attrs.Timestamp { + t.Errorf("timestamp mismatch: got %d want %d", payload.Timestamp, attrs.Timestamp) + } + if payload.FeeRecipient != attrs.SuggestedFeeRecipient { + t.Errorf("fee recipient mismatch: got %x want %x", payload.FeeRecipient, attrs.SuggestedFeeRecipient) + } + }) + + t.Run("wrongParentHash", func(t *testing.T) { + wrongParent := common.Hash{0x01} + _, err := api.BuildBlockV1(wrongParent, attrs, nil, nil) + if err == nil { + t.Fatal("expected error when parentHash is not current head") + } + if err.Error() != "parentHash is not current head" { + t.Errorf("unexpected error: %v", err) + } + }) + + t.Run("buildEmptyBlock", func(t *testing.T) { + emptyTxs := []hexutil.Bytes{} + envelope, err := api.BuildBlockV1(parent.Hash(), attrs, &emptyTxs, nil) + if err != nil { + t.Fatalf("BuildBlockV1 with empty txs failed: %v", err) + } + if envelope == nil || envelope.ExecutionPayload == nil { + t.Fatal("expected non-nil envelope and payload") + } + if len(envelope.ExecutionPayload.Transactions) != 0 { + t.Errorf("expected empty block, got %d transactions", len(envelope.ExecutionPayload.Transactions)) + } + }) + + t.Run("buildBlockWithTransactions", func(t *testing.T) { + enc, _ := tx.MarshalBinary() + txs := []hexutil.Bytes{enc} + envelope, err := api.BuildBlockV1(parent.Hash(), attrs, &txs, nil) + if err != nil { + t.Fatalf("BuildBlockV1 with transaction failed: %v", err) + } + if len(envelope.ExecutionPayload.Transactions) != 1 { + t.Errorf("expected 1 transaction, got %d", len(envelope.ExecutionPayload.Transactions)) + } + }) + + t.Run("buildBlockWithTransactionsFromTxPool", func(t *testing.T) { + ethservice.TxPool().Add([]*types.Transaction{tx}, true) + envelope, err := api.BuildBlockV1(parent.Hash(), attrs, nil, nil) + if err != nil { + t.Fatalf("BuildBlockV1 with transaction failed: %v", err) + } + if len(envelope.ExecutionPayload.Transactions) != 1 { + t.Errorf("expected 1 transaction, got %d", len(envelope.ExecutionPayload.Transactions)) + } + }) +} diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index 92f9798e71..452902c78c 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -17,6 +17,7 @@ package catalyst import ( + "context" "crypto/rand" "crypto/sha256" "errors" @@ -32,11 +33,13 @@ import ( "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/internal/telemetry" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/params/forks" "github.com/ethereum/go-ethereum/rpc" + "go.opentelemetry.io/otel" ) const devEpochLength = 32 @@ -100,6 +103,8 @@ type SimulatedBeacon struct { func payloadVersion(config *params.ChainConfig, time uint64) engine.PayloadVersion { switch config.LatestFork(time) { + case forks.Amsterdam: + return engine.PayloadV4 case forks.BPO5, forks.BPO4, forks.BPO3, forks.BPO2, forks.BPO1, forks.Osaka, forks.Prague, forks.Cancun: return engine.PayloadV3 case forks.Paris, forks.Shanghai: @@ -191,16 +196,23 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u } version := payloadVersion(c.eth.BlockChain().Config(), timestamp) + tracer := otel.Tracer("") var random [32]byte rand.Read(random[:]) - fcResponse, err := c.engineAPI.forkchoiceUpdated(c.curForkchoiceState, &engine.PayloadAttributes{ + + attribute := &engine.PayloadAttributes{ Timestamp: timestamp, SuggestedFeeRecipient: feeRecipient, Withdrawals: withdrawals, Random: random, BeaconRoot: &common.Hash{}, - }, version, false) + } + if c.eth.BlockChain().Config().LatestFork(timestamp) == forks.Amsterdam { + slotNumber := uint64(0) + attribute.SlotNumber = &slotNumber + } + fcResponse, err := c.engineAPI.forkchoiceUpdated(c.curForkchoiceState, attribute, version, false) if err != nil { return err } @@ -255,8 +267,16 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u requests = envelope.Requests } + // Create a server span for newPayload, simulating the consensus client + // sending the execution payload for validation. + npCtx, npSpanEnd := telemetry.StartServerSpan(context.Background(), tracer, telemetry.RPCInfo{ + System: "jsonrpc", + Service: "engine", + Method: "newPayloadV" + fmt.Sprintf("%d", version), + }) // Mark the payload as canon - _, err = c.engineAPI.newPayload(*payload, blobHashes, beaconRoot, requests, false) + _, err = c.engineAPI.newPayload(npCtx, *payload, blobHashes, beaconRoot, requests, false) + npSpanEnd(&err) if err != nil { return err } @@ -364,5 +384,6 @@ func RegisterSimulatedBeaconAPIs(stack *node.Node, sim *SimulatedBeacon) { Service: api, Version: "1.0", }, + newTestingAPI(sim.eth), }) } diff --git a/eth/catalyst/witness.go b/eth/catalyst/witness.go index 0df612a695..14ca29e079 100644 --- a/eth/catalyst/witness.go +++ b/eth/catalyst/witness.go @@ -17,6 +17,7 @@ package catalyst import ( + "context" "errors" "strconv" "time" @@ -86,16 +87,16 @@ func (api *ConsensusAPI) ForkchoiceUpdatedWithWitnessV3(update engine.Forkchoice // NewPayloadWithWitnessV1 is analogous to NewPayloadV1, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV1(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV1(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { if params.Withdrawals != nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("withdrawals not supported in V1")) } - return api.newPayload(params, nil, nil, nil, true) + return api.newPayload(ctx, params, nil, nil, nil, true) } // NewPayloadWithWitnessV2 is analogous to NewPayloadV2, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV2(ctx context.Context, params engine.ExecutableData) (engine.PayloadStatusV1, error) { var ( cancun = api.config().IsCancun(api.config().LondonBlock, params.Timestamp) shanghai = api.config().IsShanghai(api.config().LondonBlock, params.Timestamp) @@ -112,12 +113,12 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV2(params engine.ExecutableData) ( case params.BlobGasUsed != nil: return invalidStatus, paramsErr("non-nil blobGasUsed pre-cancun") } - return api.newPayload(params, nil, nil, nil, true) + return api.newPayload(ctx, params, nil, nil, nil, true) } // NewPayloadWithWitnessV3 is analogous to NewPayloadV3, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV3(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV3(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -132,12 +133,12 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV3(params engine.ExecutableData, v case !api.checkFork(params.Timestamp, forks.Cancun): return invalidStatus, unsupportedForkErr("newPayloadV3 must only be called for cancun payloads") } - return api.newPayload(params, versionedHashes, beaconRoot, nil, true) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, nil, true) } // NewPayloadWithWitnessV4 is analogous to NewPayloadV4, only it also generates // and returns a stateless witness after running the payload. -func (api *ConsensusAPI) NewPayloadWithWitnessV4(params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { +func (api *ConsensusAPI) NewPayloadWithWitnessV4(ctx context.Context, params engine.ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash, executionRequests []hexutil.Bytes) (engine.PayloadStatusV1, error) { switch { case params.Withdrawals == nil: return invalidStatus, paramsErr("nil withdrawals post-shanghai") @@ -158,7 +159,7 @@ func (api *ConsensusAPI) NewPayloadWithWitnessV4(params engine.ExecutableData, v if err := validateRequests(requests); err != nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(err) } - return api.newPayload(params, versionedHashes, beaconRoot, requests, true) + return api.newPayload(ctx, params, versionedHashes, beaconRoot, requests, true) } // ExecuteStatelessPayloadV1 is analogous to NewPayloadV1, only it operates in @@ -283,7 +284,7 @@ func (api *ConsensusAPI) executeStatelessPayload(params engine.ExecutableData, v api.lastNewPayloadUpdate.Store(time.Now().Unix()) log.Trace("Executing block statelessly", "number", block.Number(), "hash", params.BlockHash) - stateRoot, receiptRoot, err := core.ExecuteStateless(api.config(), vm.Config{}, block, witness) + stateRoot, receiptRoot, err := core.ExecuteStateless(context.Background(), api.config(), vm.Config{}, block, witness) if err != nil { log.Warn("ExecuteStatelessPayload: execution failed", "err", err) errorMsg := err.Error() diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index caeb3d64dd..1de0933842 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -959,29 +959,6 @@ func (d *Downloader) processSnapSyncContent() error { } else { // results already piled up, consume before handling pivot move results = append(append([]*fetchResult{oldPivot}, oldTail...), results...) } - // Split around the pivot block and process the two sides via snap/full sync - if !d.committed.Load() { - latest := results[len(results)-1].Header - // If the height is above the pivot block by 2 sets, it means the pivot - // became stale in the network, and it was garbage collected, move to a - // new pivot. - // - // Note, we have `reorgProtHeaderDelay` number of blocks withheld, Those - // need to be taken into account, otherwise we're detecting the pivot move - // late and will drop peers due to unavailable state!!! - if height := latest.Number.Uint64(); height >= pivot.Number.Uint64()+2*uint64(fsMinFullBlocks)-uint64(reorgProtHeaderDelay) { - log.Warn("Pivot became stale, moving", "old", pivot.Number.Uint64(), "new", height-uint64(fsMinFullBlocks)+uint64(reorgProtHeaderDelay)) - pivot = results[len(results)-1-fsMinFullBlocks+reorgProtHeaderDelay].Header // must exist as lower old pivot is uncommitted - - d.pivotLock.Lock() - d.pivotHeader = pivot - d.pivotLock.Unlock() - - // Write out the pivot into the database so a rollback beyond it will - // reenable snap sync - rawdb.WriteLastPivotNumber(d.stateDB, pivot.Number.Uint64()) - } - } P, beforeP, afterP := splitAroundPivot(pivot.Number.Uint64(), results) if err := d.commitSnapSyncData(beforeP, sync); err != nil { return err diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index e5d4a7c59b..f113cf3e9f 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -17,7 +17,6 @@ package downloader import ( - "fmt" "math/big" "sync" "sync/atomic" @@ -261,23 +260,24 @@ func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash, sink chan *et // peer in the download tester. The returned function can be used to retrieve // batches of block receipts from the particularly requested peer. func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash, sink chan *eth.Response) (*eth.Request, error) { - blobs := eth.ServiceGetReceiptsQuery68(dlp.chain, hashes) + blobs := eth.ServiceGetReceiptsQuery(dlp.chain, hashes) + receipts := make([]types.Receipts, blobs.Len()) - receipts := make([]types.Receipts, len(blobs)) - for i, blob := range blobs { - rlp.DecodeBytes(blob, &receipts[i]) - } + // compute hashes + hashes = make([]common.Hash, blobs.Len()) hasher := trie.NewStackTrie(nil) - hashes = make([]common.Hash, len(receipts)) - for i, receipt := range receipts { - hashes[i] = types.DeriveSha(receipt, hasher) + receiptLists, err := blobs.Items() + if err != nil { + panic(err) } - req := ð.Request{ - Peer: dlp.id, + for i, rl := range receiptLists { + hashes[i] = types.DeriveSha(rl.Derivable(), hasher) } + + // deliver the response right away resp := eth.ReceiptsRLPResponse(types.EncodeBlockReceiptLists(receipts)) res := ð.Response{ - Req: req, + Req: ð.Request{Peer: dlp.id}, Res: &resp, Meta: hashes, Time: 1, @@ -286,7 +286,7 @@ func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash, sink chan * go func() { sink <- res }() - return req, nil + return res.Req, nil } // ID retrieves the peer's unique identifier. @@ -398,8 +398,8 @@ func assertOwnChain(t *testing.T, tester *downloadTester, length int) { } } -func TestCanonicalSynchronisation68Full(t *testing.T) { testCanonSync(t, eth.ETH68, FullSync) } -func TestCanonicalSynchronisation68Snap(t *testing.T) { testCanonSync(t, eth.ETH68, SnapSync) } +func TestCanonicalSynchronisationFull(t *testing.T) { testCanonSync(t, eth.ETH69, FullSync) } +func TestCanonicalSynchronisationSnap(t *testing.T) { testCanonSync(t, eth.ETH69, SnapSync) } func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { success := make(chan struct{}) @@ -426,8 +426,8 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. -func TestThrottling68Full(t *testing.T) { testThrottling(t, eth.ETH68, FullSync) } -func TestThrottling68Snap(t *testing.T) { testThrottling(t, eth.ETH68, SnapSync) } +func TestThrottlingFull(t *testing.T) { testThrottling(t, eth.ETH69, FullSync) } +func TestThrottlingSnap(t *testing.T) { testThrottling(t, eth.ETH69, SnapSync) } func testThrottling(t *testing.T, protocol uint, mode SyncMode) { tester := newTester(t, mode) @@ -504,8 +504,8 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { } // Tests that a canceled download wipes all previously accumulated state. -func TestCancel68Full(t *testing.T) { testCancel(t, eth.ETH68, FullSync) } -func TestCancel68Snap(t *testing.T) { testCancel(t, eth.ETH68, SnapSync) } +func TestCancelFull(t *testing.T) { testCancel(t, eth.ETH69, FullSync) } +func TestCancelSnap(t *testing.T) { testCancel(t, eth.ETH69, SnapSync) } func testCancel(t *testing.T, protocol uint, mode SyncMode) { complete := make(chan struct{}) @@ -534,49 +534,10 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { } } -// Tests that synchronisations behave well in multi-version protocol environments -// and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation68Full(t *testing.T) { testMultiProtoSync(t, eth.ETH68, FullSync) } -func TestMultiProtoSynchronisation68Snap(t *testing.T) { testMultiProtoSync(t, eth.ETH68, SnapSync) } - -func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { - complete := make(chan struct{}) - success := func() { - close(complete) - } - tester := newTesterWithNotification(t, mode, success) - defer tester.terminate() - - // Create a small enough block chain to download - chain := testChainBase.shorten(blockCacheMaxItems - 15) - - // Create peers of every type - tester.newPeer("peer 68", eth.ETH68, chain.blocks[1:]) - - if err := tester.downloader.BeaconSync(chain.blocks[len(chain.blocks)-1].Header(), nil); err != nil { - t.Fatalf("failed to start beacon sync: %v", err) - } - select { - case <-complete: - break - case <-time.NewTimer(time.Second * 3).C: - t.Fatalf("Failed to sync chain in three seconds") - } - assertOwnChain(t, tester, len(chain.blocks)) - - // Check that no peers have been dropped off - for _, version := range []int{68} { - peer := fmt.Sprintf("peer %d", version) - if _, ok := tester.peers[peer]; !ok { - t.Errorf("%s dropped", peer) - } - } -} - // Tests that if a block is empty (e.g. header only), no body request should be // made, and instead the header should be assembled into a whole block in itself. -func TestEmptyShortCircuit68Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, FullSync) } -func TestEmptyShortCircuit68Snap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH68, SnapSync) } +func TestEmptyShortCircuitFull(t *testing.T) { testEmptyShortCircuit(t, eth.ETH69, FullSync) } +func TestEmptyShortCircuitSnap(t *testing.T) { testEmptyShortCircuit(t, eth.ETH69, SnapSync) } func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { success := make(chan struct{}) @@ -644,8 +605,8 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync // Tests that peers below a pre-configured checkpoint block are prevented from // being fast-synced from, avoiding potential cheap eclipse attacks. -func TestBeaconSync68Full(t *testing.T) { testBeaconSync(t, eth.ETH68, FullSync) } -func TestBeaconSync68Snap(t *testing.T) { testBeaconSync(t, eth.ETH68, SnapSync) } +func TestBeaconSyncFull(t *testing.T) { testBeaconSync(t, eth.ETH69, FullSync) } +func TestBeaconSyncSnap(t *testing.T) { testBeaconSync(t, eth.ETH69, SnapSync) } func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) { var cases = []struct { @@ -690,8 +651,8 @@ func testBeaconSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. -func TestSyncProgress68Full(t *testing.T) { testSyncProgress(t, eth.ETH68, FullSync) } -func TestSyncProgress68Snap(t *testing.T) { testSyncProgress(t, eth.ETH68, SnapSync) } +func TestSyncProgressFull(t *testing.T) { testSyncProgress(t, eth.ETH69, FullSync) } +func TestSyncProgressSnap(t *testing.T) { testSyncProgress(t, eth.ETH69, SnapSync) } func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { success := make(chan struct{}) diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index 5c54b4b5c2..8c38e9d0c5 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -845,7 +845,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { // Create a peer set to feed headers through peerset := newPeerSet() for _, peer := range tt.peers { - peerset.Register(newPeerConnection(peer.id, eth.ETH68, peer, log.New("id", peer.id))) + peerset.Register(newPeerConnection(peer.id, eth.ETH69, peer, log.New("id", peer.id))) } // Create a peer dropper to track malicious peers dropped := make(map[string]int) @@ -912,7 +912,7 @@ func TestSkeletonSyncRetrievals(t *testing.T) { // Apply the post-init events if there's any endpeers := tt.peers if tt.newPeer != nil { - if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH68, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil { + if err := peerset.Register(newPeerConnection(tt.newPeer.id, eth.ETH69, tt.newPeer, log.New("id", tt.newPeer.id))); err != nil { t.Errorf("test %d: failed to register new peer: %v", i, err) } time.Sleep(time.Millisecond * 50) // given time for peer registration diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 05bf524c09..4aae914846 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -60,11 +60,11 @@ var Defaults = Config{ StateHistory: pathdb.Defaults.StateHistory, TrienodeHistory: pathdb.Defaults.TrienodeHistory, NodeFullValueCheckpoint: pathdb.Defaults.FullValueCheckpoint, - DatabaseCache: 512, - TrieCleanCache: 154, - TrieDirtyCache: 256, + DatabaseCache: 2048, + TrieCleanCache: 614, + TrieDirtyCache: 1024, + SnapshotCache: 409, TrieTimeout: 60 * time.Minute, - SnapshotCache: 102, FilterLogCacheSize: 32, LogQueryLimit: 1000, Miner: miner.DefaultConfig, diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go index fd1678dd30..3c0d6a8fd8 100644 --- a/eth/fetcher/metrics.go +++ b/eth/fetcher/metrics.go @@ -24,6 +24,7 @@ var ( txAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/in", nil) txAnnounceKnownMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/known", nil) txAnnounceUnderpricedMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/underpriced", nil) + txAnnounceOnchainMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/onchain", nil) txAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/dos", nil) txBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/in", nil) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 50d6f2f7ad..5817dfbcf5 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" @@ -71,6 +72,11 @@ const ( // addTxsBatchSize it the max number of transactions to add in a single batch from a peer. addTxsBatchSize = 128 + + // txOnChainCacheLimit is number of on-chain transactions to keep in a cache to avoid + // re-fetching them soon after they are mined. + // Approx 1MB for 30 minutes of transactions at 18 tps + txOnChainCacheLimit = 32768 ) var ( @@ -152,6 +158,9 @@ type TxFetcher struct { txSeq uint64 // Unique transaction sequence number underpriced *lru.Cache[common.Hash, time.Time] // Transactions discarded as too cheap (don't re-fetch) + chain *core.BlockChain // Blockchain interface for on-chain checks + txOnChainCache *lru.Cache[common.Hash, struct{}] // Cache to avoid fetching once the tx gets on chain + // Stage 1: Waiting lists for newly discovered transactions that might be // broadcast without needing explicit request/reply round trips. waitlist map[common.Hash]map[string]struct{} // Transactions waiting for an potential broadcast @@ -184,36 +193,40 @@ type TxFetcher struct { // NewTxFetcher creates a transaction fetcher to retrieve transaction // based on hash announcements. -func NewTxFetcher(validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string)) *TxFetcher { - return NewTxFetcherForTests(validateMeta, addTxs, fetchTxs, dropPeer, mclock.System{}, time.Now, nil) +// Chain can be nil to disable on-chain checks. +func NewTxFetcher(chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string)) *TxFetcher { + return NewTxFetcherForTests(chain, validateMeta, addTxs, fetchTxs, dropPeer, mclock.System{}, time.Now, nil) } // NewTxFetcherForTests is a testing method to mock out the realtime clock with // a simulated version and the internal randomness with a deterministic one. +// Chain can be nil to disable on-chain checks. func NewTxFetcherForTests( - validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), + chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), clock mclock.Clock, realTime func() time.Time, rand *mrand.Rand) *TxFetcher { return &TxFetcher{ - notify: make(chan *txAnnounce), - cleanup: make(chan *txDelivery), - drop: make(chan *txDrop), - quit: make(chan struct{}), - waitlist: make(map[common.Hash]map[string]struct{}), - waittime: make(map[common.Hash]mclock.AbsTime), - waitslots: make(map[string]map[common.Hash]*txMetadataWithSeq), - announces: make(map[string]map[common.Hash]*txMetadataWithSeq), - announced: make(map[common.Hash]map[string]struct{}), - fetching: make(map[common.Hash]string), - requests: make(map[string]*txRequest), - alternates: make(map[common.Hash]map[string]struct{}), - underpriced: lru.NewCache[common.Hash, time.Time](maxTxUnderpricedSetSize), - validateMeta: validateMeta, - addTxs: addTxs, - fetchTxs: fetchTxs, - dropPeer: dropPeer, - clock: clock, - realTime: realTime, - rand: rand, + notify: make(chan *txAnnounce), + cleanup: make(chan *txDelivery), + drop: make(chan *txDrop), + quit: make(chan struct{}), + waitlist: make(map[common.Hash]map[string]struct{}), + waittime: make(map[common.Hash]mclock.AbsTime), + waitslots: make(map[string]map[common.Hash]*txMetadataWithSeq), + announces: make(map[string]map[common.Hash]*txMetadataWithSeq), + announced: make(map[common.Hash]map[string]struct{}), + fetching: make(map[common.Hash]string), + requests: make(map[string]*txRequest), + alternates: make(map[common.Hash]map[string]struct{}), + underpriced: lru.NewCache[common.Hash, time.Time](maxTxUnderpricedSetSize), + txOnChainCache: lru.NewCache[common.Hash, struct{}](txOnChainCacheLimit), + chain: chain, + validateMeta: validateMeta, + addTxs: addTxs, + fetchTxs: fetchTxs, + dropPeer: dropPeer, + clock: clock, + realTime: realTime, + rand: rand, } } @@ -233,6 +246,7 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c unknownMetas = make([]txMetadata, 0, len(hashes)) duplicate int64 + onchain int64 underpriced int64 ) for i, hash := range hashes { @@ -245,6 +259,12 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c continue } + // check on chain as well (no need to check limbo separately, as chain checks limbo too) + if _, exist := f.txOnChainCache.Get(hash); exist { + onchain++ + continue + } + if f.isKnownUnderpriced(hash) { underpriced++ continue @@ -259,6 +279,7 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c } txAnnounceKnownMeter.Mark(duplicate) txAnnounceUnderpricedMeter.Mark(underpriced) + txAnnounceOnchainMeter.Mark(onchain) // If anything's left to announce, push it into the internal loop if len(unknownHashes) == 0 { @@ -412,7 +433,20 @@ func (f *TxFetcher) loop() { waitTrigger = make(chan struct{}, 1) timeoutTrigger = make(chan struct{}, 1) + + oldHead *types.Header ) + + // Subscribe to chain events to know when transactions are added to chain + var headEventCh chan core.ChainEvent + if f.chain != nil { + headEventCh = make(chan core.ChainEvent, 10) + sub := f.chain.SubscribeChainEvent(headEventCh) + if sub != nil { + defer sub.Unsubscribe() + } + } + for { select { case ann := <-f.notify: @@ -837,6 +871,21 @@ func (f *TxFetcher) loop() { f.rescheduleTimeout(timeoutTimer, timeoutTrigger) } + case ev := <-headEventCh: + // New head(s) added + newHead := ev.Header + if oldHead != nil && newHead.ParentHash != oldHead.Hash() { + // Reorg or setHead detected, clear the cache. We could be smarter here and + // only remove/add the diff, but this is simpler and not being exact here + // only results in a few more fetches. + f.txOnChainCache.Purge() + } + oldHead = newHead + // Add all transactions from the new block to the on-chain cache + for _, tx := range ev.Transactions { + f.txOnChainCache.Add(tx.Hash(), struct{}{}) + } + case <-f.quit: return } diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index 87fbe9f38c..6c2719631e 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -91,6 +91,7 @@ type txFetcherTest struct { // and deterministic randomness. func newTestTxFetcher() *TxFetcher { return NewTxFetcher( + nil, func(common.Hash, byte) error { return nil }, func(txs []*types.Transaction) []error { return make([]error, len(txs)) @@ -2191,6 +2192,7 @@ func TestTransactionForgotten(t *testing.T) { } fetcher := NewTxFetcherForTests( + nil, func(common.Hash, byte) error { return nil }, func(txs []*types.Transaction) []error { errs := make([]error, len(txs)) diff --git a/eth/filters/api.go b/eth/filters/api.go index f4bed35b26..2cb72dc114 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -187,11 +187,13 @@ func (api *FilterAPI) NewPendingTransactions(ctx context.Context, fullTx *bool) return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported } - rpcSub := notifier.CreateSubscription() + var ( + rpcSub = notifier.CreateSubscription() + txs = make(chan []*types.Transaction, 128) + pendingTxSub = api.events.SubscribePendingTxs(txs) + ) go func() { - txs := make(chan []*types.Transaction, 128) - pendingTxSub := api.events.SubscribePendingTxs(txs) defer pendingTxSub.Unsubscribe() chainConfig := api.sys.backend.ChainConfig() @@ -260,11 +262,13 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported } - rpcSub := notifier.CreateSubscription() + var ( + rpcSub = notifier.CreateSubscription() + headers = make(chan *types.Header) + headersSub = api.events.SubscribeNewHeads(headers) + ) go func() { - headers := make(chan *types.Header) - headersSub := api.events.SubscribeNewHeads(headers) defer headersSub.Unsubscribe() for { diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go index 6e79fbd62b..80aeb3d3b2 100644 --- a/eth/gasestimator/gasestimator.go +++ b/eth/gasestimator/gasestimator.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "math" "math/big" "github.com/ethereum/go-ethereum/common" @@ -268,7 +267,7 @@ func run(ctx context.Context, call *core.Message, opts *Options) (*core.Executio evm.Cancel() }() // Execute the call, returning a wrapped error or the result - result, err := core.ApplyMessage(evm, call, new(core.GasPool).AddGas(math.MaxUint64)) + result, err := core.ApplyMessage(evm, call, nil) if vmerr := dirtyState.Error(); vmerr != nil { return nil, vmerr } diff --git a/eth/handler.go b/eth/handler.go index 46634cae88..bb2cd5f88b 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -179,7 +179,6 @@ func newHandler(config *handlerConfig) (*handler, error) { addTxs := func(txs []*types.Transaction) []error { return h.txpool.Add(txs, false) } - validateMeta := func(tx common.Hash, kind byte) error { if h.txpool.Has(tx) { return txpool.ErrAlreadyKnown @@ -189,8 +188,7 @@ func newHandler(config *handlerConfig) (*handler, error) { } return nil } - - h.txFetcher = fetcher.NewTxFetcher(validateMeta, addTxs, fetchTx, h.removePeer) + h.txFetcher = fetcher.NewTxFetcher(h.chain, validateMeta, addTxs, fetchTx, h.removePeer) return h, nil } diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 0330713071..68e91fa897 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -38,9 +38,8 @@ import ( // testEthHandler is a mock event handler to listen for inbound network requests // on the `eth` protocol and convert them into a more easily testable form. type testEthHandler struct { - blockBroadcasts event.Feed - txAnnounces event.Feed - txBroadcasts event.Feed + txAnnounces event.Feed + txBroadcasts event.Feed } func (h *testEthHandler) Chain() *core.BlockChain { panic("no backing chain") } @@ -51,10 +50,6 @@ func (h *testEthHandler) PeerInfo(enode.ID) interface{} { panic("not used func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { switch packet := packet.(type) { - case *eth.NewBlockPacket: - h.blockBroadcasts.Send(packet.Block) - return nil - case *eth.NewPooledTransactionHashesPacket: h.txAnnounces.Send(packet.Hashes) return nil @@ -82,7 +77,7 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. -func TestForkIDSplit68(t *testing.T) { testForkIDSplit(t, eth.ETH68) } +func TestForkIDSplit69(t *testing.T) { testForkIDSplit(t, eth.ETH69) } func testForkIDSplit(t *testing.T, protocol uint) { t.Parallel() @@ -234,7 +229,7 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. -func TestRecvTransactions68(t *testing.T) { testRecvTransactions(t, eth.ETH68) } +func TestRecvTransactions69(t *testing.T) { testRecvTransactions(t, eth.ETH69) } func testRecvTransactions(t *testing.T, protocol uint) { t.Parallel() @@ -263,7 +258,8 @@ func testRecvTransactions(t *testing.T, protocol uint) { return eth.Handle((*ethHandler)(handler.handler), peer) }) // Run the handshake locally to avoid spinning up a source handler - if err := src.Handshake(1, handler.chain, eth.BlockRangeUpdatePacket{}); err != nil { + head := handler.chain.CurrentBlock() + if err := src.Handshake(1, handler.chain, eth.BlockRangeUpdatePacket{EarliestBlock: 0, LatestBlock: head.Number.Uint64(), LatestBlockHash: head.Hash()}); err != nil { t.Fatalf("failed to run protocol handshake") } // Send the transaction to the sink and verify that it's added to the tx pool @@ -286,7 +282,7 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. -func TestSendTransactions68(t *testing.T) { testSendTransactions(t, eth.ETH68) } +func TestSendTransactions69(t *testing.T) { testSendTransactions(t, eth.ETH69) } func testSendTransactions(t *testing.T, protocol uint) { t.Parallel() @@ -318,7 +314,8 @@ func testSendTransactions(t *testing.T, protocol uint) { return eth.Handle((*ethHandler)(handler.handler), peer) }) // Run the handshake locally to avoid spinning up a source handler - if err := sink.Handshake(1, handler.chain, eth.BlockRangeUpdatePacket{}); err != nil { + head := handler.chain.CurrentBlock() + if err := sink.Handshake(1, handler.chain, eth.BlockRangeUpdatePacket{EarliestBlock: 0, LatestBlock: head.Number.Uint64(), LatestBlockHash: head.Hash()}); err != nil { t.Fatalf("failed to run protocol handshake") } // After the handshake completes, the source handler should stream the sink @@ -338,22 +335,16 @@ func testSendTransactions(t *testing.T, protocol uint) { // Make sure we get all the transactions on the correct channels seen := make(map[common.Hash]struct{}) for len(seen) < len(insert) { - switch protocol { - case 68: - select { - case hashes := <-anns: - for _, hash := range hashes { - if _, ok := seen[hash]; ok { - t.Errorf("duplicate transaction announced: %x", hash) - } - seen[hash] = struct{}{} + select { + case hashes := <-anns: + for _, hash := range hashes { + if _, ok := seen[hash]; ok { + t.Errorf("duplicate transaction announced: %x", hash) } - case <-bcasts: - t.Errorf("initial tx broadcast received on post eth/66") + seen[hash] = struct{}{} } - - default: - panic("unsupported protocol, please extend test") + case <-bcasts: + t.Errorf("initial tx broadcast received on post eth/66") } } for _, tx := range insert { @@ -365,7 +356,7 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. -func TestTransactionPropagation68(t *testing.T) { testTransactionPropagation(t, eth.ETH68) } +func TestTransactionPropagation69(t *testing.T) { testTransactionPropagation(t, eth.ETH69) } func testTransactionPropagation(t *testing.T, protocol uint) { t.Parallel() diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 2467e0c713..aa1c7d45bc 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -166,21 +166,6 @@ type Decoder interface { Time() time.Time } -var eth68 = map[uint64]msgHandler{ - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - GetBlockHeadersMsg: handleGetBlockHeaders, - BlockHeadersMsg: handleBlockHeaders, - GetBlockBodiesMsg: handleGetBlockBodies, - BlockBodiesMsg: handleBlockBodies, - GetReceiptsMsg: handleGetReceipts68, - ReceiptsMsg: handleReceipts[*ReceiptList68], - GetPooledTransactionsMsg: handleGetPooledTransactions, - PooledTransactionsMsg: handlePooledTransactions, -} - var eth69 = map[uint64]msgHandler{ TransactionsMsg: handleTransactions, NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, @@ -188,8 +173,8 @@ var eth69 = map[uint64]msgHandler{ BlockHeadersMsg: handleBlockHeaders, GetBlockBodiesMsg: handleGetBlockBodies, BlockBodiesMsg: handleBlockBodies, - GetReceiptsMsg: handleGetReceipts69, - ReceiptsMsg: handleReceipts[*ReceiptList69], + GetReceiptsMsg: handleGetReceipts, + ReceiptsMsg: handleReceipts, GetPooledTransactionsMsg: handleGetPooledTransactions, PooledTransactionsMsg: handlePooledTransactions, BlockRangeUpdateMsg: handleBlockRangeUpdate, @@ -209,9 +194,7 @@ func handleMessage(backend Backend, peer *Peer) error { defer msg.Discard() var handlers map[uint64]msgHandler - if peer.version == ETH68 { - handlers = eth68 - } else if peer.version == ETH69 { + if peer.version == ETH69 { handlers = eth69 } else { return fmt.Errorf("unknown eth protocol version: %v", peer.version) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 8f7f82c3a1..2e0ce0408b 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -174,7 +174,7 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. -func TestGetBlockHeaders68(t *testing.T) { testGetBlockHeaders(t, ETH68) } +func TestGetBlockHeaders69(t *testing.T) { testGetBlockHeaders(t, ETH69) } func testGetBlockHeaders(t *testing.T, protocol uint) { t.Parallel() @@ -387,7 +387,7 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. -func TestGetBlockBodies68(t *testing.T) { testGetBlockBodies(t, ETH68) } +func TestGetBlockBodies69(t *testing.T) { testGetBlockBodies(t, ETH69) } func testGetBlockBodies(t *testing.T, protocol uint) { t.Parallel() @@ -536,7 +536,7 @@ func TestHashBody(t *testing.T) { } // Tests that the transaction receipts can be retrieved based on hashes. -func TestGetBlockReceipts68(t *testing.T) { testGetBlockReceipts(t, ETH68) } +func TestGetBlockReceipts69(t *testing.T) { testGetBlockReceipts(t, ETH69) } func testGetBlockReceipts(t *testing.T, protocol uint) { t.Parallel() @@ -586,13 +586,13 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { // Collect the hashes to request, and the response to expect var ( hashes []common.Hash - receipts rlp.RawList[*ReceiptList68] + receipts rlp.RawList[*ReceiptList] ) for i := uint64(0); i <= backend.chain.CurrentBlock().Number.Uint64(); i++ { block := backend.chain.GetBlockByNumber(i) hashes = append(hashes, block.Hash()) - trs := backend.chain.GetReceiptsByHash(block.Hash()) - receipts.Append(NewReceiptList68(trs)) + br := backend.chain.GetReceiptsByHash(block.Hash()) + receipts.Append(NewReceiptList(br)) } // Send the hash request and verify the response @@ -600,7 +600,7 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { RequestId: 123, GetReceiptsRequest: hashes, }) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, &ReceiptsPacket[*ReceiptList68]{ + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, &ReceiptsPacket{ RequestId: 123, List: receipts, }); err != nil { @@ -656,7 +656,7 @@ func setup() (*testBackend, *testPeer) { } } backend := newTestBackendWithGenerator(maxBodiesServe+15, true, false, gen) - peer, _ := newTestPeer("peer", ETH68, backend) + peer, _ := newTestPeer("peer", ETH69, backend) // Discard all messages go func() { for { @@ -701,7 +701,7 @@ func testGetPooledTransaction(t *testing.T, blobTx bool) { backend := newTestBackendWithGenerator(0, true, true, nil) defer backend.close() - peer, _ := newTestPeer("peer", ETH68, backend) + peer, _ := newTestPeer("peer", ETH69, backend) defer peer.close() var ( diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 7f1ccc360d..90717472f9 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -19,7 +19,6 @@ package eth import ( "bytes" "encoding/json" - "errors" "fmt" "math" @@ -247,36 +246,27 @@ func ServiceGetBlockBodiesQuery(chain *core.BlockChain, query GetBlockBodiesRequ return bodies } -func handleGetReceipts68(backend Backend, msg Decoder, peer *Peer) error { +func handleGetReceipts(backend Backend, msg Decoder, peer *Peer) error { // Decode the block receipts retrieval message var query GetReceiptsPacket if err := msg.Decode(&query); err != nil { return err } - response := ServiceGetReceiptsQuery68(backend.Chain(), query.GetReceiptsRequest) + response := ServiceGetReceiptsQuery(backend.Chain(), query.GetReceiptsRequest) return peer.ReplyReceiptsRLP(query.RequestId, response) } -func handleGetReceipts69(backend Backend, msg Decoder, peer *Peer) error { - // Decode the block receipts retrieval message - var query GetReceiptsPacket - if err := msg.Decode(&query); err != nil { - return err - } - response := serviceGetReceiptsQuery69(backend.Chain(), query.GetReceiptsRequest) - return peer.ReplyReceiptsRLP(query.RequestId, response) -} - -// ServiceGetReceiptsQuery68 assembles the response to a receipt query. It is -// exposed to allow external packages to test protocol behavior. -func ServiceGetReceiptsQuery68(chain *core.BlockChain, query GetReceiptsRequest) []rlp.RawValue { +// ServiceGetReceiptsQuery assembles the response to a receipt query. +// It does not send the bloom filters for the receipts. It is exposed +// to allow external packages to test protocol behavior. +func ServiceGetReceiptsQuery(chain *core.BlockChain, query GetReceiptsRequest) rlp.RawList[*ReceiptList] { // Gather state data until the fetch or network limits is reached var ( bytes int - receipts []rlp.RawValue + receipts rlp.RawList[*ReceiptList] ) for lookups, hash := range query { - if bytes >= softResponseLimit || len(receipts) >= maxReceiptsServe || + if bytes >= softResponseLimit || receipts.Len() >= maxReceiptsServe || lookups >= 2*maxReceiptsServe { break } @@ -292,63 +282,18 @@ func ServiceGetReceiptsQuery68(chain *core.BlockChain, query GetReceiptsRequest) continue } var err error - results, err = blockReceiptsToNetwork68(results, body) + results, err = blockReceiptsToNetwork(results, body) if err != nil { log.Error("Error in block receipts conversion", "hash", hash, "err", err) continue } } - receipts = append(receipts, results) + receipts.AppendRaw(results) bytes += len(results) } return receipts } -// serviceGetReceiptsQuery69 assembles the response to a receipt query. -// It does not send the bloom filters for the receipts -func serviceGetReceiptsQuery69(chain *core.BlockChain, query GetReceiptsRequest) []rlp.RawValue { - // Gather state data until the fetch or network limits is reached - var ( - bytes int - receipts []rlp.RawValue - ) - for lookups, hash := range query { - if bytes >= softResponseLimit || len(receipts) >= maxReceiptsServe || - lookups >= 2*maxReceiptsServe { - break - } - // Retrieve the requested block's receipts - results := chain.GetReceiptsRLP(hash) - if results == nil { - if header := chain.GetHeaderByHash(hash); header == nil || header.ReceiptHash != types.EmptyRootHash { - continue - } - } else { - body := chain.GetBodyRLP(hash) - if body == nil { - continue - } - var err error - results, err = blockReceiptsToNetwork69(results, body) - if err != nil { - log.Error("Error in block receipts conversion", "hash", hash, "err", err) - continue - } - } - receipts = append(receipts, results) - bytes += len(results) - } - return receipts -} - -func handleNewBlockhashes(backend Backend, msg Decoder, peer *Peer) error { - return errors.New("block announcements disallowed") // We dropped support for non-merge networks -} - -func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { - return errors.New("block broadcasts disallowed") // We dropped support for non-merge networks -} - func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { // A batch of headers arrived to one of our previous requests res := new(BlockHeadersPacket) @@ -490,9 +435,9 @@ func writeTxForHash(tx []byte, buf *bytes.Buffer) { } } -func handleReceipts[L ReceiptsList](backend Backend, msg Decoder, peer *Peer) error { +func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { // A batch of receipts arrived to one of our previous requests - res := new(ReceiptsPacket[L]) + res := new(ReceiptsPacket) if err := msg.Decode(res); err != nil { return err } @@ -502,16 +447,10 @@ func handleReceipts[L ReceiptsList](backend Backend, msg Decoder, peer *Peer) er return fmt.Errorf("Receipts: %w", err) } - // Assign temporary hashing buffer to each list item, the same buffer is shared - // between all receipt list instances. receiptLists, err := res.List.Items() if err != nil { return fmt.Errorf("Receipts: %w", err) } - buffers := new(receiptListBuffers) - for i := range receiptLists { - receiptLists[i].setBuffers(buffers) - } metadata := func() interface{} { hasher := trie.NewStackTrie(nil) @@ -521,6 +460,7 @@ func handleReceipts[L ReceiptsList](backend Backend, msg Decoder, peer *Peer) er } return hashes } + var enc ReceiptsRLPResponse for i := range receiptLists { encReceipts, err := receiptLists[i].EncodeForStorage() diff --git a/eth/protocols/eth/handshake.go b/eth/protocols/eth/handshake.go index bb3d1b8eb4..359e4e36bb 100644 --- a/eth/protocols/eth/handshake.go +++ b/eth/protocols/eth/handshake.go @@ -36,62 +36,6 @@ const ( // Handshake executes the eth protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. func (p *Peer) Handshake(networkID uint64, chain forkid.Blockchain, rangeMsg BlockRangeUpdatePacket) error { - switch p.version { - case ETH69: - return p.handshake69(networkID, chain, rangeMsg) - case ETH68: - return p.handshake68(networkID, chain) - default: - return errors.New("unsupported protocol version") - } -} - -func (p *Peer) handshake68(networkID uint64, chain forkid.Blockchain) error { - var ( - genesis = chain.Genesis() - latest = chain.CurrentHeader() - forkID = forkid.NewID(chain.Config(), genesis, latest.Number.Uint64(), latest.Time) - forkFilter = forkid.NewFilter(chain) - ) - errc := make(chan error, 2) - go func() { - pkt := &StatusPacket68{ - ProtocolVersion: uint32(p.version), - NetworkID: networkID, - Head: latest.Hash(), - Genesis: genesis.Hash(), - ForkID: forkID, - } - errc <- p2p.Send(p.rw, StatusMsg, pkt) - }() - var status StatusPacket68 // safe to read after two values have been received from errc - go func() { - errc <- p.readStatus68(networkID, &status, genesis.Hash(), forkFilter) - }() - - return waitForHandshake(errc, p) -} - -func (p *Peer) readStatus68(networkID uint64, status *StatusPacket68, genesis common.Hash, forkFilter forkid.Filter) error { - if err := p.readStatusMsg(status); err != nil { - return err - } - if status.NetworkID != networkID { - return fmt.Errorf("%w: %d (!= %d)", errNetworkIDMismatch, status.NetworkID, networkID) - } - if uint(status.ProtocolVersion) != p.version { - return fmt.Errorf("%w: %d (!= %d)", errProtocolVersionMismatch, status.ProtocolVersion, p.version) - } - if status.Genesis != genesis { - return fmt.Errorf("%w: %x (!= %x)", errGenesisMismatch, status.Genesis, genesis) - } - if err := forkFilter(status.ForkID); err != nil { - return fmt.Errorf("%w: %v", errForkIDRejected, err) - } - return nil -} - -func (p *Peer) handshake69(networkID uint64, chain forkid.Blockchain, rangeMsg BlockRangeUpdatePacket) error { var ( genesis = chain.Genesis() latest = chain.CurrentHeader() @@ -101,7 +45,7 @@ func (p *Peer) handshake69(networkID uint64, chain forkid.Blockchain, rangeMsg B errc := make(chan error, 2) go func() { - pkt := &StatusPacket69{ + pkt := &StatusPacket{ ProtocolVersion: uint32(p.version), NetworkID: networkID, Genesis: genesis.Hash(), @@ -112,15 +56,15 @@ func (p *Peer) handshake69(networkID uint64, chain forkid.Blockchain, rangeMsg B } errc <- p2p.Send(p.rw, StatusMsg, pkt) }() - var status StatusPacket69 // safe to read after two values have been received from errc + var status StatusPacket // safe to read after two values have been received from errc go func() { - errc <- p.readStatus69(networkID, &status, genesis.Hash(), forkFilter) + errc <- p.readStatus(networkID, &status, genesis.Hash(), forkFilter) }() return waitForHandshake(errc, p) } -func (p *Peer) readStatus69(networkID uint64, status *StatusPacket69, genesis common.Hash, forkFilter forkid.Filter) error { +func (p *Peer) readStatus(networkID uint64, status *StatusPacket, genesis common.Hash, forkFilter forkid.Filter) error { if err := p.readStatusMsg(status); err != nil { return err } @@ -191,16 +135,16 @@ func markError(p *Peer, err error) { return } m := meters.get(p.Inbound()) - switch errors.Unwrap(err) { - case errNetworkIDMismatch: + switch { + case errors.Is(err, errNetworkIDMismatch): m.networkIDMismatch.Mark(1) - case errProtocolVersionMismatch: + case errors.Is(err, errProtocolVersionMismatch): m.protocolVersionMismatch.Mark(1) - case errGenesisMismatch: + case errors.Is(err, errGenesisMismatch): m.genesisMismatch.Mark(1) - case errForkIDRejected: + case errors.Is(err, errForkIDRejected): m.forkidRejected.Mark(1) - case p2p.DiscReadTimeout: + case errors.Is(err, p2p.DiscReadTimeout): m.timeoutError.Mark(1) default: m.peerError.Mark(1) diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index 2fab3ea5a8..e2f1e7592a 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -18,7 +18,6 @@ package eth import ( "errors" - "math/big" "testing" "github.com/ethereum/go-ethereum/common" @@ -28,7 +27,7 @@ import ( ) // Tests that handshake failures are detected and reported correctly. -func TestHandshake68(t *testing.T) { testHandshake(t, ETH68) } +func TestHandshake69(t *testing.T) { testHandshake(t, ETH69) } func testHandshake(t *testing.T, protocol uint) { t.Parallel() @@ -52,21 +51,25 @@ func testHandshake(t *testing.T, protocol uint) { want: errNoStatusMsg, }, { - code: StatusMsg, data: StatusPacket68{10, 1, new(big.Int), head.Hash(), genesis.Hash(), forkID}, + code: StatusMsg, data: StatusPacket{10, 1, genesis.Hash(), forkID, 0, head.Number.Uint64(), head.Hash()}, want: errProtocolVersionMismatch, }, { - code: StatusMsg, data: StatusPacket68{uint32(protocol), 999, new(big.Int), head.Hash(), genesis.Hash(), forkID}, + code: StatusMsg, data: StatusPacket{uint32(protocol), 999, genesis.Hash(), forkID, 0, head.Number.Uint64(), head.Hash()}, want: errNetworkIDMismatch, }, { - code: StatusMsg, data: StatusPacket68{uint32(protocol), 1, new(big.Int), head.Hash(), common.Hash{3}, forkID}, + code: StatusMsg, data: StatusPacket{uint32(protocol), 1, common.Hash{3}, forkID, 0, head.Number.Uint64(), head.Hash()}, want: errGenesisMismatch, }, { - code: StatusMsg, data: StatusPacket68{uint32(protocol), 1, new(big.Int), head.Hash(), genesis.Hash(), forkid.ID{Hash: [4]byte{0x00, 0x01, 0x02, 0x03}}}, + code: StatusMsg, data: StatusPacket{uint32(protocol), 1, genesis.Hash(), forkid.ID{Hash: [4]byte{0x00, 0x01, 0x02, 0x03}}, 0, head.Number.Uint64(), head.Hash()}, want: errForkIDRejected, }, + { + code: StatusMsg, data: StatusPacket{uint32(protocol), 1, genesis.Hash(), forkID, head.Number.Uint64() + 1, head.Number.Uint64(), head.Hash()}, + want: errInvalidBlockRange, + }, } for i, test := range tests { // Create the two peers to shake with each other diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 4ea2d7158c..3c6d58d670 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -215,10 +215,10 @@ func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { } // ReplyReceiptsRLP is the response to GetReceipts. -func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { - return p2p.Send(p.rw, ReceiptsMsg, &ReceiptsRLPPacket{ - RequestId: id, - ReceiptsRLPResponse: receipts, +func (p *Peer) ReplyReceiptsRLP(id uint64, receipts rlp.RawList[*ReceiptList]) error { + return p2p.Send(p.rw, ReceiptsMsg, &ReceiptsPacket{ + RequestId: id, + List: receipts, }) } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 6ab800f4f4..ef65a7d034 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -20,7 +20,6 @@ import ( "errors" "fmt" "io" - "math/big" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/forkid" @@ -30,7 +29,6 @@ import ( // Constants to match up protocol versions and messages const ( - ETH68 = 68 ETH69 = 69 ) @@ -40,11 +38,11 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH69, ETH68} +var ProtocolVersions = []uint{ETH69} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH68: 17, ETH69: 18} +var protocolLengths = map[uint]uint64{ETH69: 18} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 @@ -88,17 +86,7 @@ type Packet interface { } // StatusPacket is the network packet for the status message. -type StatusPacket68 struct { - ProtocolVersion uint32 - NetworkID uint64 - TD *big.Int - Head common.Hash - Genesis common.Hash - ForkID forkid.ID -} - -// StatusPacket69 is the network packet for the status message. -type StatusPacket69 struct { +type StatusPacket struct { ProtocolVersion uint32 NetworkID uint64 Genesis common.Hash @@ -109,26 +97,6 @@ type StatusPacket69 struct { LatestBlockHash common.Hash } -// NewBlockHashesPacket is the network packet for the block announcements. -type NewBlockHashesPacket []struct { - Hash common.Hash // Hash of one particular block being announced - Number uint64 // Number of one particular block being announced -} - -// Unpack retrieves the block hashes and numbers from the announcement packet -// and returns them in a split flat format that's more consistent with the -// internal data structures. -func (p *NewBlockHashesPacket) Unpack() ([]common.Hash, []uint64) { - var ( - hashes = make([]common.Hash, len(*p)) - numbers = make([]uint64, len(*p)) - ) - for i, body := range *p { - hashes[i], numbers[i] = body.Hash, body.Number - } - return hashes, numbers -} - // TransactionsPacket is the network packet for broadcasting new transactions. type TransactionsPacket struct { rlp.RawList[*types.Transaction] @@ -203,12 +171,6 @@ type BlockHeadersRLPPacket struct { BlockHeadersRLPResponse } -// NewBlockPacket is the network packet for the block propagation message. -type NewBlockPacket struct { - Block *types.Block - TD *big.Int -} - // GetBlockBodiesRequest represents a block body query. type GetBlockBodiesRequest []common.Hash @@ -258,30 +220,16 @@ type GetReceiptsPacket struct { // ReceiptsResponse is the network packet for block receipts distribution. type ReceiptsResponse []types.Receipts -// ReceiptsList is a type constraint for block receceipt list types. -type ReceiptsList interface { - *ReceiptList68 | *ReceiptList69 - setBuffers(*receiptListBuffers) - EncodeForStorage() (rlp.RawValue, error) - Derivable() types.DerivableList -} - // ReceiptsPacket is the network packet for block receipts distribution with // request ID wrapping. -type ReceiptsPacket[L ReceiptsList] struct { +type ReceiptsPacket struct { RequestId uint64 - List rlp.RawList[L] + List rlp.RawList[*ReceiptList] } // ReceiptsRLPResponse is used for receipts, when we already have it encoded type ReceiptsRLPResponse []rlp.RawValue -// ReceiptsRLPPacket is ReceiptsRLPResponse with request ID wrapping. -type ReceiptsRLPPacket struct { - RequestId uint64 - ReceiptsRLPResponse -} - // NewPooledTransactionHashesPacket represents a transaction announcement packet on eth/68 and newer. type NewPooledTransactionHashesPacket struct { Types []byte @@ -325,14 +273,8 @@ type BlockRangeUpdatePacket struct { LatestBlockHash common.Hash } -func (*StatusPacket68) Name() string { return "Status" } -func (*StatusPacket68) Kind() byte { return StatusMsg } - -func (*StatusPacket69) Name() string { return "Status" } -func (*StatusPacket69) Kind() byte { return StatusMsg } - -func (*NewBlockHashesPacket) Name() string { return "NewBlockHashes" } -func (*NewBlockHashesPacket) Kind() byte { return NewBlockHashesMsg } +func (*StatusPacket) Name() string { return "Status" } +func (*StatusPacket) Kind() byte { return StatusMsg } func (*TransactionsPacket) Name() string { return "Transactions" } func (*TransactionsPacket) Kind() byte { return TransactionsMsg } @@ -349,9 +291,6 @@ func (*GetBlockBodiesRequest) Kind() byte { return GetBlockBodiesMsg } func (*BlockBodiesResponse) Name() string { return "BlockBodies" } func (*BlockBodiesResponse) Kind() byte { return BlockBodiesMsg } -func (*NewBlockPacket) Name() string { return "NewBlock" } -func (*NewBlockPacket) Kind() byte { return NewBlockMsg } - func (*NewPooledTransactionHashesPacket) Name() string { return "NewPooledTransactionHashes" } func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransactionHashesMsg } diff --git a/eth/protocols/eth/protocol_test.go b/eth/protocols/eth/protocol_test.go index e37d72dcd6..f93d01123b 100644 --- a/eth/protocols/eth/protocol_test.go +++ b/eth/protocols/eth/protocol_test.go @@ -95,8 +95,7 @@ func TestEmptyMessages(t *testing.T) { BlockBodiesRLPPacket{1111, BlockBodiesRLPResponse([]rlp.RawValue{})}, // Receipts GetReceiptsPacket{1111, GetReceiptsRequest([]common.Hash{})}, - ReceiptsPacket[*ReceiptList68]{1111, encodeRL([]*ReceiptList68{})}, - ReceiptsPacket[*ReceiptList69]{1111, encodeRL([]*ReceiptList69{})}, + ReceiptsPacket{1111, encodeRL([]*ReceiptList{})}, // Transactions GetPooledTransactionsPacket{1111, GetPooledTransactionsRequest([]common.Hash{})}, PooledTransactionsPacket{1111, encodeRL([]*types.Transaction{})}, @@ -122,7 +121,6 @@ func TestMessages(t *testing.T) { txRlps []rlp.RawValue hashes []common.Hash receipts []*types.Receipt - receiptsRlp rlp.RawValue err error ) @@ -191,11 +189,6 @@ func TestMessages(t *testing.T) { } miniDeriveFields(receipts[0], 0) miniDeriveFields(receipts[1], 1) - rlpData, err := rlp.EncodeToBytes(receipts) - if err != nil { - t.Fatal(err) - } - receiptsRlp = rlpData } for i, tc := range []struct { @@ -231,16 +224,7 @@ func TestMessages(t *testing.T) { common.FromHex("f847820457f842a000000000000000000000000000000000000000000000000000000000deadc0dea000000000000000000000000000000000000000000000000000000000feedbeef"), }, { - ReceiptsPacket[*ReceiptList68]{1111, encodeRL([]*ReceiptList68{NewReceiptList68(receipts)})}, - common.FromHex("f902e6820457f902e0f902ddf901688082014db9010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000014000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000004000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ffb9016f01f9016b018201bcb9010000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000001000000000000000000000000000000000000000000000000040000000000000000000000000004000000000000000000000000000000000000000000000000000000008000400000000000000000000000000000000000000000000000000000000000000000000000000000040f862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"), - }, - { - // Identical to the eth/68 encoding above. - ReceiptsRLPPacket{1111, ReceiptsRLPResponse([]rlp.RawValue{receiptsRlp})}, - common.FromHex("f902e6820457f902e0f902ddf901688082014db9010000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000014000000000000000000000000000000000000000000000000000000000000000000000000000010000000000000000000000000004000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000f85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100ffb9016f01f9016b018201bcb9010000000000000000000000000000000000000000000000000000000000002000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000040000000001000000000000000000000000000000000000000000000000040000000000000000000000000004000000000000000000000000000000000000000000000000000000008000400000000000000000000000000000000000000000000000000000000000000000000000000000040f862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"), - }, - { - ReceiptsPacket[*ReceiptList69]{1111, encodeRL([]*ReceiptList69{NewReceiptList69(receipts)})}, + ReceiptsPacket{1111, encodeRL([]*ReceiptList{NewReceiptList(receipts)})}, common.FromHex("f8da820457f8d5f8d3f866808082014df85ff85d940000000000000000000000000000000000000011f842a0000000000000000000000000000000000000000000000000000000000000deada0000000000000000000000000000000000000000000000000000000000000beef830100fff86901018201bcf862f860940000000000000000000000000000000000000022f842a00000000000000000000000000000000000000000000000000000000000005668a0000000000000000000000000000000000000000000000000000000000000977386020f0f0f0608"), }, { diff --git a/eth/protocols/eth/receipt.go b/eth/protocols/eth/receipt.go index 06956df9f2..96e8c4399b 100644 --- a/eth/protocols/eth/receipt.go +++ b/eth/protocols/eth/receipt.go @@ -27,9 +27,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// This is just a sanity limit for the size of a single receipt. -const maxReceiptSize = 16 * 1024 * 1024 - // Receipt is the representation of receipts for networking purposes. type Receipt struct { TxType byte @@ -49,154 +46,18 @@ func newReceipt(tr *types.Receipt) Receipt { return r } -// decode68 parses a receipt in the eth/68 network encoding. -func (r *Receipt) decode68(b []byte) error { - k, content, _, err := rlp.Split(b) - if err != nil { - return err - } - - *r = Receipt{} - if k == rlp.List { - // Legacy receipt. - return r.decodeInnerList(b, false, true) - } - // Typed receipt. - if len(content) < 2 || len(content) > maxReceiptSize { - return fmt.Errorf("invalid receipt size %d", len(content)) - } - r.TxType = content[0] - return r.decodeInnerList(content[1:], false, true) -} - -// decode69 parses a receipt in the eth/69 network encoding. -func (r *Receipt) decode69(b []byte) error { - *r = Receipt{} - return r.decodeInnerList(b, true, false) -} - -// decodeDatabase parses a receipt in the basic database encoding. -func (r *Receipt) decodeDatabase(txType byte, b []byte) error { - *r = Receipt{TxType: txType} - return r.decodeInnerList(b, false, false) -} - -func (r *Receipt) decodeInnerList(input []byte, readTxType, readBloom bool) error { - input, _, err := rlp.SplitList(input) - if err != nil { - return fmt.Errorf("inner list: %v", err) - } - - // txType - if readTxType { - var txType uint64 - txType, input, err = rlp.SplitUint64(input) - if err != nil { - return fmt.Errorf("invalid txType: %w", err) - } - if txType > 0x7f { - return fmt.Errorf("invalid txType: too large") - } - r.TxType = byte(txType) - } - - // status - r.PostStateOrStatus, input, err = rlp.SplitString(input) - if err != nil { - return fmt.Errorf("invalid postStateOrStatus: %w", err) - } - if len(r.PostStateOrStatus) > 1 && len(r.PostStateOrStatus) != 32 { - return fmt.Errorf("invalid postStateOrStatus length %d", len(r.PostStateOrStatus)) - } - - // gas - r.GasUsed, input, err = rlp.SplitUint64(input) - if err != nil { - return fmt.Errorf("invalid gasUsed: %w", err) - } - - // bloom - if readBloom { - var bloomBytes []byte - bloomBytes, input, err = rlp.SplitString(input) - if err != nil { - return fmt.Errorf("invalid bloom: %v", err) - } - if len(bloomBytes) != types.BloomByteLength { - return fmt.Errorf("invalid bloom length %d", len(bloomBytes)) - } - } - - // logs - _, rest, err := rlp.SplitList(input) - if err != nil { - return fmt.Errorf("invalid logs: %w", err) - } - if len(rest) != 0 { - return fmt.Errorf("junk at end of receipt") - } - r.Logs = input - return nil -} - -// encodeForStorage produces the the storage encoding, i.e. the result matches -// the RLP encoding of types.ReceiptForStorage. -func (r *Receipt) encodeForStorage(w *rlp.EncoderBuffer) { - list := w.List() - w.WriteBytes(r.PostStateOrStatus) - w.WriteUint64(r.GasUsed) - w.Write(r.Logs) - w.ListEnd(list) -} - -// encodeForNetwork68 produces the eth/68 network protocol encoding of a receipt. -// Note this recomputes the bloom filter of the receipt. -func (r *Receipt) encodeForNetwork68(buf *receiptListBuffers, w *rlp.EncoderBuffer) { - writeInner := func(w *rlp.EncoderBuffer) { - list := w.List() - w.WriteBytes(r.PostStateOrStatus) - w.WriteUint64(r.GasUsed) - bloom := r.bloom(&buf.bloom) - w.WriteBytes(bloom[:]) - w.Write(r.Logs) - w.ListEnd(list) - } - - if r.TxType == 0 { - writeInner(w) - } else { - buf.tmp.Reset() - buf.tmp.WriteByte(r.TxType) - buf.enc.Reset(&buf.tmp) - writeInner(&buf.enc) - buf.enc.Flush() - w.WriteBytes(buf.tmp.Bytes()) - } -} - -// encodeForNetwork69 produces the eth/69 network protocol encoding of a receipt. -func (r *Receipt) encodeForNetwork69(w *rlp.EncoderBuffer) { - list := w.List() - w.WriteUint64(uint64(r.TxType)) - w.WriteBytes(r.PostStateOrStatus) - w.WriteUint64(r.GasUsed) - w.Write(r.Logs) - w.ListEnd(list) -} - // encodeForHash encodes a receipt for the block receiptsRoot derivation. -func (r *Receipt) encodeForHash(buf *receiptListBuffers, out *bytes.Buffer) { +func (r *Receipt) encodeForHash(bloomBuf *[6]byte, out *bytes.Buffer) { // For typed receipts, add the tx type. if r.TxType != 0 { out.WriteByte(r.TxType) } // Encode list = [postStateOrStatus, gasUsed, bloom, logs]. - w := &buf.enc - w.Reset(out) + w := rlp.NewEncoderBuffer(out) l := w.List() w.WriteBytes(r.PostStateOrStatus) w.WriteUint64(r.GasUsed) - bloom := r.bloom(&buf.bloom) + bloom := r.bloom(bloomBuf) w.WriteBytes(bloom[:]) w.Write(r.Logs) w.ListEnd(l) @@ -229,31 +90,98 @@ func (r *Receipt) bloom(buffer *[6]byte) types.Bloom { return b } -type receiptListBuffers struct { - enc rlp.EncoderBuffer - bloom [6]byte - tmp bytes.Buffer -} - -func initBuffers(buf **receiptListBuffers) { - if *buf == nil { - *buf = new(receiptListBuffers) +// decode assigns the fields of r by decoding the network format. +func (r *Receipt) decode(input []byte) error { + input, _, err := rlp.SplitList(input) + if err != nil { + return fmt.Errorf("inner list: %v", err) } + + // txType + var txType uint64 + txType, input, err = rlp.SplitUint64(input) + if err != nil { + return fmt.Errorf("invalid txType: %w", err) + } + if txType > 0x7f { + return fmt.Errorf("invalid txType: too large") + } + r.TxType = byte(txType) + + // status + r.PostStateOrStatus, input, err = rlp.SplitString(input) + if err != nil { + return fmt.Errorf("invalid postStateOrStatus: %w", err) + } + if len(r.PostStateOrStatus) > 1 && len(r.PostStateOrStatus) != 32 { + return fmt.Errorf("invalid postStateOrStatus length %d", len(r.PostStateOrStatus)) + } + + // gas + r.GasUsed, input, err = rlp.SplitUint64(input) + if err != nil { + return fmt.Errorf("invalid gasUsed: %w", err) + } + + // logs + _, rest, err := rlp.SplitList(input) + if err != nil { + return fmt.Errorf("invalid logs: %w", err) + } + if len(rest) != 0 { + return fmt.Errorf("junk at end of receipt") + } + r.Logs = input + return nil } -// encodeForStorage encodes a list of receipts for the database. -func (buf *receiptListBuffers) encodeForStorage(rs rlp.RawList[Receipt], decode func([]byte, *Receipt) error) (rlp.RawValue, error) { +// ReceiptList is the block receipt list as downloaded by eth/69. +type ReceiptList struct { + items rlp.RawList[Receipt] +} + +// NewReceiptList creates a receipt list. +// This is slow, and exists for testing purposes. +func NewReceiptList(trs []*types.Receipt) *ReceiptList { + rl := new(ReceiptList) + for _, tr := range trs { + r := newReceipt(tr) + encoded, _ := rlp.EncodeToBytes(&r) + rl.items.AppendRaw(encoded) + } + return rl +} + +// DecodeRLP decodes a list receipts from the network format. +func (rl *ReceiptList) DecodeRLP(s *rlp.Stream) error { + return rl.items.DecodeRLP(s) +} + +// EncodeRLP encodes the list into the network format of eth/69. +func (rl *ReceiptList) EncodeRLP(w io.Writer) error { + return rl.items.EncodeRLP(w) +} + +// EncodeForStorage encodes a list of receipts for the database. +// It only strips the first element (TxType) from each receipt's +// raw RLP without the actual decoding and re-encoding. +func (rl *ReceiptList) EncodeForStorage() (rlp.RawValue, error) { var out bytes.Buffer - w := &buf.enc - w.Reset(&out) + w := rlp.NewEncoderBuffer(&out) outer := w.List() - it := rs.ContentIterator() + it := rl.items.ContentIterator() for it.Next() { - var receipt Receipt - if err := decode(it.Value(), &receipt); err != nil { - return nil, err + content, _, err := rlp.SplitList(it.Value()) + if err != nil { + return nil, fmt.Errorf("bad receipt: %v", err) } - receipt.encodeForStorage(w) + _, _, rest, err := rlp.Split(content) + if err != nil { + return nil, fmt.Errorf("bad receipt: %v", err) + } + inner := w.List() + w.Write(rest) + w.ListEnd(inner) } if it.Err() != nil { return nil, fmt.Errorf("bad list: %v", it.Err()) @@ -263,149 +191,20 @@ func (buf *receiptListBuffers) encodeForStorage(rs rlp.RawList[Receipt], decode return out.Bytes(), nil } -// ReceiptList68 is a block receipt list as downloaded by eth/68. -// This also implements types.DerivableList for validation purposes. -type ReceiptList68 struct { - buf *receiptListBuffers - items rlp.RawList[Receipt] -} - -// NewReceiptList68 creates a receipt list. -// This is slow, and exists for testing purposes. -func NewReceiptList68(trs []*types.Receipt) *ReceiptList68 { - rl := new(ReceiptList68) - initBuffers(&rl.buf) - enc := rlp.NewEncoderBuffer(nil) - for _, tr := range trs { - r := newReceipt(tr) - r.encodeForNetwork68(rl.buf, &enc) - rl.items.AppendRaw(enc.ToBytes()) - enc.Reset(nil) - } - return rl -} - -func blockReceiptsToNetwork68(blockReceipts, blockBody rlp.RawValue) ([]byte, error) { - txTypesIter, err := txTypesInBody(blockBody) - if err != nil { - return nil, fmt.Errorf("invalid block body: %v", err) - } - nextTxType, stopTxTypes := iter.Pull(txTypesIter) - defer stopTxTypes() - - var ( - out bytes.Buffer - buf receiptListBuffers - ) - blockReceiptIter, _ := rlp.NewListIterator(blockReceipts) - w := rlp.NewEncoderBuffer(&out) - outer := w.List() - for i := 0; blockReceiptIter.Next(); i++ { - txType, _ := nextTxType() - var r Receipt - if err := r.decodeDatabase(txType, blockReceiptIter.Value()); err != nil { - return nil, fmt.Errorf("invalid database receipt %d: %v", i, err) - } - r.encodeForNetwork68(&buf, &w) - } - w.ListEnd(outer) - w.Flush() - return out.Bytes(), nil -} - -// setBuffers implements ReceiptsList. -func (rl *ReceiptList68) setBuffers(buf *receiptListBuffers) { - rl.buf = buf -} - -// EncodeForStorage encodes the receipts for storage into the database. -func (rl *ReceiptList68) EncodeForStorage() (rlp.RawValue, error) { - initBuffers(&rl.buf) - return rl.buf.encodeForStorage(rl.items, func(data []byte, r *Receipt) error { - return r.decode68(data) - }) -} - -// Derivable turns the receipts into a list that can derive the root hash. -func (rl *ReceiptList68) Derivable() types.DerivableList { - initBuffers(&rl.buf) +// Derivable returns a DerivableList, which can be used to decode +func (rl *ReceiptList) Derivable() types.DerivableList { + var bloomBuf [6]byte return newDerivableRawList(&rl.items, func(data []byte, outbuf *bytes.Buffer) { var r Receipt - if r.decode68(data) == nil { - r.encodeForHash(rl.buf, outbuf) + if r.decode(data) == nil { + r.encodeForHash(&bloomBuf, outbuf) } }) } -// DecodeRLP decodes a list of receipts from the network format. -func (rl *ReceiptList68) DecodeRLP(s *rlp.Stream) error { - return rl.items.DecodeRLP(s) -} - -// EncodeRLP encodes the list into the network format of eth/68. -func (rl *ReceiptList68) EncodeRLP(w io.Writer) error { - return rl.items.EncodeRLP(w) -} - -// ReceiptList69 is the block receipt list as downloaded by eth/69. -// This implements types.DerivableList for validation purposes. -type ReceiptList69 struct { - buf *receiptListBuffers - items rlp.RawList[Receipt] -} - -// NewReceiptList69 creates a receipt list. -// This is slow, and exists for testing purposes. -func NewReceiptList69(trs []*types.Receipt) *ReceiptList69 { - rl := new(ReceiptList69) - enc := rlp.NewEncoderBuffer(nil) - for _, tr := range trs { - r := newReceipt(tr) - r.encodeForNetwork69(&enc) - rl.items.AppendRaw(enc.ToBytes()) - enc.Reset(nil) - } - return rl -} - -// setBuffers implements ReceiptsList. -func (rl *ReceiptList69) setBuffers(buf *receiptListBuffers) { - rl.buf = buf -} - -// EncodeForStorage encodes the receipts for storage into the database. -func (rl *ReceiptList69) EncodeForStorage() (rlp.RawValue, error) { - initBuffers(&rl.buf) - return rl.buf.encodeForStorage(rl.items, func(data []byte, r *Receipt) error { - return r.decode69(data) - }) -} - -// Derivable turns the receipts into a list that can derive the root hash. -func (rl *ReceiptList69) Derivable() types.DerivableList { - initBuffers(&rl.buf) - return newDerivableRawList(&rl.items, func(data []byte, outbuf *bytes.Buffer) { - var r Receipt - if r.decode69(data) == nil { - r.encodeForHash(rl.buf, outbuf) - } - }) -} - -// DecodeRLP decodes a list receipts from the network format. -func (rl *ReceiptList69) DecodeRLP(s *rlp.Stream) error { - return rl.items.DecodeRLP(s) -} - -// EncodeRLP encodes the list into the network format of eth/69. -func (rl *ReceiptList69) EncodeRLP(w io.Writer) error { - return rl.items.EncodeRLP(w) -} - -// blockReceiptsToNetwork69 takes a slice of rlp-encoded receipts, and transactions, -// and applies the type-encoding on the receipts (for non-legacy receipts). -// e.g. for non-legacy receipts: receipt-data -> {tx-type || receipt-data} -func blockReceiptsToNetwork69(blockReceipts, blockBody rlp.RawValue) ([]byte, error) { +// blockReceiptsToNetwork takes a slice of rlp-encoded receipts, and transactions, +// and re-encodes them for the network protocol. +func blockReceiptsToNetwork(blockReceipts, blockBody rlp.RawValue) ([]byte, error) { txTypesIter, err := txTypesInBody(blockBody) if err != nil { return nil, fmt.Errorf("invalid block body: %v", err) diff --git a/eth/protocols/eth/receipt_test.go b/eth/protocols/eth/receipt_test.go index 39a2728f7f..693ccd6918 100644 --- a/eth/protocols/eth/receipt_test.go +++ b/eth/protocols/eth/receipt_test.go @@ -95,7 +95,7 @@ func init() { } } -func TestReceiptList69(t *testing.T) { +func TestReceiptList(t *testing.T) { for i, test := range receiptsTests { // encode receipts from types.ReceiptForStorage object. canonDB, _ := rlp.EncodeToBytes(test.input) @@ -105,13 +105,13 @@ func TestReceiptList69(t *testing.T) { canonBody, _ := rlp.EncodeToBytes(blockBody) // convert from storage encoding to network encoding - network, err := blockReceiptsToNetwork69(canonDB, canonBody) + network, err := blockReceiptsToNetwork(canonDB, canonBody) if err != nil { - t.Fatalf("test[%d]: blockReceiptsToNetwork69 error: %v", i, err) + t.Fatalf("test[%d]: blockReceiptsToNetwork error: %v", i, err) } // parse as Receipts response list from network encoding - var rl ReceiptList69 + var rl ReceiptList if err := rlp.DecodeBytes(network, &rl); err != nil { t.Fatalf("test[%d]: can't decode network receipts: %v", i, err) } @@ -127,50 +127,10 @@ func TestReceiptList69(t *testing.T) { t.Fatalf("test[%d]: re-encoded network receipt list not equal\nhave: %x\nwant: %x", i, rlNetworkEnc, network) } - // compute root hash from ReceiptList69 and compare. + // compute root hash from ReceiptList and compare. responseHash := types.DeriveSha(rl.Derivable(), trie.NewStackTrie(nil)) if responseHash != test.root { - t.Fatalf("test[%d]: wrong root hash from ReceiptList69\nhave: %v\nwant: %v", i, responseHash, test.root) - } - } -} - -func TestReceiptList68(t *testing.T) { - for i, test := range receiptsTests { - // encode receipts from types.ReceiptForStorage object. - canonDB, _ := rlp.EncodeToBytes(test.input) - - // encode block body from types object. - blockBody := types.Body{Transactions: test.txs} - canonBody, _ := rlp.EncodeToBytes(blockBody) - - // convert from storage encoding to network encoding - network, err := blockReceiptsToNetwork68(canonDB, canonBody) - if err != nil { - t.Fatalf("test[%d]: blockReceiptsToNetwork68 error: %v", i, err) - } - - // parse as Receipts response list from network encoding - var rl ReceiptList68 - if err := rlp.DecodeBytes(network, &rl); err != nil { - t.Fatalf("test[%d]: can't decode network receipts: %v", i, err) - } - rlStorageEnc, err := rl.EncodeForStorage() - if err != nil { - t.Fatalf("test[%d]: error from EncodeForStorage: %v", i, err) - } - if !bytes.Equal(rlStorageEnc, canonDB) { - t.Fatalf("test[%d]: re-encoded receipts not equal\nhave: %x\nwant: %x", i, rlStorageEnc, canonDB) - } - rlNetworkEnc, _ := rlp.EncodeToBytes(&rl) - if !bytes.Equal(rlNetworkEnc, network) { - t.Fatalf("test[%d]: re-encoded network receipt list not equal\nhave: %x\nwant: %x", i, rlNetworkEnc, network) - } - - // compute root hash from ReceiptList68 and compare. - responseHash := types.DeriveSha(rl.Derivable(), trie.NewStackTrie(nil)) - if responseHash != test.root { - t.Fatalf("test[%d]: wrong root hash from ReceiptList68\nhave: %v\nwant: %v", i, responseHash, test.root) + t.Fatalf("test[%d]: wrong root hash from ReceiptList\nhave: %v\nwant: %v", i, responseHash, test.root) } } } diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 08e85c896a..841bfb446e 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -1699,9 +1699,13 @@ func (s *Syncer) revertAccountRequest(req *accountRequest) { } close(req.stale) - // Remove the request from the tracked set + // Remove the request from the tracked set and restore the peer to the + // idle pool so it can be reassigned work (skip if peer already left). s.lock.Lock() delete(s.accountReqs, req.id) + if _, ok := s.peers[req.peer]; ok { + s.accountIdlers[req.peer] = struct{}{} + } s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the account @@ -1740,9 +1744,13 @@ func (s *Syncer) revertBytecodeRequest(req *bytecodeRequest) { } close(req.stale) - // Remove the request from the tracked set + // Remove the request from the tracked set and restore the peer to the + // idle pool so it can be reassigned work (skip if peer already left). s.lock.Lock() delete(s.bytecodeReqs, req.id) + if _, ok := s.peers[req.peer]; ok { + s.bytecodeIdlers[req.peer] = struct{}{} + } s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code @@ -1781,9 +1789,13 @@ func (s *Syncer) revertStorageRequest(req *storageRequest) { } close(req.stale) - // Remove the request from the tracked set + // Remove the request from the tracked set and restore the peer to the + // idle pool so it can be reassigned work (skip if peer already left). s.lock.Lock() delete(s.storageReqs, req.id) + if _, ok := s.peers[req.peer]; ok { + s.storageIdlers[req.peer] = struct{}{} + } s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the storage @@ -1826,9 +1838,13 @@ func (s *Syncer) revertTrienodeHealRequest(req *trienodeHealRequest) { } close(req.stale) - // Remove the request from the tracked set + // Remove the request from the tracked set and restore the peer to the + // idle pool so it can be reassigned work (skip if peer already left). s.lock.Lock() delete(s.trienodeHealReqs, req.id) + if _, ok := s.peers[req.peer]; ok { + s.trienodeHealIdlers[req.peer] = struct{}{} + } s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the trie node @@ -1867,9 +1883,13 @@ func (s *Syncer) revertBytecodeHealRequest(req *bytecodeHealRequest) { } close(req.stale) - // Remove the request from the tracked set + // Remove the request from the tracked set and restore the peer to the + // idle pool so it can be reassigned work (skip if peer already left). s.lock.Lock() delete(s.bytecodeHealReqs, req.id) + if _, ok := s.peers[req.peer]; ok { + s.bytecodeHealIdlers[req.peer] = struct{}{} + } s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 79c91043a3..871f2c9269 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -147,7 +147,7 @@ func (eth *Ethereum) hashState(ctx context.Context, block *types.Block, reexec u if current = eth.blockchain.GetBlockByNumber(next); current == nil { return nil, nil, fmt.Errorf("block #%d not found", next) } - _, err := eth.blockchain.Processor().Process(current, statedb, vm.Config{}) + _, err := eth.blockchain.Processor().Process(ctx, current, statedb, vm.Config{}) if err != nil { return nil, nil, fmt.Errorf("processing block %d failed: %v", current.NumberU64(), err) } @@ -265,7 +265,7 @@ func (eth *Ethereum) stateAtTransaction(ctx context.Context, block *types.Block, // Not yet the searched for transaction, execute on top of the current state statedb.SetTxContext(tx.Hash(), idx) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } // Ensure any modifications are committed to the state diff --git a/eth/sync_test.go b/eth/sync_test.go index 509b836f82..77a50bf6d3 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,7 +28,7 @@ import ( ) // Tests that snap sync is disabled after a successful sync cycle. -func TestSnapSyncDisabling68(t *testing.T) { testSnapSyncDisabling(t, eth.ETH68, snap.SNAP1) } +func TestSnapSyncDisabling69(t *testing.T) { testSnapSyncDisabling(t, eth.ETH69, snap.SNAP1) } // Tests that snap sync gets disabled as soon as a real block is successfully // imported into the blockchain. @@ -82,15 +82,18 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) { if err := empty.handler.downloader.BeaconSync(full.chain.CurrentBlock(), nil); err != nil { t.Fatal("sync failed:", err) } - // Downloader internally has to wait for a timer (3s) to be expired before - // exiting. Poll after to determine if sync is disabled. - time.Sleep(time.Second * 3) - for timeout := time.After(time.Second); ; { + // Snap sync and mode switching happen asynchronously, poll for completion. + timeout := time.NewTimer(15 * time.Second) + tick := time.NewTicker(100 * time.Millisecond) + defer timeout.Stop() + defer tick.Stop() + + for { select { - case <-timeout: + case <-timeout.C: t.Fatalf("snap sync not disabled after successful synchronisation") - case <-time.After(100 * time.Millisecond): - if empty.handler.downloader.ConfigSyncMode() == ethconfig.FullSync { + case <-tick.C: + if empty.handler.synced.Load() && empty.handler.downloader.ConfigSyncMode() == ethconfig.FullSync { return } } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 5f2f16627a..eed404622e 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -551,7 +551,7 @@ func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config } msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) statedb.SetTxContext(tx.Hash(), i) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) // We intentionally don't return the error here: if we do, then the RPC server will not // return the roots. Most likely, the caller already knows that a certain transaction fails to @@ -707,7 +707,7 @@ txloop: // Generate the next state snapshot fast without tracing msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) statedb.SetTxContext(tx.Hash(), i) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { failed = err break txloop } @@ -792,7 +792,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) if txHash != (common.Hash{}) && tx.Hash() != txHash { // Process the tx to update state, but don't trace it. - _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) + _, err := core.ApplyMessage(evm, msg, nil) if err != nil { return dumps, err } @@ -827,7 +827,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block if tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } - _, err = core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) + _, err = core.ApplyMessage(evm, msg, nil) if writer != nil { writer.Flush() } @@ -1011,7 +1011,6 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor tracer *Tracer err error timeout = defaultTraceTimeout - usedGas uint64 ) if config == nil { config = &TraceConfig{} @@ -1055,7 +1054,8 @@ func (api *API) traceTx(ctx context.Context, tx *types.Transaction, message *cor // Call Prepare to clear out the statedb access list statedb.SetTxContext(txctx.TxHash, txctx.TxIndex) - _, err = core.ApplyTransactionWithEVM(message, new(core.GasPool).AddGas(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, vmctx.Time, tx, &usedGas, evm) + + _, err = core.ApplyTransactionWithEVM(message, core.NewGasPool(message.GasLimit), statedb, vmctx.BlockNumber, txctx.BlockHash, vmctx.Time, tx, evm) if err != nil { return nil, fmt.Errorf("tracing failed: %w", err) } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index f76c35a1d5..1d5024ad08 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -188,7 +188,7 @@ func (b *testBackend) StateAtTransaction(ctx context.Context, block *types.Block return tx, context, statedb, release, nil } msg, _ := core.TransactionToMessage(tx, signer, block.BaseFee()) - if _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())); err != nil { + if _, err := core.ApplyMessage(evm, msg, nil); err != nil { return nil, vm.BlockContext{}, nil, nil, fmt.Errorf("transaction %#x failed: %v", tx.Hash(), err) } statedb.Finalise(evm.ChainConfig().IsEIP158(block.Number())) diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index 08bdafd91f..85eaef32ce 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -132,7 +132,7 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { } evm := vm.NewEVM(context, logState, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } @@ -224,7 +224,7 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { if tracer.OnTxStart != nil { tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) } - _, err = core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + _, err = core.ApplyMessage(evm, msg, nil) if err != nil { b.Fatalf("failed to execute transaction: %v", err) } @@ -374,7 +374,7 @@ func TestInternals(t *testing.T) { t.Fatalf("test %v: failed to create message: %v", tc.name, err) } tc.tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("test %v: failed to execute transaction: %v", tc.name, err) } diff --git a/eth/tracers/internal/tracetest/erc7562_tracer_test.go b/eth/tracers/internal/tracetest/erc7562_tracer_test.go index f6e81f5886..02377b8dcb 100644 --- a/eth/tracers/internal/tracetest/erc7562_tracer_test.go +++ b/eth/tracers/internal/tracetest/erc7562_tracer_test.go @@ -124,7 +124,7 @@ func TestErc7562Tracer(t *testing.T) { } evm := vm.NewEVM(context, logState, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } diff --git a/eth/tracers/internal/tracetest/flat_calltrace_test.go b/eth/tracers/internal/tracetest/flat_calltrace_test.go index 1882ef315e..37a05966ee 100644 --- a/eth/tracers/internal/tracetest/flat_calltrace_test.go +++ b/eth/tracers/internal/tracetest/flat_calltrace_test.go @@ -113,7 +113,7 @@ func flatCallTracerTestRunner(tracerName string, filename string, dirPath string } evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { return fmt.Errorf("failed to execute transaction: %v", err) } diff --git a/eth/tracers/internal/tracetest/prestate_test.go b/eth/tracers/internal/tracetest/prestate_test.go index 456d962c69..23216fa78c 100644 --- a/eth/tracers/internal/tracetest/prestate_test.go +++ b/eth/tracers/internal/tracetest/prestate_test.go @@ -105,7 +105,7 @@ func testPrestateTracer(tracerName string, dirPath string, t *testing.T) { } evm := vm.NewEVM(context, state.StateDB, test.Genesis.Config, vm.Config{Tracer: tracer.Hooks}) tracer.OnTxStart(evm.GetVMContext(), tx, msg.From) - vmRet, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + vmRet, err := core.ApplyMessage(evm, msg, nil) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } diff --git a/eth/tracers/internal/tracetest/selfdestruct_state_test.go b/eth/tracers/internal/tracetest/selfdestruct_state_test.go index 2c714b6dce..bb1a3d9f18 100644 --- a/eth/tracers/internal/tracetest/selfdestruct_state_test.go +++ b/eth/tracers/internal/tracetest/selfdestruct_state_test.go @@ -620,8 +620,7 @@ func TestSelfdestructStateTracer(t *testing.T) { } context := core.NewEVMBlockContext(block.Header(), blockchain, nil) evm := vm.NewEVM(context, hookedState, tt.genesis.Config, vm.Config{Tracer: tracer.Hooks()}) - usedGas := uint64(0) - _, err = core.ApplyTransactionWithEVM(msg, new(core.GasPool).AddGas(tx.Gas()), statedb, block.Number(), block.Hash(), block.Time(), tx, &usedGas, evm) + _, err = core.ApplyTransactionWithEVM(msg, core.NewGasPool(msg.GasLimit), statedb, block.Number(), block.Hash(), block.Time(), tx, evm) if err != nil { t.Fatalf("failed to execute transaction: %v", err) } diff --git a/eth/tracers/logger/access_list_tracer.go b/eth/tracers/logger/access_list_tracer.go index 0d51f40522..749aade61b 100644 --- a/eth/tracers/logger/access_list_tracer.go +++ b/eth/tracers/logger/access_list_tracer.go @@ -18,6 +18,7 @@ package logger import ( "maps" + "slices" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/tracing" @@ -84,12 +85,17 @@ func (al accessList) equal(other accessList) bool { func (al accessList) accessList() types.AccessList { acl := make(types.AccessList, 0, len(al)) for addr, slots := range al { - tuple := types.AccessTuple{Address: addr, StorageKeys: []common.Hash{}} - for slot := range slots { - tuple.StorageKeys = append(tuple.StorageKeys, slot) + keys := slices.SortedFunc(maps.Keys(slots), common.Hash.Cmp) + // Ensure keys is never nil to avoid JSON serialization issues. + // When slots is empty, slices.SortedFunc returns nil, but JSON marshaling + // will serialize nil slice as null instead of [], which breaks clients + // that expect storageKeys to always be an array. + if keys == nil { + keys = []common.Hash{} } - acl = append(acl, tuple) + acl = append(acl, types.AccessTuple{Address: addr, StorageKeys: keys}) } + slices.SortFunc(acl, func(a, b types.AccessTuple) int { return a.Address.Cmp(b.Address) }) return acl } diff --git a/eth/tracers/native/mux.go b/eth/tracers/native/mux.go index 37fc64f3f5..b7d6f29a6a 100644 --- a/eth/tracers/native/mux.go +++ b/eth/tracers/native/mux.go @@ -28,7 +28,7 @@ import ( ) func init() { - tracers.DefaultDirectory.Register("muxTracer", newMuxTracer, false) + tracers.DefaultDirectory.Register("muxTracer", newMuxTracerFromConfig, false) } // muxTracer is a go implementation of the Tracer interface which @@ -38,8 +38,8 @@ type muxTracer struct { tracers []*tracers.Tracer } -// newMuxTracer returns a new mux tracer. -func newMuxTracer(ctx *tracers.Context, cfg json.RawMessage, chainConfig *params.ChainConfig) (*tracers.Tracer, error) { +// newMuxTracerFromConfig returns a new mux tracer. +func newMuxTracerFromConfig(ctx *tracers.Context, cfg json.RawMessage, chainConfig *params.ChainConfig) (*tracers.Tracer, error) { var config map[string]json.RawMessage if err := json.Unmarshal(cfg, &config); err != nil { return nil, err @@ -54,7 +54,16 @@ func newMuxTracer(ctx *tracers.Context, cfg json.RawMessage, chainConfig *params objects = append(objects, t) names = append(names, k) } + return NewMuxTracer(names, objects) +} +// NewMuxTracer creates a multiplexing tracer that fans out tracing hooks to +// multiple child tracers. Each hook invocation is forwarded to all children, +// in the order they are provided. +// +// The names parameter associates a label with each tracer, used as keys in +// the aggregated JSON result returned by GetResult. +func NewMuxTracer(names []string, objects []*tracers.Tracer) (*tracers.Tracer, error) { t := &muxTracer{names: names, tracers: objects} return &tracers.Tracer{ Hooks: &tracing.Hooks{ diff --git a/eth/tracers/native/opcode_counter.go b/eth/tracers/native/opcode_counter.go new file mode 100644 index 0000000000..d859b275f0 --- /dev/null +++ b/eth/tracers/native/opcode_counter.go @@ -0,0 +1,55 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package native + +import ( + "encoding/json" + + "github.com/ethereum/go-ethereum/core/tracing" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers" +) + +// opcodeCounter is a simple tracer that counts how many times each opcode is executed. +type opcodeCounter struct { + counts [256]uint64 +} + +// NewOpcodeCounter returns a new opcodeCounter tracer. +func NewOpcodeCounter() *tracers.Tracer { + c := &opcodeCounter{} + return &tracers.Tracer{ + Hooks: &tracing.Hooks{ + OnOpcode: func(pc uint64, op byte, gas, cost uint64, scope tracing.OpContext, rData []byte, depth int, err error) { + c.counts[op]++ + }, + }, + GetResult: c.getResult, + Stop: func(err error) {}, + } +} + +// getResult returns the opcode counts keyed by opcode name. +func (c *opcodeCounter) getResult() (json.RawMessage, error) { + out := make(map[string]uint64) + for op, count := range c.counts { + if count != 0 { + out[vm.OpCode(op).String()] = count + } + } + return json.Marshal(out) +} diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 06edeaf698..24f9b3701e 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -91,7 +91,7 @@ func BenchmarkTransactionTraceV2(b *testing.B) { evm.Config.Tracer = tracer snap := state.StateDB.Snapshot() - _, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + _, err := core.ApplyMessage(evm, msg, nil) if err != nil { b.Fatal(err) } diff --git a/ethdb/batch.go b/ethdb/batch.go index 45b3781cb0..b93636c865 100644 --- a/ethdb/batch.go +++ b/ethdb/batch.go @@ -37,6 +37,9 @@ type Batch interface { // Replay replays the batch contents. Replay(w KeyValueWriter) error + + // Close closes the batch and releases all associated resources. + Close() } // Batcher wraps the NewBatch method of a backing data store. diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index b6c93907b1..c235d5f445 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -518,6 +518,9 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { return b.b.Replay(&replayer{writer: w}) } +// Close closes the batch and releases all associated resources. +func (b *batch) Close() {} + // replayer is a small wrapper to implement the correct replay methods. type replayer struct { writer ethdb.KeyValueWriter diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index 200ad60245..29ed0aaea1 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -338,6 +338,9 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { return nil } +// Close closes the batch and releases all associated resources. +func (b *batch) Close() {} + // iterator can walk over the (potentially partial) keyspace of a memory key // value store. Internally it is a deep copy of the entire iterated state, // sorted by keys. diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index 6b549f40d9..7654d582c4 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -731,6 +731,12 @@ func (b *batch) Replay(w ethdb.KeyValueWriter) error { } } +// Close closes the batch and releases all associated resources. After it is +// closed, any subsequent operations on this batch are undefined. +func (b *batch) Close() { + b.b.Close() +} + // pebbleIterator is a wrapper of underlying iterator in storage engine. // The purpose of this structure is to implement the missing APIs. // diff --git a/go.mod b/go.mod index e15d29a6c5..bfe2df8c0c 100644 --- a/go.mod +++ b/go.mod @@ -14,14 +14,14 @@ require ( github.com/cloudflare/cloudflare-go v0.114.0 github.com/cockroachdb/pebble v1.1.5 github.com/consensys/gnark-crypto v0.18.1 - github.com/crate-crypto/go-eth-kzg v1.4.0 + github.com/crate-crypto/go-eth-kzg v1.5.0 github.com/davecgh/go-spew v1.1.1 github.com/dchest/siphash v1.2.3 github.com/deckarep/golang-set/v2 v2.6.0 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0 github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 - github.com/ethereum/c-kzg-4844/v2 v2.1.5 + github.com/ethereum/c-kzg-4844/v2 v2.1.6 github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab github.com/fatih/color v1.16.0 github.com/ferranbt/fastssz v0.1.4 @@ -58,20 +58,20 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.2.0 github.com/stretchr/testify v1.11.1 - github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe + github.com/supranational/blst v0.3.16 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/urfave/cli/v2 v2.27.5 - go.opentelemetry.io/otel v1.39.0 + go.opentelemetry.io/otel v1.40.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 - go.opentelemetry.io/otel/sdk v1.39.0 - go.opentelemetry.io/otel/trace v1.39.0 + go.opentelemetry.io/otel/sdk v1.40.0 + go.opentelemetry.io/otel/trace v1.40.0 go.uber.org/automaxprocs v1.5.2 go.uber.org/goleak v1.3.0 golang.org/x/crypto v0.44.0 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df golang.org/x/sync v0.18.0 - golang.org/x/sys v0.39.0 + golang.org/x/sys v0.40.0 golang.org/x/text v0.31.0 golang.org/x/time v0.9.0 golang.org/x/tools v0.38.0 @@ -86,7 +86,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/otel/metric v1.40.0 // indirect go.opentelemetry.io/proto/otlp v1.9.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect diff --git a/go.sum b/go.sum index fe2a64eaec..b8c0558c8c 100644 --- a/go.sum +++ b/go.sum @@ -81,8 +81,8 @@ github.com/consensys/gnark-crypto v0.18.1 h1:RyLV6UhPRoYYzaFnPQA4qK3DyuDgkTgskDd github.com/consensys/gnark-crypto v0.18.1/go.mod h1:L3mXGFTe1ZN+RSJ+CLjUt9x7PNdx8ubaYfDROyp2Z8c= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-eth-kzg v1.4.0 h1:WzDGjHk4gFg6YzV0rJOAsTK4z3Qkz5jd4RE3DAvPFkg= -github.com/crate-crypto/go-eth-kzg v1.4.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= +github.com/crate-crypto/go-eth-kzg v1.5.0 h1:FYRiJMJG2iv+2Dy3fi14SVGjcPteZ5HAAUe4YWlJygc= +github.com/crate-crypto/go-eth-kzg v1.5.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -113,8 +113,8 @@ github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/emicklei/dot v1.6.2 h1:08GN+DD79cy/tzN6uLCT84+2Wk9u+wvqP+Hkx/dIR8A= github.com/emicklei/dot v1.6.2/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= -github.com/ethereum/c-kzg-4844/v2 v2.1.5 h1:aVtoLK5xwJ6c5RiqO8g8ptJ5KU+2Hdquf6G3aXiHh5s= -github.com/ethereum/c-kzg-4844/v2 v2.1.5/go.mod h1:u59hRTTah4Co6i9fDWtiCjTrblJv0UwsqZKCc0GfgUs= +github.com/ethereum/c-kzg-4844/v2 v2.1.6 h1:xQymkKCT5E2Jiaoqf3v4wsNgjZLY0lRSkZn27fRjSls= +github.com/ethereum/c-kzg-4844/v2 v2.1.6/go.mod h1:8HMkUZ5JRv4hpw/XUrYWSQNAUzhHMg2UDb/U+5m+XNw= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab h1:rvv6MJhy07IMfEKuARQ9TKojGqLVNxQajaXEp/BoqSk= github.com/ethereum/go-bigmodexpfix v0.0.0-20250911101455-f9e208c548ab/go.mod h1:IuLm4IsPipXKF7CW5Lzf68PIbZ5yl7FFd74l/E0o9A8= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -360,8 +360,8 @@ github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe h1:nbdqkIGOGfUAD54q1s2YBcBz/WcsxCO9HUQ4aGV5hUw= -github.com/supranational/blst v0.3.16-0.20250831170142-f48500c1fdbe/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.16 h1:bTDadT+3fK497EvLdWRQEjiGnUtzJ7jjIUMF0jqwYhE= +github.com/supranational/blst v0.3.16/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= @@ -380,20 +380,20 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms= +go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g= +go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc= +go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8= +go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE= +go.opentelemetry.io/otel/sdk/metric v1.40.0 h1:mtmdVqgQkeRxHgRv4qhyJduP3fYJRMX4AtAlbuWdCYw= +go.opentelemetry.io/otel/sdk/metric v1.40.0/go.mod h1:4Z2bGMf0KSK3uRjlczMOeMhKU2rhUqdWNoKcYrtcBPg= +go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw= +go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA= go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME= @@ -477,8 +477,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/graphql/graphql.go b/graphql/graphql.go index f5eec210a5..f25bfd127a 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -1093,6 +1093,18 @@ func (b *Block) ExcessBlobGas(ctx context.Context) (*hexutil.Uint64, error) { return &ret, nil } +func (b *Block) SlotNumber(ctx context.Context) (*hexutil.Uint64, error) { + header, err := b.resolveHeader(ctx) + if err != nil { + return nil, err + } + if header.SlotNumber == nil { + return nil, nil + } + ret := hexutil.Uint64(*header.SlotNumber) + return &ret, nil +} + // BlockFilterCriteria encapsulates criteria passed to a `logs` accessor inside // a block. type BlockFilterCriteria struct { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index a13e3ee186..4de8760db2 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -53,6 +53,10 @@ import ( // allowed to produce in order to speed up calculations. const estimateGasErrorRatio = 0.015 +// maxGetStorageSlots is the maximum total number of storage slots that can +// be requested in a single eth_getStorageValues call. +const maxGetStorageSlots = 1024 + var errBlobTxNotSupported = errors.New("signing blob transactions not supported") var errSubClosed = errors.New("chain subscription closed") @@ -589,6 +593,41 @@ func (api *BlockChainAPI) GetStorageAt(ctx context.Context, address common.Addre return res[:], state.Error() } +// GetStorageValues returns multiple storage slot values for multiple accounts +// at the given block. +func (api *BlockChainAPI) GetStorageValues(ctx context.Context, requests map[common.Address][]common.Hash, blockNrOrHash rpc.BlockNumberOrHash) (map[common.Address][]hexutil.Bytes, error) { + // Count total slots requested. + var totalSlots int + for _, keys := range requests { + totalSlots += len(keys) + if totalSlots > maxGetStorageSlots { + return nil, &clientLimitExceededError{message: fmt.Sprintf("too many slots (max %d)", maxGetStorageSlots)} + } + } + if totalSlots == 0 { + return nil, &invalidParamsError{message: "empty request"} + } + + state, _, err := api.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { + return nil, err + } + + result := make(map[common.Address][]hexutil.Bytes, len(requests)) + for addr, keys := range requests { + vals := make([]hexutil.Bytes, len(keys)) + for i, key := range keys { + v := state.GetState(addr, key) + vals[i] = v[:] + } + if err := state.Error(); err != nil { + return nil, err + } + result[addr] = vals + } + return result, nil +} + // GetBlockReceipts returns the block receipts for the given block hash or number or tag. func (api *BlockChainAPI) GetBlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]map[string]interface{}, error) { var ( @@ -702,11 +741,10 @@ func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.S // Make sure the context is cancelled when the call has completed // this makes sure resources are cleaned up. defer cancel() - gp := new(core.GasPool) + + gp := core.NewGasPool(globalGasCap) if globalGasCap == 0 { - gp.AddGas(gomath.MaxUint64) - } else { - gp.AddGas(globalGasCap) + gp = core.NewGasPool(gomath.MaxUint64) } return applyMessage(ctx, b, args, state, header, timeout, gp, &blockCtx, &vm.Config{NoBaseFee: true}, precompiles) } @@ -811,17 +849,12 @@ func (api *BlockChainAPI) SimulateV1(ctx context.Context, opts simOpts, blockNrO if state == nil || err != nil { return nil, err } - gasCap := api.b.RPCGasCap() - if gasCap == 0 { - gasCap = gomath.MaxUint64 - } sim := &simulator{ - b: api.b, - state: state, - base: base, - chainConfig: api.b.ChainConfig(), - // Each tx and all the series of txes shouldn't consume more gas than cap - gp: new(core.GasPool).AddGas(gasCap), + b: api.b, + state: state, + base: base, + chainConfig: api.b.ChainConfig(), + budget: newGasBudget(api.b.RPCGasCap()), traceTransfers: opts.TraceTransfers, validate: opts.Validation, fullTx: opts.ReturnFullTransactions, @@ -932,6 +965,9 @@ func RPCMarshalHeader(head *types.Header) map[string]interface{} { if head.RequestsHash != nil { result["requestsHash"] = head.RequestsHash } + if head.SlotNumber != nil { + result["slotNumber"] = head.SlotNumber + } return result } @@ -1327,7 +1363,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if msg.BlobGasFeeCap != nil && msg.BlobGasFeeCap.BitLen() == 0 { evm.Context.BlobBaseFee = new(big.Int) } - res, err := core.ApplyMessage(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) + res, err := core.ApplyMessage(evm, msg, nil) if err != nil { return nil, 0, nil, fmt.Errorf("failed to apply transaction: %v err: %v", args.ToTransaction(types.LegacyTxType).Hash(), err) } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 11949fca16..d35c5b5c12 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -24,7 +24,6 @@ import ( "encoding/json" "errors" "fmt" - "math" "math/big" "os" "path/filepath" @@ -2516,7 +2515,7 @@ func TestSimulateV1ChainLinkage(t *testing.T) { state: stateDB, base: baseHeader, chainConfig: backend.ChainConfig(), - gp: new(core.GasPool).AddGas(math.MaxUint64), + budget: newGasBudget(0), traceTransfers: false, validate: false, fullTx: false, @@ -2601,7 +2600,7 @@ func TestSimulateV1TxSender(t *testing.T) { state: stateDB, base: baseHeader, chainConfig: backend.ChainConfig(), - gp: new(core.GasPool).AddGas(math.MaxUint64), + budget: newGasBudget(0), traceTransfers: false, validate: false, fullTx: true, @@ -4075,6 +4074,94 @@ func TestSendRawTransactionSync_Timeout(t *testing.T) { } } +func TestGetStorageValues(t *testing.T) { + t.Parallel() + + var ( + addr1 = common.HexToAddress("0x1111") + addr2 = common.HexToAddress("0x2222") + slot0 = common.Hash{} + slot1 = common.BigToHash(big.NewInt(1)) + slot2 = common.BigToHash(big.NewInt(2)) + val0 = common.BigToHash(big.NewInt(42)) + val1 = common.BigToHash(big.NewInt(100)) + val2 = common.BigToHash(big.NewInt(200)) + + genesis = &core.Genesis{ + Config: params.MergedTestChainConfig, + Alloc: types.GenesisAlloc{ + addr1: { + Balance: big.NewInt(params.Ether), + Storage: map[common.Hash]common.Hash{ + slot0: val0, + slot1: val1, + }, + }, + addr2: { + Balance: big.NewInt(params.Ether), + Storage: map[common.Hash]common.Hash{ + slot2: val2, + }, + }, + }, + } + ) + api := NewBlockChainAPI(newTestBackend(t, 1, genesis, beacon.New(ethash.NewFaker()), func(i int, b *core.BlockGen) { + b.SetPoS() + })) + latest := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) + + // Happy path: multiple addresses, multiple slots. + result, err := api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{ + addr1: {slot0, slot1}, + addr2: {slot2}, + }, latest) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(result) != 2 { + t.Fatalf("expected 2 addresses in result, got %d", len(result)) + } + if got := common.BytesToHash(result[addr1][0]); got != val0 { + t.Errorf("addr1 slot0: want %x, got %x", val0, got) + } + if got := common.BytesToHash(result[addr1][1]); got != val1 { + t.Errorf("addr1 slot1: want %x, got %x", val1, got) + } + if got := common.BytesToHash(result[addr2][0]); got != val2 { + t.Errorf("addr2 slot2: want %x, got %x", val2, got) + } + + // Missing slot returns zero. + result, err = api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{ + addr1: {common.HexToHash("0xff")}, + }, latest) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got := common.BytesToHash(result[addr1][0]); got != (common.Hash{}) { + t.Errorf("missing slot: want zero, got %x", got) + } + + // Empty request returns error. + _, err = api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{}, latest) + if err == nil { + t.Fatal("expected error for empty request") + } + + // Exceeding slot limit returns error. + tooMany := make([]common.Hash, maxGetStorageSlots+1) + for i := range tooMany { + tooMany[i] = common.BigToHash(big.NewInt(int64(i))) + } + _, err = api.GetStorageValues(context.Background(), map[common.Address][]common.Hash{ + addr1: tooMany, + }, latest) + if err == nil { + t.Fatal("expected error for exceeding slot limit") + } +} + func TestGetTransactionBySenderAndNonce(t *testing.T) { t.Parallel() diff --git a/internal/ethapi/override/override.go b/internal/ethapi/override/override.go index 9d57a78651..96ba77ab0a 100644 --- a/internal/ethapi/override/override.go +++ b/internal/ethapi/override/override.go @@ -19,7 +19,9 @@ package override import ( "errors" "fmt" + "maps" "math/big" + "slices" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" @@ -58,9 +60,13 @@ func (diff *StateOverride) Apply(statedb *state.StateDB, precompiles vm.Precompi if diff == nil { return nil } + // Iterate in deterministic order so error messages and behavior are stable (e.g. for tests). + addrs := slices.SortedFunc(maps.Keys(*diff), common.Address.Cmp) + // Tracks destinations of precompiles that were moved. dirtyAddrs := make(map[common.Address]struct{}) - for addr, account := range *diff { + for _, addr := range addrs { + account := (*diff)[addr] // If a precompile was moved to this address already, it can't be overridden. if _, ok := dirtyAddrs[addr]; ok { return fmt.Errorf("account %s has already been overridden by a precompile", addr.Hex()) diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index c9396cd327..f0c69e39a4 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -21,6 +21,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "math/big" "time" @@ -59,6 +60,7 @@ type simCallResult struct { ReturnValue hexutil.Bytes `json:"returnData"` Logs []*types.Log `json:"logs"` GasUsed hexutil.Uint64 `json:"gasUsed"` + MaxUsedGas hexutil.Uint64 `json:"maxUsedGas"` Status hexutil.Uint64 `json:"status"` Error *callError `json:"error,omitempty"` } @@ -149,6 +151,39 @@ func (m *simChainHeadReader) GetHeaderByHash(hash common.Hash) *types.Header { return header } +// gasBudget tracks the remaining gas allowed across all simulated blocks. +// It enforces the RPC-level gas cap to prevent DoS. +type gasBudget struct { + remaining uint64 +} + +// newGasBudget creates a gas budget with the given cap. +// A cap of 0 is treated as unlimited. +func newGasBudget(cap uint64) *gasBudget { + if cap == 0 { + cap = math.MaxUint64 + } + return &gasBudget{remaining: cap} +} + +// cap returns the given gas value clamped to the remaining budget. +func (b *gasBudget) cap(gas uint64) uint64 { + if gas > b.remaining { + return b.remaining + } + return gas +} + +// consume deducts the given amount from the budget. +// Returns an error if the amount exceeds the remaining budget. +func (b *gasBudget) consume(amount uint64) error { + if amount > b.remaining { + return fmt.Errorf("RPC gas cap exhausted: need %d, remaining %d", amount, b.remaining) + } + b.remaining -= amount + return nil +} + // simulator is a stateful object that simulates a series of blocks. // it is not safe for concurrent use. type simulator struct { @@ -156,7 +191,7 @@ type simulator struct { state *state.StateDB base *types.Header chainConfig *params.ChainConfig - gp *core.GasPool + budget *gasBudget traceTransfers bool validate bool fullTx bool @@ -200,7 +235,13 @@ func (sim *simulator) execute(ctx context.Context, blocks []simBlock) ([]*simBlo return nil, err } headers[bi] = result.Header() - results[bi] = &simBlockResult{fullTx: sim.fullTx, chainConfig: sim.chainConfig, Block: result, Calls: callResults, senders: senders} + results[bi] = &simBlockResult{ + fullTx: sim.fullTx, + chainConfig: sim.chainConfig, + Block: result, + Calls: callResults, + senders: senders, + } parent = result.Header() } return results, nil @@ -234,15 +275,19 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, blockContext.BlobBaseFee = block.BlockOverrides.BlobBaseFee.ToInt() } precompiles := sim.activePrecompiles(header) + // State overrides are applied prior to execution of a block if err := block.StateOverrides.Apply(sim.state, precompiles); err != nil { return nil, nil, nil, err } var ( - gasUsed, blobGasUsed uint64 - txes = make([]*types.Transaction, len(block.Calls)) - callResults = make([]simCallResult, len(block.Calls)) - receipts = make([]*types.Receipt, len(block.Calls)) + gp = core.NewGasPool(blockContext.GasLimit) + blobGasUsed uint64 + + txes = make([]*types.Transaction, len(block.Calls)) + callResults = make([]simCallResult, len(block.Calls)) + receipts = make([]*types.Receipt, len(block.Calls)) + // Block hash will be repaired after execution. tracer = newTracer(sim.traceTransfers, blockContext.BlockNumber.Uint64(), blockContext.Time, common.Hash{}, common.Hash{}, 0) vmConfig = &vm.Config{ @@ -272,10 +317,11 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } var allLogs []*types.Log for i, call := range block.Calls { + // Terminate if the context is cancelled if err := ctx.Err(); err != nil { return nil, nil, nil, err } - if err := sim.sanitizeCall(&call, sim.state, header, blockContext, &gasUsed); err != nil { + if err := sim.sanitizeCall(&call, sim.state, header, gp); err != nil { return nil, nil, nil, err } var ( @@ -285,10 +331,11 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, txes[i] = tx senders[txHash] = call.from() tracer.reset(txHash, uint(i)) - sim.state.SetTxContext(txHash, i) + // EoA check is always skipped, even in validation mode. + sim.state.SetTxContext(txHash, i) msg := call.ToMessage(header.BaseFee, !sim.validate) - result, err := applyMessageWithEVM(ctx, evm, msg, timeout, sim.gp) + result, err := applyMessageWithEVM(ctx, evm, msg, timeout, gp) if err != nil { txErr := txValidationError(err) return nil, nil, nil, txErr @@ -300,11 +347,17 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } else { root = sim.state.IntermediateRoot(sim.chainConfig.IsEIP158(blockContext.BlockNumber)).Bytes() } - gasUsed += result.UsedGas - receipts[i] = core.MakeReceipt(evm, result, sim.state, blockContext.BlockNumber, common.Hash{}, blockContext.Time, tx, gasUsed, root) + receipts[i] = core.MakeReceipt(evm, result, sim.state, blockContext.BlockNumber, common.Hash{}, blockContext.Time, tx, gp.CumulativeUsed(), root) blobGasUsed += receipts[i].BlobGasUsed + + // Make sure the gas cap is still enforced. It's only for + // internally protection. + if err := sim.budget.consume(result.UsedGas); err != nil { + return nil, nil, nil, err + } + logs := tracer.Logs() - callRes := simCallResult{ReturnValue: result.Return(), Logs: logs, GasUsed: hexutil.Uint64(result.UsedGas)} + callRes := simCallResult{ReturnValue: result.Return(), Logs: logs, GasUsed: hexutil.Uint64(result.UsedGas), MaxUsedGas: hexutil.Uint64(result.MaxUsedGas)} if result.Failed() { callRes.Status = hexutil.Uint64(types.ReceiptStatusFailed) if errors.Is(result.Err, vm.ErrExecutionReverted) { @@ -320,12 +373,14 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } callResults[i] = callRes } - header.GasUsed = gasUsed + // Assign total consumed gas to the header + header.GasUsed = gp.Used() if sim.chainConfig.IsCancun(header.Number, header.Time) { header.BlobGasUsed = &blobGasUsed } - var requests [][]byte + // Process EIP-7685 requests + var requests [][]byte if sim.chainConfig.IsPrague(header.Number, header.Time) { requests = [][]byte{} // EIP-6110 @@ -345,7 +400,11 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, reqHash := types.CalcRequestsHash(requests) header.RequestsHash = &reqHash } - blockBody := &types.Body{Transactions: txes, Withdrawals: *block.BlockOverrides.Withdrawals} + + blockBody := &types.Body{ + Transactions: txes, + Withdrawals: *block.BlockOverrides.Withdrawals, + } chainHeadReader := &simChainHeadReader{ctx, sim.b} b, err := sim.b.Engine().FinalizeAndAssemble(chainHeadReader, header, sim.state, blockBody, receipts) if err != nil { @@ -366,23 +425,24 @@ func repairLogs(calls []simCallResult, hash common.Hash) { } } -func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, header *types.Header, blockContext vm.BlockContext, gasUsed *uint64) error { +func (sim *simulator) sanitizeCall(call *TransactionArgs, state vm.StateDB, header *types.Header, gp *core.GasPool) error { if call.Nonce == nil { nonce := state.GetNonce(call.from()) call.Nonce = (*hexutil.Uint64)(&nonce) } // Let the call run wild unless explicitly specified. + remaining := gp.Gas() if call.Gas == nil { - remaining := blockContext.GasLimit - *gasUsed call.Gas = (*hexutil.Uint64)(&remaining) } - if *gasUsed+uint64(*call.Gas) > blockContext.GasLimit { - return &blockGasLimitReachedError{fmt.Sprintf("block gas limit reached: %d >= %d", *gasUsed, blockContext.GasLimit)} + if remaining < uint64(*call.Gas) { + return &blockGasLimitReachedError{fmt.Sprintf("block gas limit reached: remaining: %d, required: %d", remaining, *call.Gas)} } - if err := call.CallDefaults(sim.gp.Gas(), header.BaseFee, sim.chainConfig.ChainID); err != nil { - return err - } - return nil + // Clamp to the cross-block gas budget. + gas := sim.budget.cap(uint64(*call.Gas)) + call.Gas = (*hexutil.Uint64)(&gas) + + return call.CallDefaults(0, header.BaseFee, sim.chainConfig.ChainID) } func (sim *simulator) activePrecompiles(base *types.Header) vm.PrecompiledContracts { @@ -473,12 +533,14 @@ func (sim *simulator) makeHeaders(blocks []simBlock) ([]*types.Header, error) { } overrides := block.BlockOverrides - var withdrawalsHash *common.Hash number := overrides.Number.ToInt() timestamp := (uint64)(*overrides.Time) + + var withdrawalsHash *common.Hash if sim.chainConfig.IsShanghai(number, timestamp) { withdrawalsHash = &types.EmptyWithdrawalsHash } + var parentBeaconRoot *common.Hash if sim.chainConfig.IsCancun(number, timestamp) { parentBeaconRoot = &common.Hash{} @@ -508,7 +570,11 @@ func (sim *simulator) makeHeaders(blocks []simBlock) ([]*types.Header, error) { } func (sim *simulator) newSimulatedChainContext(ctx context.Context, headers []*types.Header) *ChainContext { - return NewChainContext(ctx, &simBackend{base: sim.base, b: sim.b, headers: headers}) + return NewChainContext(ctx, &simBackend{ + base: sim.base, + b: sim.b, + headers: headers, + }) } type simBackend struct { diff --git a/internal/ethapi/simulate_test.go b/internal/ethapi/simulate_test.go index c747b76477..6a83e744de 100644 --- a/internal/ethapi/simulate_test.go +++ b/internal/ethapi/simulate_test.go @@ -80,7 +80,10 @@ func TestSimulateSanitizeBlockOrder(t *testing.T) { err: "block timestamps must be in order: 72 <= 72", }, } { - sim := &simulator{base: &types.Header{Number: big.NewInt(int64(tc.baseNumber)), Time: tc.baseTimestamp}} + sim := &simulator{ + base: &types.Header{Number: big.NewInt(int64(tc.baseNumber)), Time: tc.baseTimestamp}, + budget: newGasBudget(0), + } res, err := sim.sanitizeChain(tc.blocks) if err != nil { if err.Error() == tc.err { diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 22d1adfa57..4fb30e6289 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -155,6 +155,7 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend, config AccessList: args.AccessList, BlobFeeCap: args.BlobFeeCap, BlobHashes: args.BlobHashes, + AuthorizationList: args.AuthorizationList, } latestBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) estimated, err := DoEstimateGas(ctx, b, callArgs, latestBlockNr, nil, nil, b.RPCGasCap()) diff --git a/core/rawdb/database_tablewriter.go b/internal/tablewriter/database_tablewriter.go similarity index 97% rename from core/rawdb/database_tablewriter.go rename to internal/tablewriter/database_tablewriter.go index e1cda5c93f..c080a69c15 100644 --- a/core/rawdb/database_tablewriter.go +++ b/internal/tablewriter/database_tablewriter.go @@ -16,7 +16,7 @@ // Naive stub implementation for tablewriter -package rawdb +package tablewriter import ( "errors" @@ -37,7 +37,7 @@ type Table struct { rows [][]string } -func NewTableWriter(w io.Writer) *Table { +func NewWriter(w io.Writer) *Table { return &Table{out: w} } @@ -58,12 +58,12 @@ func (t *Table) SetFooter(footer []string) { t.footer = footer } -// AppendBulk sets all data rows for the table at once, replacing any existing rows. +// AppendBulk appends one or more data rows to the table. // // Each row must have the same number of columns as the headers, or validation // will fail during Render(). func (t *Table) AppendBulk(rows [][]string) { - t.rows = rows + t.rows = append(t.rows, rows...) } // Render outputs the complete table to the configured writer. The table is rendered diff --git a/core/rawdb/database_tablewriter_test.go b/internal/tablewriter/database_tablewriter_test.go similarity index 94% rename from core/rawdb/database_tablewriter_test.go rename to internal/tablewriter/database_tablewriter_test.go index e9de5d8ce8..b915dcdda8 100644 --- a/core/rawdb/database_tablewriter_test.go +++ b/internal/tablewriter/database_tablewriter_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package rawdb +package tablewriter import ( "bytes" @@ -24,7 +24,7 @@ import ( func TestTableWriterTinyGo(t *testing.T) { var buf bytes.Buffer - table := NewTableWriter(&buf) + table := NewWriter(&buf) headers := []string{"Database", "Size", "Items", "Status"} rows := [][]string{ @@ -48,7 +48,7 @@ func TestTableWriterValidationErrors(t *testing.T) { // Test missing headers t.Run("MissingHeaders", func(t *testing.T) { var buf bytes.Buffer - table := NewTableWriter(&buf) + table := NewWriter(&buf) rows := [][]string{{"x", "y", "z"}} @@ -63,7 +63,7 @@ func TestTableWriterValidationErrors(t *testing.T) { t.Run("NotEnoughRowColumns", func(t *testing.T) { var buf bytes.Buffer - table := NewTableWriter(&buf) + table := NewWriter(&buf) headers := []string{"A", "B", "C"} badRows := [][]string{ @@ -82,7 +82,7 @@ func TestTableWriterValidationErrors(t *testing.T) { t.Run("TooManyRowColumns", func(t *testing.T) { var buf bytes.Buffer - table := NewTableWriter(&buf) + table := NewWriter(&buf) headers := []string{"A", "B", "C"} badRows := [][]string{ @@ -102,7 +102,7 @@ func TestTableWriterValidationErrors(t *testing.T) { // Test mismatched footer columns t.Run("MismatchedFooterColumns", func(t *testing.T) { var buf bytes.Buffer - table := NewTableWriter(&buf) + table := NewWriter(&buf) headers := []string{"A", "B", "C"} rows := [][]string{{"x", "y", "z"}} diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index 470bafed25..27fe9b0a7a 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -52,6 +52,11 @@ func StartSpan(ctx context.Context, spanName string, attributes ...Attribute) (c // StartSpanWithTracer requires a tracer to be passed in and creates a SpanKind=INTERNAL span. func StartSpanWithTracer(ctx context.Context, tracer trace.Tracer, name string, attributes ...Attribute) (context.Context, trace.Span, func(*error)) { + // Don't create a span if there's no parent span in the context. + parent := trace.SpanFromContext(ctx) + if !parent.SpanContext().IsValid() { + return ctx, parent, func(*error) {} + } return startSpan(ctx, tracer, trace.SpanKindInternal, name, attributes...) } diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go new file mode 100644 index 0000000000..def85b735e --- /dev/null +++ b/internal/telemetry/telemetry_test.go @@ -0,0 +1,98 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package telemetry + +import ( + "context" + "testing" + + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" +) + +// newTestTracer creates a TracerProvider backed by an in-memory exporter. +func newTestTracer(t *testing.T) (trace.Tracer, *sdktrace.TracerProvider, *tracetest.InMemoryExporter) { + t.Helper() + exporter := tracetest.NewInMemoryExporter() + tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter)) + t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + return tp.Tracer("test"), tp, exporter +} + +func TestStartSpanWithTracer_NoParent(t *testing.T) { + t.Parallel() + tracer, tp, exporter := newTestTracer(t) + + // Create a span without a parent. + ctx := context.Background() + retCtx, _, endSpan := StartSpanWithTracer(ctx, tracer, "should-not-exist") + endSpan(nil) + + // The returned context should be the original context (unchanged). + if retCtx != ctx { + t.Fatal("expected original context to be returned unchanged") + } + + // Flush and verify no spans were recorded. + if err := tp.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + if len(spans) != 0 { + t.Fatalf("expected no spans, got %d", len(spans)) + } +} + +func TestStartSpanWithTracer_WithParent(t *testing.T) { + t.Parallel() + tracer, tp, exporter := newTestTracer(t) + + // Create a parent span to establish a valid span context. + ctx, parentSpan := tracer.Start(context.Background(), "parent") + defer parentSpan.End() + + // Should create a real child span. + _, _, endSpan := StartSpanWithTracer(ctx, tracer, "child") + endSpan(nil) + + // Flush and verify the child span was recorded. + parentSpan.End() + if err := tp.ForceFlush(context.Background()); err != nil { + t.Fatalf("failed to flush: %v", err) + } + spans := exporter.GetSpans() + var childSpan *tracetest.SpanStub + for i := range spans { + if spans[i].Name == "child" { + childSpan = &spans[i] + break + } + } + if childSpan == nil { + t.Fatal("child span not found") + } + + // Verify it is parented to the correct trace. + if childSpan.Parent.TraceID() != parentSpan.SpanContext().TraceID() { + t.Errorf("trace ID mismatch: got %s, want %s", + childSpan.Parent.TraceID(), parentSpan.SpanContext().TraceID()) + } + if childSpan.SpanKind != trace.SpanKindInternal { + t.Errorf("expected SpanKindInternal, got %v", childSpan.SpanKind) + } +} diff --git a/internal/telemetry/tracesetup/setup.go b/internal/telemetry/tracesetup/setup.go index 9637ca1a9b..444416dd26 100644 --- a/internal/telemetry/tracesetup/setup.go +++ b/internal/telemetry/tracesetup/setup.go @@ -113,7 +113,7 @@ func SetupTelemetry(cfg node.OpenTelemetryConfig, stack *node.Node) error { // Define batch span processor options batchOpts := []sdktrace.BatchSpanProcessorOption{ // The maximum number of spans that can be queued before dropping - sdktrace.WithMaxQueueSize(sdktrace.DefaultMaxExportBatchSize), + sdktrace.WithMaxQueueSize(sdktrace.DefaultMaxQueueSize), // The maximum number of spans to export in a single batch sdktrace.WithMaxExportBatchSize(sdktrace.DefaultMaxExportBatchSize), // How long an export operation can take before timing out diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 0aedffe230..1d1b5fbcd1 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -427,11 +427,6 @@ web3._extend({ params: 2, inputFormatter:[null, null], }), - new web3._extend.Method({ - name: 'freezeClient', - call: 'debug_freezeClient', - params: 1, - }), new web3._extend.Method({ name: 'getAccessibleState', call: 'debug_getAccessibleState', @@ -474,6 +469,12 @@ web3._extend({ params: 1, inputFormatter: [null], }), + new web3._extend.Method({ + name: 'executionWitness', + call: 'debug_executionWitness', + params: 1, + inputFormatter: [null], + }), ], properties: [] }); @@ -567,6 +568,12 @@ web3._extend({ params: 3, inputFormatter: [web3._extend.formatters.inputAddressFormatter, null, web3._extend.formatters.inputBlockNumberFormatter] }), + new web3._extend.Method({ + name: 'getStorageValues', + call: 'eth_getStorageValues', + params: 2, + inputFormatter: [null, web3._extend.formatters.inputBlockNumberFormatter] + }), new web3._extend.Method({ name: 'createAccessList', call: 'eth_createAccessList', diff --git a/metrics/config.go b/metrics/config.go index 72f94dd194..3d3cb693fd 100644 --- a/metrics/config.go +++ b/metrics/config.go @@ -16,18 +16,21 @@ package metrics +import "time" + // Config contains the configuration for the metric collection. type Config struct { - Enabled bool `toml:",omitempty"` - EnabledExpensive bool `toml:"-"` - HTTP string `toml:",omitempty"` - Port int `toml:",omitempty"` - EnableInfluxDB bool `toml:",omitempty"` - InfluxDBEndpoint string `toml:",omitempty"` - InfluxDBDatabase string `toml:",omitempty"` - InfluxDBUsername string `toml:",omitempty"` - InfluxDBPassword string `toml:",omitempty"` - InfluxDBTags string `toml:",omitempty"` + Enabled bool `toml:",omitempty"` + EnabledExpensive bool `toml:"-"` + HTTP string `toml:",omitempty"` + Port int `toml:",omitempty"` + EnableInfluxDB bool `toml:",omitempty"` + InfluxDBEndpoint string `toml:",omitempty"` + InfluxDBDatabase string `toml:",omitempty"` + InfluxDBUsername string `toml:",omitempty"` + InfluxDBPassword string `toml:",omitempty"` + InfluxDBTags string `toml:",omitempty"` + InfluxDBInterval time.Duration `toml:",omitempty"` EnableInfluxDBV2 bool `toml:",omitempty"` InfluxDBToken string `toml:",omitempty"` @@ -47,6 +50,7 @@ var DefaultConfig = Config{ InfluxDBUsername: "test", InfluxDBPassword: "test", InfluxDBTags: "host=localhost", + InfluxDBInterval: 10 * time.Second, // influxdbv2-specific flags EnableInfluxDBV2: false, diff --git a/miner/miner_test.go b/miner/miner_test.go index 575ee4d0fd..13475a19b6 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -155,7 +155,7 @@ func createMiner(t *testing.T) *Miner { if err != nil { t.Fatalf("can't create new chain %v", err) } - statedb, _ := state.New(bc.Genesis().Root(), bc.StateCache()) + statedb, _ := state.New(bc.Genesis().Root(), state.NewDatabase(bc.TrieDB(), bc.CodeDB())) blockchain := &testBlockChain{bc.Genesis().Root(), chainConfig, statedb, 10000000, new(event.Feed)} pool := legacypool.New(testTxPoolConfig, blockchain) diff --git a/miner/payload_building.go b/miner/payload_building.go index 6b010186bf..8bd9552338 100644 --- a/miner/payload_building.go +++ b/miner/payload_building.go @@ -43,6 +43,7 @@ type BuildPayloadArgs struct { Random common.Hash // The provided randomness value Withdrawals types.Withdrawals // The provided withdrawals BeaconRoot *common.Hash // The provided beaconRoot (Cancun) + SlotNum *uint64 // The provided slotNumber Version engine.PayloadVersion // Versioning byte for payload id calculation. } @@ -57,6 +58,9 @@ func (args *BuildPayloadArgs) Id() engine.PayloadID { if args.BeaconRoot != nil { hasher.Write(args.BeaconRoot[:]) } + if args.SlotNum != nil { + binary.Write(hasher, binary.BigEndian, args.SlotNum) + } var out engine.PayloadID copy(out[:], hasher.Sum(nil)[:8]) out[0] = byte(args.Version) @@ -218,6 +222,7 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload random: args.Random, withdrawals: args.Withdrawals, beaconRoot: args.BeaconRoot, + slotNum: args.SlotNum, noTxs: true, } empty := miner.generateWork(emptyParams, witness) @@ -248,12 +253,24 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload random: args.Random, withdrawals: args.Withdrawals, beaconRoot: args.BeaconRoot, + slotNum: args.SlotNum, noTxs: false, } for { select { case <-timer.C: + // When block building takes close to the full recommit interval, + // the timer fires near-instantly on the next iteration. If the + // payload was resolved during that build, both timer.C and + // payload.stop are ready and Go's select picks one at random. + // Check payload.stop first to avoid an unnecessary generateWork. + select { + case <-payload.stop: + log.Info("Stopping work on payload", "id", payload.id, "reason", "delivery") + return + default: + } start := time.Now() r := miner.generateWork(fullParams, witness) if r.err == nil { @@ -261,7 +278,7 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload } else { log.Info("Error while generating work", "id", payload.id, "err", r.err) } - timer.Reset(miner.config.Recommit) + timer.Reset(max(0, miner.config.Recommit-time.Since(start))) case <-payload.stop: log.Info("Stopping work on payload", "id", payload.id, "reason", "delivery") return @@ -273,3 +290,26 @@ func (miner *Miner) buildPayload(args *BuildPayloadArgs, witness bool) (*Payload }() return payload, nil } + +// BuildTestingPayload is for testing_buildBlockV*. It creates a block with the exact content given +// by the parameters instead of using the locally available transactions. +func (miner *Miner) BuildTestingPayload(args *BuildPayloadArgs, transactions []*types.Transaction, empty bool, extraData []byte) (*engine.ExecutionPayloadEnvelope, error) { + fullParams := &generateParams{ + timestamp: args.Timestamp, + forceTime: true, + parentHash: args.Parent, + coinbase: args.FeeRecipient, + random: args.Random, + withdrawals: args.Withdrawals, + beaconRoot: args.BeaconRoot, + noTxs: empty, + forceOverrides: true, + overrideExtraData: extraData, + overrideTxs: transactions, + } + res := miner.generateWork(fullParams, false) + if res.err != nil { + return nil, res.err + } + return engine.BlockToExecutableData(res.block, new(big.Int), res.sidecars, res.requests), nil +} diff --git a/miner/stress/main.go b/miner/stress/main.go new file mode 100644 index 0000000000..aaf0993c37 --- /dev/null +++ b/miner/stress/main.go @@ -0,0 +1,210 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// This file contains a miner stress test based on the Engine API flow. +package main + +import ( + "crypto/ecdsa" + "math/big" + "math/rand" + "os" + "os/signal" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/fdlimit" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/catalyst" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/params" +) + +var refundContract = common.HexToAddress("0x1000000000000000000000000000000000000001") + +func main() { + log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelInfo, true))) + fdlimit.Raise(2048) + + // Generate a batch of accounts to seal and fund with + faucets := make([]*ecdsa.PrivateKey, 128) + for i := 0; i < len(faucets); i++ { + faucets[i], _ = crypto.GenerateKey() + } + // Create a post-merge network where blocks are built/inserted through + // engine API calls driven by a simulated beacon client. + genesis := makeGenesis(faucets) + + // Handle interrupts. + interruptCh := make(chan os.Signal, 5) + signal.Notify(interruptCh, os.Interrupt) + + // Start one node that accepts transactions and builds/inserts blocks via + // Engine API (through the simulated beacon driver). + stack, backend, beacon, err := makeNode(genesis) + if err != nil { + panic(err) + } + defer stack.Close() + defer beacon.Stop() + + // Start injecting transactions from the faucet like crazy + var ( + sent uint64 + nonces = make([]uint64, len(faucets)) + signer = types.LatestSigner(genesis.Config) + refundSet = true // slot 0 starts as non-zero in genesis + ) + for { + // Stop when interrupted. + select { + case <-interruptCh: + return + default: + } + + var ( + tx *types.Transaction + err error + ) + // Every third tx targets a contract path that alternates set/clear. + // Clearing a previously non-zero slot triggers gas refund. + if sent%3 == 0 { + var data []byte + if refundSet { + data = nil // empty calldata => clear slot to zero (refund path) + } else { + data = []byte{0x01} // non-empty calldata => set slot to one + } + tx, err = types.SignTx(types.NewTransaction(nonces[0], refundContract, new(big.Int), 50000, big.NewInt(100000000000), data), signer, faucets[0]) + if err != nil { + panic(err) + } + nonces[0]++ + refundSet = !refundSet + } else { + index := 1 + rand.Intn(len(faucets)-1) + tx, err = types.SignTx(types.NewTransaction(nonces[index], crypto.PubkeyToAddress(faucets[index].PublicKey), new(big.Int), 21000, big.NewInt(100000000000), nil), signer, faucets[index]) + if err != nil { + panic(err) + } + nonces[index]++ + } + errs := backend.TxPool().Add([]*types.Transaction{tx}, true) + for _, err := range errs { + if err != nil { + panic(err) + } + } + sent++ + + // Create and import blocks through the engine API path. + if sent%256 == 0 { + beacon.Commit() + } + + // Wait if we're too saturated + if pend, _ := backend.TxPool().Stats(); pend > 4096 { + beacon.Commit() + time.Sleep(50 * time.Millisecond) + } + } +} + +// makeGenesis creates a post-merge genesis block. +func makeGenesis(faucets []*ecdsa.PrivateKey) *core.Genesis { + config := *params.AllDevChainProtocolChanges + config.ChainID = big.NewInt(18) + + blockZero := uint64(0) + config.AmsterdamTime = &blockZero + config.BlobScheduleConfig.Amsterdam = ¶ms.BlobConfig{ + Target: 14, + Max: 21, + UpdateFraction: 13739630, + } + + genesis := &core.Genesis{ + Config: &config, + GasLimit: 25000000, + Alloc: types.GenesisAlloc{}, + } + for _, faucet := range faucets { + genesis.Alloc[crypto.PubkeyToAddress(faucet.PublicKey)] = types.Account{ + Balance: new(big.Int).Exp(big.NewInt(2), big.NewInt(128), nil), + } + } + // Runtime code: + // - empty calldata: SSTORE(0,0) + // - non-empty calldata: SSTORE(0,1) + // Slot 0 is initialized to 1 so the first clear includes gas refund. + genesis.Alloc[refundContract] = types.Account{ + Code: common.FromHex("0x3615600b576001600055005b600060005500"), + Storage: map[common.Hash]common.Hash{ + common.Hash{}: common.BigToHash(big.NewInt(1)), + }, + } + return genesis +} + +func makeNode(genesis *core.Genesis) (*node.Node, *eth.Ethereum, *catalyst.SimulatedBeacon, error) { + // Define the basic configurations for the Ethereum node + datadir, _ := os.MkdirTemp("", "") + + config := &node.Config{ + Name: "geth", + DataDir: datadir, + } + // Start the node and configure a full Ethereum node on it + stack, err := node.New(config) + if err != nil { + return nil, nil, nil, err + } + // Create and register the backend + ethBackend, err := eth.New(stack, ðconfig.Config{ + Genesis: genesis, + NetworkId: genesis.Config.ChainID.Uint64(), + SyncMode: downloader.FullSync, + DatabaseCache: 256, + DatabaseHandles: 256, + TxPool: legacypool.DefaultConfig, + GPO: ethconfig.Defaults.GPO, + Miner: ethconfig.Defaults.Miner, + SlowBlockThreshold: time.Second, + }) + if err != nil { + return nil, nil, nil, err + } + + if err := stack.Start(); err != nil { + return nil, nil, nil, err + } + driver, err := catalyst.NewSimulatedBeacon(0, common.Address{}, ethBackend) + if err != nil { + return nil, nil, nil, err + } + if err := driver.Start(); err != nil { + return nil, nil, nil, err + } + return stack, ethBackend, driver, nil +} diff --git a/miner/worker.go b/miner/worker.go index 45d7073ed7..bfeeaa248b 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -78,6 +78,11 @@ func (env *environment) txFitsSize(tx *types.Transaction) bool { return env.size+tx.Size() < params.MaxBlockSize-maxBlockSizeBufferZone } +// discard terminates the background threads before discarding it. +func (env *environment) discard() { + env.state.StopPrefetcher() +} + const ( commitInterruptNone int32 = iota commitInterruptNewHead @@ -111,7 +116,12 @@ type generateParams struct { random common.Hash // The randomness generated by beacon chain, empty before the merge withdrawals types.Withdrawals // List of withdrawals to include in block (shanghai field) beaconRoot *common.Hash // The beacon root (cancun field). + slotNum *uint64 // The slot number (amsterdam field). noTxs bool // Flag whether an empty block without any transaction is expected + + forceOverrides bool // Flag whether we should overwrite extraData and transactions + overrideExtraData []byte + overrideTxs []*types.Transaction } // generateWork generates a sealing block based on the given parameters. @@ -120,6 +130,7 @@ func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPay if err != nil { return &newPayloadResult{err: err} } + defer work.discard() // Check withdrawals fit max block size. // Due to the cap on withdrawal count, this can actually never happen, but we still need to @@ -132,15 +143,27 @@ func (miner *Miner) generateWork(genParam *generateParams, witness bool) *newPay work.size += uint64(genParam.withdrawals.Size()) if !genParam.noTxs { - interrupt := new(atomic.Int32) - timer := time.AfterFunc(miner.config.Recommit, func() { - interrupt.Store(commitInterruptTimeout) - }) - defer timer.Stop() + // If forceOverrides is true and overrideTxs is not empty, commit the override transactions + // otherwise, fill the block with the current transactions from the txpool + if genParam.forceOverrides && len(genParam.overrideTxs) > 0 { + for _, tx := range genParam.overrideTxs { + work.state.SetTxContext(tx.Hash(), work.tcount) + if err := miner.commitTransaction(work, tx); err != nil { + // all passed transactions HAVE to be valid at this point + return &newPayloadResult{err: err} + } + } + } else { + interrupt := new(atomic.Int32) + timer := time.AfterFunc(miner.config.Recommit, func() { + interrupt.Store(commitInterruptTimeout) + }) + defer timer.Stop() - err := miner.fillTransactions(interrupt, work) - if errors.Is(err, errBlockInterruptedByTimeout) { - log.Warn("Block building is interrupted", "allowance", common.PrettyDuration(miner.config.Recommit)) + err := miner.fillTransactions(interrupt, work) + if errors.Is(err, errBlockInterruptedByTimeout) { + log.Warn("Block building is interrupted", "allowance", common.PrettyDuration(miner.config.Recommit)) + } } } body := types.Body{Transactions: work.txs, Withdrawals: genParam.withdrawals} @@ -224,6 +247,9 @@ func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*envir if len(miner.config.ExtraData) != 0 { header.Extra = miner.config.ExtraData } + if genParams.forceOverrides { + header.Extra = genParams.overrideExtraData + } // Set the randomness field from the beacon chain if it's available. if genParams.random != (common.Hash{}) { header.MixDigest = genParams.random @@ -252,6 +278,13 @@ func (miner *Miner) prepareWork(genParams *generateParams, witness bool) (*envir header.ExcessBlobGas = &excessBlobGas header.ParentBeaconRoot = genParams.beaconRoot } + // Apply EIP-7843. + if miner.chainConfig.IsAmsterdam(header.Number, header.Time) { + if genParams.slotNum == nil { + return nil, errors.New("no slot number set post-amsterdam") + } + header.SlotNumber = genParams.slotNum + } // Could potentially happen if starting to mine in an odd state. // Note genParams.coinbase can be different with header.Coinbase // since clique algorithm can modify the coinbase field in header. @@ -276,19 +309,21 @@ func (miner *Miner) makeEnv(parent *types.Header, header *types.Header, coinbase if err != nil { return nil, err } + var bundle *stateless.Witness if witness { - bundle, err := stateless.NewWitness(header, miner.chain) + bundle, err = stateless.NewWitness(header, miner.chain) if err != nil { return nil, err } - state.StartPrefetcher("miner", bundle, nil) } + state.StartPrefetcher("miner", bundle, nil) // Note the passed coinbase may be different with header.Coinbase. return &environment{ signer: types.MakeSigner(miner.chainConfig, header.Number, header.Time), state: state, size: uint64(header.Size()), coinbase: coinbase, + gasPool: core.NewGasPool(header.GasLimit), header: header, witness: state.Witness(), evm: vm.NewEVM(core.NewEVMBlockContext(header, miner.chain, &coinbase), state, miner.chainConfig, vm.Config{}), @@ -342,24 +377,20 @@ func (miner *Miner) commitBlobTransaction(env *environment, tx *types.Transactio func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (*types.Receipt, error) { var ( snap = env.state.Snapshot() - gp = env.gasPool.Gas() + gp = env.gasPool.Snapshot() ) - receipt, err := core.ApplyTransaction(env.evm, env.gasPool, env.state, env.header, tx, &env.header.GasUsed) + receipt, err := core.ApplyTransaction(env.evm, env.gasPool, env.state, env.header, tx) if err != nil { env.state.RevertToSnapshot(snap) - env.gasPool.SetGas(gp) + env.gasPool.Set(gp) + return nil, err } - return receipt, err + env.header.GasUsed = env.gasPool.Used() + return receipt, nil } func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *transactionsByPriceAndNonce, interrupt *atomic.Int32) error { - var ( - isCancun = miner.chainConfig.IsCancun(env.header.Number, env.header.Time) - gasLimit = env.header.GasLimit - ) - if env.gasPool == nil { - env.gasPool = new(core.GasPool).AddGas(gasLimit) - } + isCancun := miner.chainConfig.IsCancun(env.header.Number, env.header.Time) for { // Check interruption signal and abort building if it's fired. if interrupt != nil { diff --git a/node/node.go b/node/node.go index f9ebb243b0..01318881d4 100644 --- a/node/node.go +++ b/node/node.go @@ -152,8 +152,10 @@ func New(conf *Config) (*Node, error) { // Configure RPC servers. node.http = newHTTPServer(node.log, conf.HTTPTimeouts) node.httpAuth = newHTTPServer(node.log, conf.HTTPTimeouts) + node.httpAuth.disableHTTP2 = true // Engine API does not need HTTP/2 node.ws = newHTTPServer(node.log, rpc.DefaultHTTPTimeouts) node.wsAuth = newHTTPServer(node.log, rpc.DefaultHTTPTimeouts) + node.wsAuth.disableHTTP2 = true node.ipc = newIPCServer(node.log, conf.IPCEndpoint()) return node, nil diff --git a/node/rpcstack.go b/node/rpcstack.go index a9ac88e4de..20d488b734 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -90,6 +90,9 @@ type httpServer struct { port int handlerNames map[string]string + + // disableHTTP2 disables HTTP/2 support on this server when set to true. + disableHTTP2 bool } const ( @@ -140,7 +143,9 @@ func (h *httpServer) start() error { h.server = &http.Server{Handler: h} h.server.Protocols = new(http.Protocols) h.server.Protocols.SetHTTP1(true) - h.server.Protocols.SetUnencryptedHTTP2(true) + if !h.disableHTTP2 { + h.server.Protocols.SetUnencryptedHTTP2(true) + } if h.timeouts != (rpc.HTTPTimeouts{}) { CheckTimeouts(&h.timeouts) h.server.ReadTimeout = h.timeouts.ReadTimeout diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 6abe20d7a4..9429fbaf0a 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -856,10 +856,10 @@ type testCodecFrame struct { } func (c *testCodec) Encode(toID enode.ID, addr string, p v5wire.Packet, _ *v5wire.Whoareyou) ([]byte, v5wire.Nonce, error) { - // To match the behavior of v5wire.Codec, we return the cached encoding of - // WHOAREYOU challenges. - if wp, ok := p.(*v5wire.Whoareyou); ok && len(wp.Encoded) > 0 { - return wp.Encoded, wp.Nonce, nil + if wp, ok := p.(*v5wire.Whoareyou); ok && len(wp.ChallengeData) > 0 { + // To match the behavior of v5wire.Codec, we return the cached encoding of + // WHOAREYOU challenges. + return wp.ChallengeData, wp.Nonce, nil } c.ctr++ @@ -874,7 +874,7 @@ func (c *testCodec) Encode(toID enode.ID, addr string, p v5wire.Packet, _ *v5wir // Store recently sent challenges. if w, ok := p.(*v5wire.Whoareyou); ok { w.Nonce = authTag - w.Encoded = frame + w.ChallengeData = frame if c.sentChallenges == nil { c.sentChallenges = make(map[enode.ID]*v5wire.Whoareyou) } @@ -911,6 +911,7 @@ func (c *testCodec) decodeFrame(input []byte) (frame testCodecFrame, p v5wire.Pa case v5wire.WhoareyouPacket: dec := new(v5wire.Whoareyou) err = rlp.DecodeBytes(frame.Packet, &dec) + dec.ChallengeData = bytes.Clone(input) p = dec default: p, err = v5wire.DecodeMessage(frame.Ptype, frame.Packet) diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index d6a30a17ca..9a5bbd4612 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -190,10 +190,16 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar ) switch { case packet.Kind() == WhoareyouPacket: - // just send the WHOAREYOU packet raw again, rather than the re-encoded challenge data w := packet.(*Whoareyou) - if len(w.Encoded) > 0 { - return w.Encoded, w.Nonce, nil + if len(w.ChallengeData) > 0 { + // This WHOAREYOU packet was encoded before, so it's a resend. + // The unmasked packet content is stored in w.ChallengeData. + // Just apply the masking again to finish encoding. + c.buf.Reset() + c.buf.Write(w.ChallengeData) + copy(head.IV[:], w.ChallengeData) + enc := applyMasking(id, head.IV, c.buf.Bytes()) + return enc, w.Nonce, nil } head, err = c.encodeWhoareyou(id, packet.(*Whoareyou)) case challenge != nil: @@ -228,7 +234,6 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar if err != nil { return nil, Nonce{}, err } - challenge.Encoded = bytes.Clone(enc) c.sc.storeSentHandshake(id, addr, challenge) return enc, head.Nonce, err } @@ -246,14 +251,10 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar // EncodeRaw encodes a packet with the given header. func (c *Codec) EncodeRaw(id enode.ID, head Header, msgdata []byte) ([]byte, error) { + // header c.writeHeaders(&head) - - // Apply masking. - masked := c.buf.Bytes()[sizeofMaskingIV:] - mask := head.mask(id) - mask.XORKeyStream(masked[:], masked[:]) - - // Write message data. + applyMasking(id, head.IV, c.buf.Bytes()) + // message data c.buf.Write(msgdata) return c.buf.Bytes(), nil } @@ -463,7 +464,7 @@ func (c *Codec) Decode(inputData []byte, addr string) (src enode.ID, n *enode.No // Unmask the static header. var head Header copy(head.IV[:], input[:sizeofMaskingIV]) - mask := head.mask(c.localnode.ID()) + mask := createMask(c.localnode.ID(), head.IV) staticHeader := input[sizeofMaskingIV:sizeofStaticPacketData] mask.XORKeyStream(staticHeader, staticHeader) @@ -679,10 +680,17 @@ func (h *StaticHeader) checkValid(packetLen int, protocolID [6]byte) error { } // mask returns a cipher for 'masking' / 'unmasking' packet headers. -func (h *Header) mask(destID enode.ID) cipher.Stream { +func createMask(destID enode.ID, iv [16]byte) cipher.Stream { block, err := aes.NewCipher(destID[:16]) if err != nil { panic("can't create cipher") } - return cipher.NewCTR(block, h.IV[:]) + return cipher.NewCTR(block, iv[:]) +} + +func applyMasking(destID enode.ID, iv [16]byte, packet []byte) []byte { + masked := packet[sizeofMaskingIV:] + mask := createMask(destID, iv) + mask.XORKeyStream(masked[:], masked[:]) + return packet } diff --git a/p2p/discover/v5wire/encoding_test.go b/p2p/discover/v5wire/encoding_test.go index 5774cb3d8c..dd7ec6d53c 100644 --- a/p2p/discover/v5wire/encoding_test.go +++ b/p2p/discover/v5wire/encoding_test.go @@ -269,6 +269,35 @@ func TestHandshake_BadHandshakeAttack(t *testing.T) { net.nodeB.expectDecodeErr(t, errUnexpectedHandshake, findnode) } +func TestEncodeWhoareyouResend(t *testing.T) { + t.Parallel() + net := newHandshakeTest() + defer net.close() + + // A -> B WHOAREYOU + challenge := &Whoareyou{ + Nonce: Nonce{1, 2, 3, 4}, + IDNonce: testIDnonce, + RecordSeq: 0, + } + enc, _ := net.nodeA.encode(t, net.nodeB, challenge) + net.nodeB.expectDecode(t, WhoareyouPacket, enc) + whoareyou1 := bytes.Clone(enc) + + if len(challenge.ChallengeData) == 0 { + t.Fatal("ChallengeData not assigned by encode") + } + + // A -> B WHOAREYOU + // Send the same challenge again. This should produce exactly + // the same bytes as the first send. + enc, _ = net.nodeA.encode(t, net.nodeB, challenge) + whoareyou2 := bytes.Clone(enc) + if !bytes.Equal(whoareyou2, whoareyou1) { + t.Fatal("re-encoded challenge not equal to first") + } +} + // This test checks some malformed packets. func TestDecodeErrorsV5(t *testing.T) { t.Parallel() diff --git a/p2p/discover/v5wire/msg.go b/p2p/discover/v5wire/msg.go index 089fd4ebdc..e5c29276ca 100644 --- a/p2p/discover/v5wire/msg.go +++ b/p2p/discover/v5wire/msg.go @@ -63,19 +63,20 @@ type ( // WHOAREYOU contains the handshake challenge. Whoareyou struct { - ChallengeData []byte // Encoded challenge - Nonce Nonce // Nonce of request packet - IDNonce [16]byte // Identity proof data - RecordSeq uint64 // ENR sequence number of recipient + Nonce Nonce // Nonce of request packet + IDNonce [16]byte // Identity proof data + RecordSeq uint64 // ENR sequence number of recipient // Node is the locally known node record of recipient. // This must be set by the caller of Encode. - Node *enode.Node + Node *enode.Node `rlp:"-"` + + // ChallengeData stores the unmasked encoding of the whole packet. This is the + // input data for verification. It is assigned by both Encode and Decode + // operations. + ChallengeData []byte `rlp:"-"` sent mclock.AbsTime // for handshake GC. - - // Encoded is packet raw data for sending out, but should not be include in the RLP encoding. - Encoded []byte `rlp:"-"` } // PING is sent during liveness checks. diff --git a/p2p/tracker/tracker.go b/p2p/tracker/tracker.go index 3a9135aa3b..a5c790d780 100644 --- a/p2p/tracker/tracker.go +++ b/p2p/tracker/tracker.go @@ -138,6 +138,10 @@ func (t *Tracker) clean() { t.lock.Lock() defer t.lock.Unlock() + if t.expire == nil { + return // Tracker was stopped. + } + // Expire anything within a certain threshold (might be no items at all if // we raced with the delivery) for t.expire.Len() > 0 { @@ -162,14 +166,15 @@ func (t *Tracker) clean() { t.schedule() } -// schedule starts a timer to trigger on the expiration of the first network -// packet. +// schedule starts a timer to trigger on the expiration of the first network packet. func (t *Tracker) schedule() { if t.expire.Len() == 0 { t.wake = nil return } - t.wake = time.AfterFunc(time.Until(t.pending[t.expire.Front().Value.(uint64)].time.Add(t.timeout)), t.clean) + nextID := t.expire.Front().Value.(uint64) + nextTime := t.pending[nextID].time + t.wake = time.AfterFunc(time.Until(nextTime.Add(t.timeout)), t.clean) } // Stop reclaims resources of the tracker. diff --git a/p2p/tracker/tracker_test.go b/p2p/tracker/tracker_test.go index a37a59f70f..95e9629641 100644 --- a/p2p/tracker/tracker_test.go +++ b/p2p/tracker/tracker_test.go @@ -24,6 +24,25 @@ import ( "github.com/ethereum/go-ethereum/p2p" ) +// TestCleanAfterStop verifies that the clean method does not crash when called +// after Stop. This can happen because clean is scheduled via time.AfterFunc and +// may fire after Stop sets t.expire to nil. +func TestCleanAfterStop(t *testing.T) { + cap := p2p.Cap{Name: "test", Version: 1} + timeout := 50 * time.Millisecond + tr := New(cap, "peer1", timeout) + + // Track a request to start the expiration timer. + tr.Track(Request{ID: 1, ReqCode: 0x01, RespCode: 0x02, Size: 1}) + + // Stop the tracker, then wait for the timer to fire. + tr.Stop() + time.Sleep(timeout + 50*time.Millisecond) + + // Also verify that calling clean directly after stop doesn't panic. + tr.clean() +} + // This checks that metrics gauges for pending requests are be decremented when a // Tracker is stopped. func TestMetricsOnStop(t *testing.T) { diff --git a/params/protocol_params.go b/params/protocol_params.go index bb506af015..cebf5008c8 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -134,8 +134,10 @@ const ( DefaultElasticityMultiplier = 2 // Bounds the maximum gas limit an EIP-1559 block may have. InitialBaseFee = 1000000000 // Initial base fee for EIP-1559 blocks. - MaxCodeSize = 24576 // Maximum bytecode to permit for a contract - MaxInitCodeSize = 2 * MaxCodeSize // Maximum initcode to permit in a creation transaction and create instructions + MaxCodeSize = 24576 // Maximum bytecode to permit for a contract + MaxInitCodeSize = 2 * MaxCodeSize // Maximum initcode to permit in a creation transaction and create instructions + MaxCodeSizeAmsterdam = 32768 // Maximum bytecode to permit for a contract post Amsterdam + MaxInitCodeSizeAmsterdam = 2 * MaxCodeSizeAmsterdam // Maximum initcode to permit in a creation transaction and create instructions post Amsterdam // Precompiled contract gas prices diff --git a/signer/core/uiapi.go b/signer/core/uiapi.go index 2f511c7e19..09ee4b492f 100644 --- a/signer/core/uiapi.go +++ b/signer/core/uiapi.go @@ -73,8 +73,9 @@ type rawWallet struct { // Example call // {"jsonrpc":"2.0","method":"clef_listWallets","params":[], "id":5} func (api *UIServerAPI) ListWallets() []rawWallet { - wallets := make([]rawWallet, 0) // return [] instead of nil if empty - for _, wallet := range api.am.Wallets() { + allWallets := api.am.Wallets() + wallets := make([]rawWallet, 0, len(allWallets)) // return [] instead of nil if empty + for _, wallet := range allWallets { status, failure := wallet.Status() raw := rawWallet{ @@ -130,8 +131,12 @@ func (api *UIServerAPI) ImportRawKey(privkey string, password string) (accounts. if err := ValidatePasswordFormat(password); err != nil { return accounts.Account{}, fmt.Errorf("password requirements not met: %v", err) } + ks := fetchKeystore(api.am) + if ks == nil { + return accounts.Account{}, errors.New("password based accounts not supported") + } // No error - return fetchKeystore(api.am).ImportECDSA(key, password) + return ks.ImportECDSA(key, password) } // OpenWallet initiates a hardware wallet opening procedure, establishing a USB diff --git a/tests/block_test_util.go b/tests/block_test_util.go index 4f6ab65c1a..00411073e2 100644 --- a/tests/block_test_util.go +++ b/tests/block_test_util.go @@ -97,6 +97,7 @@ type btHeader struct { BlobGasUsed *uint64 ExcessBlobGas *uint64 ParentBeaconBlockRoot *common.Hash + SlotNumber *uint64 } type btHeaderMarshaling struct { @@ -109,6 +110,7 @@ type btHeaderMarshaling struct { BaseFeePerGas *math.HexOrDecimal256 BlobGasUsed *math.HexOrDecimal64 ExcessBlobGas *math.HexOrDecimal64 + SlotNumber *math.HexOrDecimal64 } func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *tracing.Hooks, postCheck func(error, *core.BlockChain)) (result error) { @@ -159,9 +161,9 @@ func (t *BlockTest) Run(snapshotter bool, scheme string, witness bool, tracer *t Preimages: true, TxLookupLimit: -1, // disable tx indexing VmConfig: vm.Config{ - Tracer: tracer, - StatelessSelfValidation: witness, + Tracer: tracer, }, + StatelessSelfValidation: witness, } if snapshotter { options.SnapshotLimit = 1 @@ -343,6 +345,9 @@ func validateHeader(h *btHeader, h2 *types.Header) error { if !reflect.DeepEqual(h.ParentBeaconBlockRoot, h2.ParentBeaconRoot) { return fmt.Errorf("parentBeaconBlockRoot: want: %v have: %v", h.ParentBeaconBlockRoot, h2.ParentBeaconRoot) } + if !reflect.DeepEqual(h.SlotNumber, h2.SlotNumber) { + return fmt.Errorf("slotNumber: want: %v have: %v", h.SlotNumber, h2.SlotNumber) + } return nil } diff --git a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go index 3baff33dcc..bcceaff383 100644 --- a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go +++ b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go @@ -78,6 +78,7 @@ func fuzz(input []byte) int { rand := rand.New(rand.NewSource(0x3a29)) // Same used in package tests!!! f := fetcher.NewTxFetcherForTests( + nil, func(common.Hash, byte) error { return nil }, func(txs []*types.Transaction) []error { return make([]error, len(txs)) diff --git a/tests/gen_btheader.go b/tests/gen_btheader.go index 80ad89e03b..eb6d9a8271 100644 --- a/tests/gen_btheader.go +++ b/tests/gen_btheader.go @@ -38,6 +38,7 @@ func (b btHeader) MarshalJSON() ([]byte, error) { BlobGasUsed *math.HexOrDecimal64 ExcessBlobGas *math.HexOrDecimal64 ParentBeaconBlockRoot *common.Hash + SlotNumber *math.HexOrDecimal64 } var enc btHeader enc.Bloom = b.Bloom @@ -61,6 +62,7 @@ func (b btHeader) MarshalJSON() ([]byte, error) { enc.BlobGasUsed = (*math.HexOrDecimal64)(b.BlobGasUsed) enc.ExcessBlobGas = (*math.HexOrDecimal64)(b.ExcessBlobGas) enc.ParentBeaconBlockRoot = b.ParentBeaconBlockRoot + enc.SlotNumber = (*math.HexOrDecimal64)(b.SlotNumber) return json.Marshal(&enc) } @@ -88,6 +90,7 @@ func (b *btHeader) UnmarshalJSON(input []byte) error { BlobGasUsed *math.HexOrDecimal64 ExcessBlobGas *math.HexOrDecimal64 ParentBeaconBlockRoot *common.Hash + SlotNumber *math.HexOrDecimal64 } var dec btHeader if err := json.Unmarshal(input, &dec); err != nil { @@ -156,5 +159,8 @@ func (b *btHeader) UnmarshalJSON(input []byte) error { if dec.ParentBeaconBlockRoot != nil { b.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot } + if dec.SlotNumber != nil { + b.SlotNumber = (*uint64)(dec.SlotNumber) + } return nil } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 7525081f84..1dd1bf6a04 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -342,9 +342,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh } // Execute the message. snapshot := st.StateDB.Snapshot() - gaspool := new(core.GasPool) - gaspool.AddGas(block.GasLimit()) - vmRet, err := core.ApplyMessage(evm, msg, gaspool) + vmRet, err := core.ApplyMessage(evm, msg, core.NewGasPool(block.GasLimit())) if err != nil { st.StateDB.RevertToSnapshot(snapshot) if tracer := evm.Config.Tracer; tracer != nil && tracer.OnTxEnd != nil { @@ -546,7 +544,7 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, snapshotter bo } snaps, _ = snapshot.New(snapconfig, db, triedb, root) } - sdb = state.NewDatabase(triedb, snaps) + sdb = state.NewDatabase(triedb, nil).WithSnapshot(snaps) statedb, _ = state.New(root, sdb) return StateTestState{statedb, triedb, snaps} } diff --git a/trie/bintrie/binary_node.go b/trie/bintrie/binary_node.go index 690489b2aa..a7392ec958 100644 --- a/trie/bintrie/binary_node.go +++ b/trie/bintrie/binary_node.go @@ -90,8 +90,18 @@ func SerializeNode(node BinaryNode) []byte { var invalidSerializedLength = errors.New("invalid serialized node length") -// DeserializeNode deserializes a binary trie node from a byte slice. +// DeserializeNode deserializes a binary trie node from a byte slice. The +// hash will be recomputed from the deserialized data. func DeserializeNode(serialized []byte, depth int) (BinaryNode, error) { + return deserializeNode(serialized, depth, common.Hash{}, true) +} + +// DeserializeNodeWithHash deserializes a binary trie node from a byte slice, using the provided hash. +func DeserializeNodeWithHash(serialized []byte, depth int, hn common.Hash) (BinaryNode, error) { + return deserializeNode(serialized, depth, hn, false) +} + +func deserializeNode(serialized []byte, depth int, hn common.Hash, mustRecompute bool) (BinaryNode, error) { if len(serialized) == 0 { return Empty{}, nil } @@ -102,9 +112,11 @@ func DeserializeNode(serialized []byte, depth int) (BinaryNode, error) { return nil, invalidSerializedLength } return &InternalNode{ - depth: depth, - left: HashedNode(common.BytesToHash(serialized[1:33])), - right: HashedNode(common.BytesToHash(serialized[33:65])), + depth: depth, + left: HashedNode(common.BytesToHash(serialized[1:33])), + right: HashedNode(common.BytesToHash(serialized[33:65])), + hash: hn, + mustRecompute: mustRecompute, }, nil case nodeTypeStem: if len(serialized) < 64 { @@ -124,9 +136,11 @@ func DeserializeNode(serialized []byte, depth int) (BinaryNode, error) { } } return &StemNode{ - Stem: serialized[NodeTypeBytes : NodeTypeBytes+StemSize], - Values: values[:], - depth: depth, + Stem: serialized[NodeTypeBytes : NodeTypeBytes+StemSize], + Values: values[:], + depth: depth, + hash: hn, + mustRecompute: mustRecompute, }, nil default: return nil, errors.New("invalid node type") diff --git a/trie/bintrie/empty.go b/trie/bintrie/empty.go index 7cfe373b35..252146a4a7 100644 --- a/trie/bintrie/empty.go +++ b/trie/bintrie/empty.go @@ -32,9 +32,10 @@ func (e Empty) Insert(key []byte, value []byte, _ NodeResolverFn, depth int) (Bi var values [256][]byte values[key[31]] = value return &StemNode{ - Stem: slices.Clone(key[:31]), - Values: values[:], - depth: depth, + Stem: slices.Clone(key[:31]), + Values: values[:], + depth: depth, + mustRecompute: true, }, nil } @@ -53,9 +54,10 @@ func (e Empty) GetValuesAtStem(_ []byte, _ NodeResolverFn) ([][]byte, error) { func (e Empty) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolverFn, depth int) (BinaryNode, error) { return &StemNode{ - Stem: slices.Clone(key[:31]), - Values: values, - depth: depth, + Stem: slices.Clone(key[:31]), + Values: values, + depth: depth, + mustRecompute: true, }, nil } diff --git a/trie/bintrie/hashed_node.go b/trie/bintrie/hashed_node.go index e4d8c2e7ac..e44c6d1e8a 100644 --- a/trie/bintrie/hashed_node.go +++ b/trie/bintrie/hashed_node.go @@ -64,7 +64,7 @@ func (h HashedNode) InsertValuesAtStem(stem []byte, values [][]byte, resolver No } // Step 3: Deserialize the resolved data into a concrete node - node, err := DeserializeNode(data, depth) + node, err := DeserializeNodeWithHash(data, depth, common.Hash(h)) if err != nil { return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) } diff --git a/trie/bintrie/hasher.go b/trie/bintrie/hasher.go new file mode 100644 index 0000000000..b81c145723 --- /dev/null +++ b/trie/bintrie/hasher.go @@ -0,0 +1,39 @@ +// Copyright 2026 go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package bintrie + +import ( + "crypto/sha256" + "hash" + "sync" +) + +var sha256Pool = sync.Pool{ + New: func() any { + return sha256.New() + }, +} + +func newSha256() hash.Hash { + h := sha256Pool.Get().(hash.Hash) + h.Reset() + return h +} + +func returnSha256(h hash.Hash) { + sha256Pool.Put(h) +} diff --git a/trie/bintrie/internal_node.go b/trie/bintrie/internal_node.go index 0a7bece521..7ad76aa9db 100644 --- a/trie/bintrie/internal_node.go +++ b/trie/bintrie/internal_node.go @@ -17,7 +17,6 @@ package bintrie import ( - "crypto/sha256" "errors" "fmt" @@ -40,6 +39,9 @@ func keyToPath(depth int, key []byte) ([]byte, error) { type InternalNode struct { left, right BinaryNode depth int + + mustRecompute bool // true if the hash needs to be recomputed + hash common.Hash // cached hash when mustRecompute == false } // GetValuesAtStem retrieves the group of values located at the given stem key. @@ -59,7 +61,7 @@ func (bt *InternalNode) GetValuesAtStem(stem []byte, resolver NodeResolverFn) ([ if err != nil { return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) } - node, err := DeserializeNode(data, bt.depth+1) + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) if err != nil { return nil, fmt.Errorf("GetValuesAtStem node deserialization error: %w", err) } @@ -77,7 +79,7 @@ func (bt *InternalNode) GetValuesAtStem(stem []byte, resolver NodeResolverFn) ([ if err != nil { return nil, fmt.Errorf("GetValuesAtStem resolve error: %w", err) } - node, err := DeserializeNode(data, bt.depth+1) + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) if err != nil { return nil, fmt.Errorf("GetValuesAtStem node deserialization error: %w", err) } @@ -108,15 +110,22 @@ func (bt *InternalNode) Insert(key []byte, value []byte, resolver NodeResolverFn // Copy creates a deep copy of the node. func (bt *InternalNode) Copy() BinaryNode { return &InternalNode{ - left: bt.left.Copy(), - right: bt.right.Copy(), - depth: bt.depth, + left: bt.left.Copy(), + right: bt.right.Copy(), + depth: bt.depth, + mustRecompute: bt.mustRecompute, + hash: bt.hash, } } // Hash returns the hash of the node. func (bt *InternalNode) Hash() common.Hash { - h := sha256.New() + if !bt.mustRecompute { + return bt.hash + } + + h := newSha256() + defer returnSha256(h) if bt.left != nil { h.Write(bt.left.Hash().Bytes()) } else { @@ -127,7 +136,9 @@ func (bt *InternalNode) Hash() common.Hash { } else { h.Write(zero[:]) } - return common.BytesToHash(h.Sum(nil)) + bt.hash = common.BytesToHash(h.Sum(nil)) + bt.mustRecompute = false + return bt.hash } // InsertValuesAtStem inserts a full value group at the given stem in the internal node. @@ -149,7 +160,7 @@ func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolve if err != nil { return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) } - node, err := DeserializeNode(data, bt.depth+1) + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) if err != nil { return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) } @@ -157,6 +168,7 @@ func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolve } bt.left, err = bt.left.InsertValuesAtStem(stem, values, resolver, depth+1) + bt.mustRecompute = true return bt, err } @@ -173,7 +185,7 @@ func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolve if err != nil { return nil, fmt.Errorf("InsertValuesAtStem resolve error: %w", err) } - node, err := DeserializeNode(data, bt.depth+1) + node, err := DeserializeNodeWithHash(data, bt.depth+1, common.Hash(hn)) if err != nil { return nil, fmt.Errorf("InsertValuesAtStem node deserialization error: %w", err) } @@ -181,6 +193,7 @@ func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolve } bt.right, err = bt.right.InsertValuesAtStem(stem, values, resolver, depth+1) + bt.mustRecompute = true return bt, err } diff --git a/trie/bintrie/internal_node_test.go b/trie/bintrie/internal_node_test.go index 158d8b7147..69097483fd 100644 --- a/trie/bintrie/internal_node_test.go +++ b/trie/bintrie/internal_node_test.go @@ -239,6 +239,7 @@ func TestInternalNodeHash(t *testing.T) { // Changing a child should change the hash node.left = HashedNode(common.HexToHash("0x3333")) + node.mustRecompute = true hash3 := node.Hash() if hash1 == hash3 { t.Error("Hash didn't change after modifying left child") @@ -246,9 +247,10 @@ func TestInternalNodeHash(t *testing.T) { // Test with nil children (should use zero hash) nodeWithNil := &InternalNode{ - depth: 0, - left: nil, - right: HashedNode(common.HexToHash("0x4444")), + depth: 0, + left: nil, + right: HashedNode(common.HexToHash("0x4444")), + mustRecompute: true, } hashWithNil := nodeWithNil.Hash() if hashWithNil == (common.Hash{}) { diff --git a/trie/bintrie/iterator.go b/trie/bintrie/iterator.go index 9b863ed1e3..917f82efc9 100644 --- a/trie/bintrie/iterator.go +++ b/trie/bintrie/iterator.go @@ -123,7 +123,7 @@ func (it *binaryNodeIterator) Next(descend bool) bool { if err != nil { panic(err) } - it.current, err = DeserializeNode(data, len(it.stack)-1) + it.current, err = DeserializeNodeWithHash(data, len(it.stack)-1, common.Hash(node)) if err != nil { panic(err) } diff --git a/trie/bintrie/key_encoding.go b/trie/bintrie/key_encoding.go index 94a22d52d0..c009f1529f 100644 --- a/trie/bintrie/key_encoding.go +++ b/trie/bintrie/key_encoding.go @@ -18,7 +18,6 @@ package bintrie import ( "bytes" - "crypto/sha256" "github.com/ethereum/go-ethereum/common" "github.com/holiman/uint256" @@ -47,13 +46,27 @@ var ( ) func GetBinaryTreeKey(addr common.Address, key []byte) []byte { - hasher := sha256.New() + return getBinaryTreeKey(addr, key, false) +} + +func getBinaryTreeKey(addr common.Address, offset []byte, overflow bool) []byte { + hasher := newSha256() + defer returnSha256(hasher) hasher.Write(zeroHash[:12]) hasher.Write(addr[:]) - hasher.Write(key[:31]) - hasher.Write([]byte{0}) + var buf [32]byte + // key is big endian, hashed value is little endian + for i := range offset[:31] { + buf[i] = offset[30-i] + } + if overflow { + // Overflow detected when adding MAIN_STORAGE_OFFSET, + // reporting it in the shifter 32 byte value. + buf[31] = 1 + } + hasher.Write(buf[:]) k := hasher.Sum(nil) - k[31] = key[31] + k[31] = offset[31] return k } @@ -69,24 +82,29 @@ func GetBinaryTreeKeyCodeHash(addr common.Address) []byte { return GetBinaryTreeKey(addr, k[:]) } -func GetBinaryTreeKeyStorageSlot(address common.Address, key []byte) []byte { - var k [32]byte +func GetBinaryTreeKeyStorageSlot(address common.Address, slotnum []byte) []byte { + var offset [32]byte // Case when the key belongs to the account header - if bytes.Equal(key[:31], zeroHash[:31]) && key[31] < 64 { - k[31] = 64 + key[31] - return GetBinaryTreeKey(address, k[:]) + if bytes.Equal(slotnum[:31], zeroHash[:31]) && slotnum[31] < 64 { + offset[31] = 64 + slotnum[31] + return GetBinaryTreeKey(address, offset[:]) } - // Set the main storage offset - // note that the first 64 bytes of the main offset storage - // are unreachable, which is consistent with the spec and - // what verkle does. - k[0] = 1 // 1 << 248 - copy(k[1:], key[:31]) - k[31] = key[31] + // Set the main storage offset offset = MAIN_STORAGE_OFFSET + slotnum + // * Note that MAIN_STORAGE_OFFSET is 1 << 248, so the number + // can overflow into a 33rd byte, but since the value is + // shifted by one byte in getBinaryTreeKey, this only takes + // note of the overflow, and the value will be added after + // the shift, in order to avoid allocating an extra byte. + // * Note that the first 64 bytes of the main offset storage + // are unreachable, which is consistent with the spec. + // * Note that `slotnum` is big-endian + overflow := slotnum[0] == 255 + copy(offset[:], slotnum) + offset[0] += 1 // 1 << 248, handle overflow out of band - return GetBinaryTreeKey(address, k[:]) + return getBinaryTreeKey(address, offset[:], overflow) } func GetBinaryTreeKeyCodeChunk(address common.Address, chunknr *uint256.Int) []byte { diff --git a/trie/bintrie/stem_node.go b/trie/bintrie/stem_node.go index 60856b42ce..3f69261d62 100644 --- a/trie/bintrie/stem_node.go +++ b/trie/bintrie/stem_node.go @@ -18,7 +18,6 @@ package bintrie import ( "bytes" - "crypto/sha256" "errors" "fmt" "slices" @@ -31,6 +30,9 @@ type StemNode struct { Stem []byte // Stem path to get to StemNodeWidth values Values [][]byte // All values, indexed by the last byte of the key. depth int // Depth of the node + + mustRecompute bool // true if the hash needs to be recomputed + hash common.Hash // cached hash when mustRecompute == false } // Get retrieves the value for the given key. @@ -43,7 +45,7 @@ func (bt *StemNode) Insert(key []byte, value []byte, _ NodeResolverFn, depth int if !bytes.Equal(bt.Stem, key[:StemSize]) { bitStem := bt.Stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 - n := &InternalNode{depth: bt.depth} + n := &InternalNode{depth: bt.depth, mustRecompute: true} bt.depth++ var child, other *BinaryNode if bitStem == 0 { @@ -68,9 +70,10 @@ func (bt *StemNode) Insert(key []byte, value []byte, _ NodeResolverFn, depth int var values [StemNodeWidth][]byte values[key[StemSize]] = value *other = &StemNode{ - Stem: slices.Clone(key[:StemSize]), - Values: values[:], - depth: depth + 1, + Stem: slices.Clone(key[:StemSize]), + Values: values[:], + depth: depth + 1, + mustRecompute: true, } } return n, nil @@ -79,6 +82,7 @@ func (bt *StemNode) Insert(key []byte, value []byte, _ NodeResolverFn, depth int return bt, errors.New("invalid insertion: value length") } bt.Values[key[StemSize]] = value + bt.mustRecompute = true return bt, nil } @@ -89,9 +93,11 @@ func (bt *StemNode) Copy() BinaryNode { values[i] = slices.Clone(v) } return &StemNode{ - Stem: slices.Clone(bt.Stem), - Values: values[:], - depth: bt.depth, + Stem: slices.Clone(bt.Stem), + Values: values[:], + depth: bt.depth, + hash: bt.hash, + mustRecompute: bt.mustRecompute, } } @@ -102,15 +108,22 @@ func (bt *StemNode) GetHeight() int { // Hash returns the hash of the node. func (bt *StemNode) Hash() common.Hash { - var data [StemNodeWidth]common.Hash - for i, v := range bt.Values { - if v != nil { - h := sha256.Sum256(v) - data[i] = common.BytesToHash(h[:]) - } + if !bt.mustRecompute { + return bt.hash } - h := sha256.New() + var data [StemNodeWidth]common.Hash + h := newSha256() + defer returnSha256(h) + for i, v := range bt.Values { + if v != nil { + h.Reset() + h.Write(v) + h.Sum(data[i][:0]) + } + } + h.Reset() + for level := 1; level <= 8; level++ { for i := range StemNodeWidth / (1 << level) { h.Reset() @@ -130,7 +143,9 @@ func (bt *StemNode) Hash() common.Hash { h.Write(bt.Stem) h.Write([]byte{0}) h.Write(data[0][:]) - return common.BytesToHash(h.Sum(nil)) + bt.hash = common.BytesToHash(h.Sum(nil)) + bt.mustRecompute = false + return bt.hash } // CollectNodes collects all child nodes at a given path, and flushes it @@ -154,7 +169,7 @@ func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolv if !bytes.Equal(bt.Stem, key[:StemSize]) { bitStem := bt.Stem[bt.depth/8] >> (7 - (bt.depth % 8)) & 1 - n := &InternalNode{depth: bt.depth} + n := &InternalNode{depth: bt.depth, mustRecompute: true} bt.depth++ var child, other *BinaryNode if bitStem == 0 { @@ -177,9 +192,10 @@ func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolv *other = Empty{} } else { *other = &StemNode{ - Stem: slices.Clone(key[:StemSize]), - Values: values, - depth: n.depth + 1, + Stem: slices.Clone(key[:StemSize]), + Values: values, + depth: n.depth + 1, + mustRecompute: true, } } return n, nil @@ -189,6 +205,7 @@ func (bt *StemNode) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolv for i, v := range values { if v != nil { bt.Values[i] = v + bt.mustRecompute = true } } return bt, nil diff --git a/trie/bintrie/stem_node_test.go b/trie/bintrie/stem_node_test.go index d8d6844427..92c1b49e02 100644 --- a/trie/bintrie/stem_node_test.go +++ b/trie/bintrie/stem_node_test.go @@ -220,6 +220,7 @@ func TestStemNodeHash(t *testing.T) { // Changing a value should change the hash node.Values[1] = common.HexToHash("0x0202").Bytes() + node.mustRecompute = true hash3 := node.Hash() if hash1 == hash3 { t.Error("Hash didn't change after modifying values") diff --git a/trie/bintrie/trie.go b/trie/bintrie/trie.go index ecdd002331..6c29239a87 100644 --- a/trie/bintrie/trie.go +++ b/trie/bintrie/trie.go @@ -143,7 +143,7 @@ func NewBinaryTrie(root common.Hash, db database.NodeDatabase) (*BinaryTrie, err if err != nil { return nil, err } - node, err := DeserializeNode(blob, 0) + node, err := DeserializeNodeWithHash(blob, 0, root) if err != nil { return nil, err } @@ -384,7 +384,7 @@ func (t *BinaryTrie) UpdateContractCode(addr common.Address, codeHash common.Has if groupOffset == 0 /* start of new group */ || chunknr == 0 /* first chunk in header group */ { values = make([][]byte, StemNodeWidth) var offset [HashSize]byte - binary.LittleEndian.PutUint64(offset[24:], chunknr+128) + binary.BigEndian.PutUint64(offset[24:], chunknr+128) key = GetBinaryTreeKey(addr, offset[:]) } values[groupOffset] = chunks[i : i+HashSize] diff --git a/trie/inspect.go b/trie/inspect.go new file mode 100644 index 0000000000..d7ca9ace63 --- /dev/null +++ b/trie/inspect.go @@ -0,0 +1,930 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "bufio" + "bytes" + "cmp" + "container/heap" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/tablewriter" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/triedb/database" + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" +) + +const ( + inspectDumpRecordSize = 32 + trieStatLevels*(3*4+8) + inspectDefaultTopN = 10 + inspectParallelism = int64(16) +) + +// inspector is used by the inner inspect function to coordinate across threads. +type inspector struct { + triedb database.NodeDatabase + root common.Hash + + config *InspectConfig + accountStat *LevelStats + + sem *semaphore.Weighted + + // Pass 1: dump file writer. + dumpMu sync.Mutex + dumpBuf *bufio.Writer + dumpFile *os.File + storageRecordsWritten atomic.Uint64 + + errMu sync.Mutex + err error +} + +// InspectConfig is a set of options to control inspection and format the output. +// TopN determines the maximum number of entries retained for each top-list. +// Path controls optional JSON output. DumpPath controls the pass-1 dump location. +type InspectConfig struct { + NoStorage bool + TopN int + Path string + DumpPath string +} + +// Inspect walks the trie with the given root and records the number and type of +// nodes at each depth. Storage trie stats are streamed to disk in fixed-size +// records, then summarized in a second pass. +func Inspect(triedb database.NodeDatabase, root common.Hash, config *InspectConfig) error { + trie, err := New(TrieID(root), triedb) + if err != nil { + return fmt.Errorf("fail to open trie %s: %w", root, err) + } + config = normalizeInspectConfig(config) + + dumpFile, err := os.OpenFile(config.DumpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("failed to create trie dump %s: %w", config.DumpPath, err) + } + in := inspector{ + triedb: triedb, + root: root, + config: config, + accountStat: NewLevelStats(), + sem: semaphore.NewWeighted(inspectParallelism), + dumpBuf: bufio.NewWriterSize(dumpFile, 1<<20), + dumpFile: dumpFile, + } + + // Start progress reporter + start := time.Now() + done := make(chan struct{}) + go func() { + ticker := time.NewTicker(8 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + accountNodes := in.accountStat.TotalNodes() + storageRecords := in.storageRecordsWritten.Load() + log.Info("Inspecting trie", + "accountNodes", accountNodes, + "storageRecords", storageRecords, + "elapsed", common.PrettyDuration(time.Since(start))) + case <-done: + return + } + } + }() + + in.recordRootSize(trie, root, in.accountStat) + in.inspect(trie, trie.root, 0, []byte{}, in.accountStat) + + // inspect is synchronous: it waits for all spawned goroutines in its + // subtree before returning, so no additional wait is needed here. + + // Persist account trie stats as the sentinel record. + in.writeDumpRecord(common.Hash{}, in.accountStat) + if err := in.closeDump(); err != nil { + in.setError(err) + } + + // Stop progress reporter + close(done) + + if err := in.getError(); err != nil { + return err + } + return Summarize(config.DumpPath, config) +} + +// InspectContract inspects the on-disk footprint of a single contract. +// It reports snapshot storage (slot count + size) and storage trie node +// statistics (node type breakdown and per-depth distribution). +func InspectContract(triedb database.NodeDatabase, db ethdb.Database, stateRoot common.Hash, address common.Address) error { + // Resolve account from the state trie. + accountHash := crypto.Keccak256Hash(address.Bytes()) + accountTrie, err := New(TrieID(stateRoot), triedb) + if err != nil { + return fmt.Errorf("failed to open account trie: %w", err) + } + accountRLP, err := accountTrie.Get(crypto.Keccak256(address.Bytes())) + if err != nil { + return fmt.Errorf("failed to read account: %w", err) + } + if accountRLP == nil { + return fmt.Errorf("account not found: %s", address) + } + var account types.StateAccount + if err := rlp.DecodeBytes(accountRLP, &account); err != nil { + return fmt.Errorf("failed to decode account: %w", err) + } + if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash { + return fmt.Errorf("account %s has no storage", address) + } + + // Look up account snapshot. + accountData := rawdb.ReadAccountSnapshot(db, accountHash) + + // Run trie walk + snap iteration in parallel. + var ( + snapSlots atomic.Uint64 + snapSize atomic.Uint64 + g errgroup.Group + start = time.Now() + ) + + // Goroutine 1: Snapshot storage iteration. + g.Go(func() error { + prefix := append(rawdb.SnapshotStoragePrefix, accountHash.Bytes()...) + it := db.NewIterator(prefix, nil) + defer it.Release() + + for it.Next() { + if !bytes.HasPrefix(it.Key(), prefix) { + break + } + snapSlots.Add(1) + snapSize.Add(uint64(len(it.Key()) + len(it.Value()))) + } + return it.Error() + }) + + // Goroutine 2: Storage trie walk using the existing inspector. + storageStat := NewLevelStats() + g.Go(func() error { + owner := accountHash + storage, err := New(StorageTrieID(stateRoot, owner, account.Root), triedb) + if err != nil { + return fmt.Errorf("failed to open storage trie: %w", err) + } + in := &inspector{ + triedb: triedb, + root: stateRoot, + config: &InspectConfig{NoStorage: true}, + accountStat: NewLevelStats(), // unused, but needed by inspector + sem: semaphore.NewWeighted(inspectParallelism), + dumpBuf: bufio.NewWriter(io.Discard), + } + + in.recordRootSize(storage, account.Root, storageStat) + in.inspect(storage, storage.root, 0, []byte{}, storageStat) + + if err := in.getError(); err != nil { + return err + } + return nil + }) + + // Progress reporter. + done := make(chan struct{}) + go func() { + ticker := time.NewTicker(8 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + log.Info("Inspecting contract", + "snapSlots", snapSlots.Load(), + "trieNodes", storageStat.TotalNodes(), + "elapsed", common.PrettyDuration(time.Since(start))) + case <-done: + return + } + } + }() + + if err := g.Wait(); err != nil { + close(done) + return err + } + close(done) + + // Display results. + fmt.Printf("\n=== Contract Inspection: %s ===\n", address) + fmt.Printf("Account hash: %s\n\n", accountHash) + + if len(accountData) == 0 { + fmt.Println("Account snapshot: not found") + } else { + fmt.Printf("Account snapshot: %s\n", common.StorageSize(len(accountData))) + } + + fmt.Printf("Snapshot storage: %d slots (%s)\n", + snapSlots.Load(), common.StorageSize(snapSize.Load())) + + // Compute trie totals from LevelStats. + var trieTotal, trieSize uint64 + for i := 0; i < trieStatLevels; i++ { + short, full, value, size := storageStat.level[i].load() + trieTotal += short + full + value + trieSize += size + } + fmt.Printf("Storage trie: %d nodes (%s)\n", trieTotal, common.StorageSize(trieSize)) + + // Depth distribution table with node type columns. + fmt.Println("\nStorage Trie Depth Distribution:") + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{"Depth", "Short", "Full", "Value", "Nodes", "Size"}) + for i := 0; i < trieStatLevels; i++ { + short, full, value, size := storageStat.level[i].load() + total := short + full + value + if total == 0 && size == 0 { + continue + } + table.AppendBulk([][]string{{ + fmt.Sprint(i), + fmt.Sprint(short), + fmt.Sprint(full), + fmt.Sprint(value), + fmt.Sprint(total), + common.StorageSize(size).String(), + }}) + } + table.Render() + fmt.Print(b.String()) + + return nil +} + +func normalizeInspectConfig(config *InspectConfig) *InspectConfig { + if config == nil { + config = &InspectConfig{} + } + if config.TopN <= 0 { + config.TopN = inspectDefaultTopN + } + if config.DumpPath == "" { + config.DumpPath = "trie-dump.bin" + } + return config +} + +func (in *inspector) recordRootSize(trie *Trie, root common.Hash, stat *LevelStats) { + if root == (common.Hash{}) || root == types.EmptyRootHash { + return + } + blob := trie.prevalueTracer.Get(nil) + if len(blob) == 0 { + resolved, err := trie.reader.Node(nil, root) + if err != nil { + log.Error("Failed to read trie root for size accounting", "trie", trie.Hash(), "root", root, "err", err) + return + } + blob = resolved + } + stat.addSize(0, uint64(len(blob))) +} + +func (in *inspector) closeDump() error { + var ret error + if in.dumpBuf != nil { + if err := in.dumpBuf.Flush(); err != nil { + ret = errors.Join(ret, fmt.Errorf("failed to flush trie dump %s: %w", in.config.DumpPath, err)) + } + } + if in.dumpFile != nil { + if err := in.dumpFile.Close(); err != nil { + ret = errors.Join(ret, fmt.Errorf("failed to close trie dump %s: %w", in.config.DumpPath, err)) + } + } + return ret +} + +func (in *inspector) setError(err error) { + if err == nil { + return + } + in.errMu.Lock() + defer in.errMu.Unlock() + in.err = errors.Join(in.err, err) +} + +func (in *inspector) getError() error { + in.errMu.Lock() + defer in.errMu.Unlock() + return in.err +} + +func (in *inspector) hasError() bool { + return in.getError() != nil +} + +// trySpawn attempts to run fn in a new goroutine, bounded by the semaphore. +// If a slot is available the goroutine is started and tracked via wg; the +// caller must call wg.Wait() before reading any state written by fn. +// Returns false (and does not start a goroutine) when no slot is available. +func (in *inspector) trySpawn(wg *sync.WaitGroup, fn func()) bool { + if !in.sem.TryAcquire(1) { + return false + } + wg.Add(1) + go func() { + defer in.sem.Release(1) + defer wg.Done() + fn() + }() + return true +} + +func (in *inspector) writeDumpRecord(owner common.Hash, s *LevelStats) { + if in.hasError() { + return + } + var buf [inspectDumpRecordSize]byte + copy(buf[:32], owner[:]) + + off := 32 + for i := 0; i < trieStatLevels; i++ { + binary.LittleEndian.PutUint32(buf[off:], uint32(s.level[i].short.Load())) + off += 4 + binary.LittleEndian.PutUint32(buf[off:], uint32(s.level[i].full.Load())) + off += 4 + binary.LittleEndian.PutUint32(buf[off:], uint32(s.level[i].value.Load())) + off += 4 + binary.LittleEndian.PutUint64(buf[off:], s.level[i].size.Load()) + off += 8 + } + in.dumpMu.Lock() + _, err := in.dumpBuf.Write(buf[:]) + in.dumpMu.Unlock() + if err != nil { + in.setError(fmt.Errorf("failed writing trie dump record: %w", err)) + } + + // Increment counter for storage tries only (not for account trie) + if owner != (common.Hash{}) { + in.storageRecordsWritten.Add(1) + } +} + +// inspect walks the trie rooted at n and records node statistics into stat. +// It may spawn goroutines for subtrees, but always waits for them before +// returning — the caller sees a fully-populated stat when inspect returns. +func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, stat *LevelStats) { + if n == nil { + return + } + + // wg tracks goroutines spawned by this call so we can wait for them + // before returning. This guarantees stat is complete when we return, + // which is critical for storage tries that write their dump record + // immediately after inspect returns. + var wg sync.WaitGroup + + // Four types of nodes can be encountered: + // - short: extend path with key, inspect single value. + // - full: inspect all 17 children, spin up new threads when possible. + // - hash: need to resolve node from disk, retry inspect on result. + // - value: if account, begin inspecting storage trie. + switch n := (n).(type) { + case *shortNode: + nextPath := slices.Concat(path, n.Key) + in.inspect(trie, n.Val, height+1, nextPath, stat) + case *fullNode: + for idx, child := range n.Children { + if child == nil { + continue + } + childPath := slices.Concat(path, []byte{byte(idx)}) + childNode := child + if in.trySpawn(&wg, func() { + in.inspect(trie, childNode, height+1, childPath, stat) + }) { + continue + } + in.inspect(trie, childNode, height+1, childPath, stat) + } + case hashNode: + blob, err := trie.reader.Node(path, common.BytesToHash(n)) + if err != nil { + log.Error("Failed to resolve HashNode", "err", err, "trie", trie.Hash(), "height", height+1, "path", path) + return + } + stat.addSize(height, uint64(len(blob))) + resolved := mustDecodeNode(n, blob) + in.inspect(trie, resolved, height, path, stat) + + // Return early here so this level isn't recorded twice. + return + case valueNode: + if !hasTerm(path) { + break + } + var account types.StateAccount + if err := rlp.Decode(bytes.NewReader(n), &account); err != nil { + // Not an account value. + break + } + if account.Root == (common.Hash{}) || account.Root == types.EmptyRootHash { + // Account is empty, nothing further to inspect. + break + } + + if !in.config.NoStorage { + owner := common.BytesToHash(hexToCompact(path)) + storage, err := New(StorageTrieID(in.root, owner, account.Root), in.triedb) + if err != nil { + log.Error("Failed to open account storage trie", "node", n, "error", err, "height", height, "path", common.Bytes2Hex(path)) + break + } + storageStat := NewLevelStats() + run := func() { + in.recordRootSize(storage, account.Root, storageStat) + in.inspect(storage, storage.root, 0, []byte{}, storageStat) + in.writeDumpRecord(owner, storageStat) + } + if in.trySpawn(&wg, run) { + break + } + run() + } + default: + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) + } + + // Wait for all goroutines spawned at this level before recording + // the current node. This ensures the entire subtree is counted + // before this call returns. + wg.Wait() + + // Record stats for current height. + stat.add(n, height) +} + +// Summarize performs pass 2 over a trie dump and reports account stats, +// aggregate storage statistics, and top-N rankings. +func Summarize(dumpPath string, config *InspectConfig) error { + config = normalizeInspectConfig(config) + if dumpPath == "" { + dumpPath = config.DumpPath + } + if dumpPath == "" { + return errors.New("missing dump path") + } + file, err := os.Open(dumpPath) + if err != nil { + return fmt.Errorf("failed to open trie dump %s: %w", dumpPath, err) + } + defer file.Close() + + if info, err := file.Stat(); err == nil { + if info.Size()%inspectDumpRecordSize != 0 { + return fmt.Errorf("invalid trie dump size %d (not a multiple of %d)", info.Size(), inspectDumpRecordSize) + } + } + + depthTop := newStorageStatsTopN(config.TopN, compareStorageStatsByDepth) + totalTop := newStorageStatsTopN(config.TopN, compareStorageStatsByTotal) + valueTop := newStorageStatsTopN(config.TopN, compareStorageStatsByValue) + + summary := &inspectSummary{} + reader := bufio.NewReaderSize(file, 1<<20) + var buf [inspectDumpRecordSize]byte + + for { + _, err := io.ReadFull(reader, buf[:]) + if errors.Is(err, io.EOF) { + break + } + if errors.Is(err, io.ErrUnexpectedEOF) { + return fmt.Errorf("truncated trie dump %s", dumpPath) + } + if err != nil { + return fmt.Errorf("failed reading trie dump %s: %w", dumpPath, err) + } + + record := decodeDumpRecord(buf[:]) + snapshot := newStorageStats(record.Owner, record.Levels) + if record.Owner == (common.Hash{}) { + summary.Account = snapshot + continue + } + summary.StorageCount++ + summary.DepthHistogram[snapshot.MaxDepth]++ + for i := 0; i < trieStatLevels; i++ { + summary.StorageLevels[i].Short += record.Levels[i].Short + summary.StorageLevels[i].Full += record.Levels[i].Full + summary.StorageLevels[i].Value += record.Levels[i].Value + summary.StorageLevels[i].Size += record.Levels[i].Size + } + + depthTop.TryInsert(snapshot) + totalTop.TryInsert(snapshot) + valueTop.TryInsert(snapshot) + } + if summary.Account == nil { + return fmt.Errorf("dump file %s does not contain the account trie sentinel record", dumpPath) + } + for i := 0; i < trieStatLevels; i++ { + summary.StorageTotals.Short += summary.StorageLevels[i].Short + summary.StorageTotals.Full += summary.StorageLevels[i].Full + summary.StorageTotals.Value += summary.StorageLevels[i].Value + summary.StorageTotals.Size += summary.StorageLevels[i].Size + } + summary.TopByDepth = depthTop.Sorted() + summary.TopByTotalNodes = totalTop.Sorted() + summary.TopByValueNodes = valueTop.Sorted() + + if config.Path != "" { + return summary.writeJSON(config.Path) + } + summary.display() + return nil +} + +type dumpRecord struct { + Owner common.Hash + Levels [trieStatLevels]jsonLevel +} + +func decodeDumpRecord(raw []byte) dumpRecord { + var ( + record dumpRecord + off = 32 + ) + copy(record.Owner[:], raw[:32]) + for i := 0; i < trieStatLevels; i++ { + record.Levels[i] = jsonLevel{ + Short: uint64(binary.LittleEndian.Uint32(raw[off:])), + Full: uint64(binary.LittleEndian.Uint32(raw[off+4:])), + Value: uint64(binary.LittleEndian.Uint32(raw[off+8:])), + Size: binary.LittleEndian.Uint64(raw[off+12:]), + } + off += 20 + } + return record +} + +type storageStats struct { + Owner common.Hash + Levels [trieStatLevels]jsonLevel + Summary jsonLevel + MaxDepth int + TotalNodes uint64 + TotalSize uint64 +} + +func newStorageStats(owner common.Hash, levels [trieStatLevels]jsonLevel) *storageStats { + snapshot := &storageStats{Owner: owner, Levels: levels} + for i := 0; i < trieStatLevels; i++ { + level := levels[i] + if level.Short != 0 || level.Full != 0 || level.Value != 0 { + snapshot.MaxDepth = i + } + snapshot.Summary.Short += level.Short + snapshot.Summary.Full += level.Full + snapshot.Summary.Value += level.Value + snapshot.Summary.Size += level.Size + } + snapshot.TotalNodes = snapshot.Summary.Short + snapshot.Summary.Full + snapshot.Summary.Value + snapshot.TotalSize = snapshot.Summary.Size + return snapshot +} + +func trimLevels(levels [trieStatLevels]jsonLevel) []jsonLevel { + n := len(levels) + for n > 0 && levels[n-1] == (jsonLevel{}) { + n-- + } + return levels[:n] +} + +func (s *storageStats) MarshalJSON() ([]byte, error) { + type jsonStorageSnapshot struct { + Owner common.Hash `json:"Owner"` + MaxDepth int `json:"MaxDepth"` + TotalNodes uint64 `json:"TotalNodes"` + TotalSize uint64 `json:"TotalSize"` + ValueNodes uint64 `json:"ValueNodes"` + Levels []jsonLevel `json:"Levels"` + Summary jsonLevel `json:"Summary"` + } + return json.Marshal(jsonStorageSnapshot{ + Owner: s.Owner, + MaxDepth: s.MaxDepth, + TotalNodes: s.TotalNodes, + TotalSize: s.TotalSize, + ValueNodes: s.Summary.Value, + Levels: trimLevels(s.Levels), + Summary: s.Summary, + }) +} + +func (s *storageStats) toLevelStats() *LevelStats { + stats := NewLevelStats() + for i := 0; i < trieStatLevels; i++ { + stats.level[i].short.Store(s.Levels[i].Short) + stats.level[i].full.Store(s.Levels[i].Full) + stats.level[i].value.Store(s.Levels[i].Value) + stats.level[i].size.Store(s.Levels[i].Size) + } + return stats +} + +type storageStatsCompare func(a, b *storageStats) int + +type storageStatsTopN struct { + limit int + cmp storageStatsCompare + heap storageStatsHeap +} + +type storageStatsHeap struct { + items []*storageStats + cmp storageStatsCompare +} + +func (h storageStatsHeap) Len() int { return len(h.items) } + +func (h storageStatsHeap) Less(i, j int) bool { + // Keep the weakest entry at the root (min-heap semantics). + return h.cmp(h.items[i], h.items[j]) < 0 +} + +func (h storageStatsHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] } + +func (h *storageStatsHeap) Push(x any) { + h.items = append(h.items, x.(*storageStats)) +} + +func (h *storageStatsHeap) Pop() any { + item := h.items[len(h.items)-1] + h.items = h.items[:len(h.items)-1] + return item +} + +func newStorageStatsTopN(limit int, cmp storageStatsCompare) *storageStatsTopN { + h := storageStatsHeap{cmp: cmp} + heap.Init(&h) + return &storageStatsTopN{limit: limit, cmp: cmp, heap: h} +} + +func (t *storageStatsTopN) TryInsert(item *storageStats) { + if t.limit <= 0 { + return + } + if t.heap.Len() < t.limit { + heap.Push(&t.heap, item) + return + } + if t.cmp(item, t.heap.items[0]) <= 0 { + return + } + heap.Pop(&t.heap) + heap.Push(&t.heap, item) +} + +func (t *storageStatsTopN) Sorted() []*storageStats { + out := append([]*storageStats(nil), t.heap.items...) + sort.Slice(out, func(i, j int) bool { return t.cmp(out[i], out[j]) > 0 }) + return out +} + +func compareStorageStatsByDepth(a, b *storageStats) int { + return cmp.Or( + cmp.Compare(a.MaxDepth, b.MaxDepth), + cmp.Compare(a.TotalNodes, b.TotalNodes), + cmp.Compare(a.Summary.Value, b.Summary.Value), + bytes.Compare(a.Owner[:], b.Owner[:]), + ) +} + +func compareStorageStatsByTotal(a, b *storageStats) int { + return cmp.Or( + cmp.Compare(a.TotalNodes, b.TotalNodes), + cmp.Compare(a.MaxDepth, b.MaxDepth), + cmp.Compare(a.Summary.Value, b.Summary.Value), + bytes.Compare(a.Owner[:], b.Owner[:]), + ) +} + +func compareStorageStatsByValue(a, b *storageStats) int { + return cmp.Or( + cmp.Compare(a.Summary.Value, b.Summary.Value), + cmp.Compare(a.MaxDepth, b.MaxDepth), + cmp.Compare(a.TotalNodes, b.TotalNodes), + bytes.Compare(a.Owner[:], b.Owner[:]), + ) +} + +type inspectSummary struct { + Account *storageStats + StorageCount uint64 + StorageTotals jsonLevel + StorageLevels [trieStatLevels]jsonLevel + DepthHistogram [trieStatLevels]uint64 + TopByDepth []*storageStats + TopByTotalNodes []*storageStats + TopByValueNodes []*storageStats +} + +func (s *inspectSummary) display() { + s.displayCombinedDepthTable() + s.Account.toLevelStats().display("Accounts trie") + fmt.Println("Storage trie aggregate summary") + fmt.Printf("Total storage tries: %d\n", s.StorageCount) + totalNodes := s.StorageTotals.Short + s.StorageTotals.Full + s.StorageTotals.Value + fmt.Printf("Total nodes: %d\n", totalNodes) + fmt.Printf("Total size: %s\n", common.StorageSize(s.StorageTotals.Size)) + fmt.Printf(" Short nodes: %d\n", s.StorageTotals.Short) + fmt.Printf(" Full nodes: %d\n", s.StorageTotals.Full) + fmt.Printf(" Value nodes: %d\n", s.StorageTotals.Value) + + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{"Max Depth", "Storage Tries"}) + for i, count := range s.DepthHistogram { + table.AppendBulk([][]string{{fmt.Sprint(i), fmt.Sprint(count)}}) + } + table.Render() + fmt.Print(b.String()) + fmt.Println() + + s.displayTop("Top storage tries by max depth", s.TopByDepth) + s.displayTop("Top storage tries by total node count", s.TopByTotalNodes) + s.displayTop("Top storage tries by value (slot) count", s.TopByValueNodes) +} + +func (s *inspectSummary) displayCombinedDepthTable() { + accountTotal := s.Account.Summary.Short + s.Account.Summary.Full + s.Account.Summary.Value + storageTotal := s.StorageTotals.Short + s.StorageTotals.Full + s.StorageTotals.Value + accountTotalSize := s.Account.Summary.Size + storageTotalSize := s.StorageTotals.Size + + fmt.Println("Trie Depth Distribution") + fmt.Printf("Account Trie: %d nodes (%s)\n", accountTotal, common.StorageSize(accountTotalSize)) + fmt.Printf("Storage Tries: %d nodes (%s) across %d tries\n", storageTotal, common.StorageSize(storageTotalSize), s.StorageCount) + + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{"Depth", "Account Nodes", "Account Size", "Storage Nodes", "Storage Size"}) + for i := 0; i < trieStatLevels; i++ { + accountNodes := s.Account.Levels[i].Short + s.Account.Levels[i].Full + s.Account.Levels[i].Value + accountSize := s.Account.Levels[i].Size + storageNodes := s.StorageLevels[i].Short + s.StorageLevels[i].Full + s.StorageLevels[i].Value + storageSize := s.StorageLevels[i].Size + if accountNodes == 0 && storageNodes == 0 { + continue + } + table.AppendBulk([][]string{{ + fmt.Sprint(i), + fmt.Sprint(accountNodes), + common.StorageSize(accountSize).String(), + fmt.Sprint(storageNodes), + common.StorageSize(storageSize).String(), + }}) + } + table.Render() + fmt.Print(b.String()) + fmt.Println() +} + +func (s *inspectSummary) displayTop(title string, list []*storageStats) { + fmt.Println(title) + if len(list) == 0 { + fmt.Println("No storage tries found") + fmt.Println() + return + } + for i, item := range list { + fmt.Printf("%d: %s\n", i+1, item.Owner) + item.toLevelStats().display("storage trie") + } +} + +func (s *inspectSummary) MarshalJSON() ([]byte, error) { + type jsonAccountTrie struct { + Name string `json:"Name"` + Levels []jsonLevel `json:"Levels"` + Summary jsonLevel `json:"Summary"` + } + type jsonStorageSummary struct { + TotalStorageTries uint64 `json:"TotalStorageTries"` + Totals jsonLevel `json:"Totals"` + Levels []jsonLevel `json:"Levels"` + DepthHistogram [trieStatLevels]uint64 `json:"DepthHistogram"` + } + type jsonInspectSummary struct { + AccountTrie jsonAccountTrie `json:"AccountTrie"` + StorageSummary jsonStorageSummary `json:"StorageSummary"` + TopByDepth []*storageStats `json:"TopByDepth"` + TopByTotalNodes []*storageStats `json:"TopByTotalNodes"` + TopByValueNodes []*storageStats `json:"TopByValueNodes"` + } + return json.Marshal(jsonInspectSummary{ + AccountTrie: jsonAccountTrie{ + Name: "account trie", + Levels: trimLevels(s.Account.Levels), + Summary: s.Account.Summary, + }, + StorageSummary: jsonStorageSummary{ + TotalStorageTries: s.StorageCount, + Totals: s.StorageTotals, + Levels: trimLevels(s.StorageLevels), + DepthHistogram: s.DepthHistogram, + }, + TopByDepth: s.TopByDepth, + TopByTotalNodes: s.TopByTotalNodes, + TopByValueNodes: s.TopByValueNodes, + }) +} + +func (s *inspectSummary) writeJSON(path string) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer file.Close() + + enc := json.NewEncoder(file) + enc.SetIndent("", " ") + return enc.Encode(s) +} + +// display will print a table displaying the trie's node statistics. +func (s *LevelStats) display(title string) { + // Shorten title if too long. + if len(title) > 32 { + title = title[0:8] + "..." + title[len(title)-8:] + } + + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{title, "Level", "Short Nodes", "Full Node", "Value Node"}) + + stat := &stat{} + for i := range s.level { + if s.level[i].empty() { + continue + } + short, full, value, _ := s.level[i].load() + table.AppendBulk([][]string{{"-", fmt.Sprint(i), fmt.Sprint(short), fmt.Sprint(full), fmt.Sprint(value)}}) + stat.add(&s.level[i]) + } + short, full, value, _ := stat.load() + table.SetFooter([]string{"Total", "", fmt.Sprint(short), fmt.Sprint(full), fmt.Sprint(value)}) + table.Render() + fmt.Print(b.String()) + fmt.Println("Max depth", s.MaxDepth()) + fmt.Println() +} + +type jsonLevel struct { + Short uint64 + Full uint64 + Value uint64 + Size uint64 +} diff --git a/trie/inspect_test.go b/trie/inspect_test.go new file mode 100644 index 0000000000..c07904c52d --- /dev/null +++ b/trie/inspect_test.go @@ -0,0 +1,256 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "encoding/json" + "math/rand" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/holiman/uint256" +) + +// TestInspect inspects a randomly generated account trie. It's useful for +// quickly verifying changes to the results display. +func TestInspect(t *testing.T) { + db := newTestDatabase(rawdb.NewMemoryDatabase(), rawdb.HashScheme) + trie, err := NewStateTrie(TrieID(types.EmptyRootHash), db) + if err != nil { + t.Fatalf("failed to create state trie: %v", err) + } + // Create a realistic looking account trie with storage. + addresses, accounts := makeAccountsWithStorage(db, 11, true) + for i := 0; i < len(addresses); i++ { + trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) + } + // Insert the accounts into the trie and hash it + root, nodes := trie.Commit(true) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Commit(root) + + tempDir := t.TempDir() + dumpPath := filepath.Join(tempDir, "trie-dump.bin") + if err := Inspect(db, root, &InspectConfig{ + TopN: 1, + DumpPath: dumpPath, + Path: filepath.Join(tempDir, "trie-summary.json"), + }); err != nil { + t.Fatalf("inspect failed: %v", err) + } + reanalysisPath := filepath.Join(tempDir, "trie-summary-reanalysis.json") + if err := Summarize(dumpPath, &InspectConfig{ + TopN: 1, + Path: reanalysisPath, + }); err != nil { + t.Fatalf("summarize failed: %v", err) + } + + inspectSummaryPath := filepath.Join(tempDir, "trie-summary.json") + inspectOut := loadInspectJSON(t, inspectSummaryPath) + reanalysisOut := loadInspectJSON(t, reanalysisPath) + + if len(inspectOut.StorageSummary.Levels) == 0 { + t.Fatal("expected StorageSummary.Levels to be populated") + } + if inspectOut.AccountTrie.Summary.Size == 0 { + t.Fatal("expected account trie size summary to be populated") + } + if inspectOut.StorageSummary.Totals.Size == 0 { + t.Fatal("expected storage trie size summary to be populated") + } + if !reflect.DeepEqual(inspectOut.AccountTrie, reanalysisOut.AccountTrie) { + t.Fatal("account trie summary mismatch between inspect and summarize") + } + if !reflect.DeepEqual(inspectOut.StorageSummary, reanalysisOut.StorageSummary) { + t.Fatal("storage summary mismatch between inspect and summarize") + } + + assertStorageTotalsMatchLevels(t, inspectOut) + assertStorageTotalsMatchLevels(t, reanalysisOut) + assertAccountTotalsMatchLevels(t, inspectOut.AccountTrie) + assertAccountTotalsMatchLevels(t, reanalysisOut.AccountTrie) + + var histogramTotal uint64 + for _, count := range inspectOut.StorageSummary.DepthHistogram { + histogramTotal += count + } + if histogramTotal != inspectOut.StorageSummary.TotalStorageTries { + t.Fatalf("depth histogram total %d does not match total storage tries %d", histogramTotal, inspectOut.StorageSummary.TotalStorageTries) + } +} + +type inspectJSONOutput struct { + // Reuse storageStats for AccountTrie JSON to avoid introducing a parallel + // account summary test type. AccountTrie JSON includes Levels+Summary, + // which map directly; other storageStats fields remain zero-values. + AccountTrie storageStats `json:"AccountTrie"` + + StorageSummary struct { + TotalStorageTries uint64 `json:"TotalStorageTries"` + Totals jsonLevel `json:"Totals"` + Levels []jsonLevel `json:"Levels"` + DepthHistogram [trieStatLevels]uint64 `json:"DepthHistogram"` + } `json:"StorageSummary"` +} + +func loadInspectJSON(t *testing.T, path string) inspectJSONOutput { + t.Helper() + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read %s: %v", path, err) + } + var out inspectJSONOutput + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("failed to decode %s: %v", path, err) + } + return out +} + +func assertStorageTotalsMatchLevels(t *testing.T, out inspectJSONOutput) { + t.Helper() + var fromLevels jsonLevel + for _, level := range out.StorageSummary.Levels { + fromLevels.Short += level.Short + fromLevels.Full += level.Full + fromLevels.Value += level.Value + fromLevels.Size += level.Size + } + if fromLevels.Short != out.StorageSummary.Totals.Short || fromLevels.Full != out.StorageSummary.Totals.Full || fromLevels.Value != out.StorageSummary.Totals.Value || fromLevels.Size != out.StorageSummary.Totals.Size { + t.Fatalf("storage totals mismatch: levels=%+v totals=%+v", fromLevels, out.StorageSummary.Totals) + } +} + +func assertAccountTotalsMatchLevels(t *testing.T, account storageStats) { + t.Helper() + var fromLevels jsonLevel + for _, level := range account.Levels { + fromLevels.Short += level.Short + fromLevels.Full += level.Full + fromLevels.Value += level.Value + fromLevels.Size += level.Size + } + if fromLevels.Short != account.Summary.Short || fromLevels.Full != account.Summary.Full || fromLevels.Value != account.Summary.Value || fromLevels.Size != account.Summary.Size { + t.Fatalf("account totals mismatch: levels=%+v totals=%+v", fromLevels, account.Summary) + } +} + +// TestInspectContract tests the InspectContract function on a single account +// with storage and snapshot data. +func TestInspectContract(t *testing.T) { + diskdb := rawdb.NewMemoryDatabase() + db := newTestDatabase(diskdb, rawdb.HashScheme) + + // Create a contract address and its storage trie. + address := common.HexToAddress("0x1234567890abcdef1234567890abcdef12345678") + accountHash := crypto.Keccak256Hash(address.Bytes()) + + // Build a storage trie with some entries. + storageTrie := NewEmpty(db) + storageSlots := make(map[common.Hash][]byte) + for i := 0; i < 10; i++ { + k := crypto.Keccak256Hash([]byte{byte(i)}) + v := []byte{byte(i + 1)} + storageTrie.MustUpdate(k.Bytes(), v) + storageSlots[k] = v + } + storageRoot, storageNodes := storageTrie.Commit(true) + db.Update(storageRoot, types.EmptyRootHash, trienode.NewWithNodeSet(storageNodes)) + db.Commit(storageRoot) + + // Build the account trie with the contract account. + account := types.StateAccount{ + Nonce: 1, + Balance: uint256.NewInt(1000), + Root: storageRoot, + CodeHash: crypto.Keccak256(nil), + } + accountRLP, err := rlp.EncodeToBytes(&account) + if err != nil { + t.Fatalf("failed to encode account: %v", err) + } + + accountTrie := NewEmpty(db) + accountTrie.MustUpdate(crypto.Keccak256(address.Bytes()), accountRLP) + stateRoot, accountNodes := accountTrie.Commit(true) + db.Update(stateRoot, types.EmptyRootHash, trienode.NewWithNodeSet(accountNodes)) + db.Commit(stateRoot) + + // Write snapshot data for the account and its storage slots. + rawdb.WriteAccountSnapshot(diskdb, accountHash, accountRLP) + for k, v := range storageSlots { + rawdb.WriteStorageSnapshot(diskdb, accountHash, k, v) + } + + // InspectContract should succeed without error. + if err := InspectContract(db, diskdb, stateRoot, address); err != nil { + t.Fatalf("InspectContract failed: %v", err) + } +} + +func makeAccountsWithStorage(db *testDb, size int, storage bool) (addresses [][20]byte, accounts [][]byte) { + // Make the random benchmark deterministic + random := rand.New(rand.NewSource(0)) + + addresses = make([][20]byte, size) + for i := 0; i < len(addresses); i++ { + data := make([]byte, 20) + random.Read(data) + copy(addresses[i][:], data) + } + accounts = make([][]byte, len(addresses)) + for i := 0; i < len(accounts); i++ { + var ( + nonce = uint64(random.Int63()) + root = types.EmptyRootHash + code = crypto.Keccak256(nil) + ) + if storage { + trie := NewEmpty(db) + for range random.Uint32()%256 + 1 { // non-zero + k, v := make([]byte, 32), make([]byte, 32) + random.Read(k) + random.Read(v) + trie.MustUpdate(k, v) + } + var nodes *trienode.NodeSet + root, nodes = trie.Commit(true) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Commit(root) + } + numBytes := random.Uint32() % 33 // [0, 32] bytes + balanceBytes := make([]byte, numBytes) + random.Read(balanceBytes) + balance := new(uint256.Int).SetBytes(balanceBytes) + data, _ := rlp.EncodeToBytes(&types.StateAccount{ + Nonce: nonce, + Balance: balance, + Root: root, + CodeHash: code, + }) + accounts[i] = data + } + return addresses, accounts +} diff --git a/trie/levelstats.go b/trie/levelstats.go new file mode 100644 index 0000000000..9168e3fbaf --- /dev/null +++ b/trie/levelstats.go @@ -0,0 +1,123 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + "sync/atomic" +) + +const trieStatLevels = 16 + +// LevelStats tracks the type and count of trie nodes at each level in a trie. +// +// Note: theoretically it is possible to have up to 64 trie levels, but +// LevelStats supports exactly 16 levels and panics on deeper paths. +type LevelStats struct { + level [trieStatLevels]stat +} + +// NewLevelStats creates an empty trie statistics collector. +func NewLevelStats() *LevelStats { + return &LevelStats{} +} + +// MaxDepth iterates each level and finds the deepest level with at least one +// trie node. +func (s *LevelStats) MaxDepth() int { + depth := 0 + for i := range s.level { + if s.level[i].short.Load() != 0 || s.level[i].full.Load() != 0 || s.level[i].value.Load() != 0 { + depth = i + } + } + return depth +} + +// TotalNodes returns the total number of nodes across all levels and types. +func (s *LevelStats) TotalNodes() uint64 { + var total uint64 + for i := range s.level { + total += s.level[i].short.Load() + s.level[i].full.Load() + s.level[i].value.Load() + } + return total +} + +// add increases the node count by one for the specified node type and depth. +func (s *LevelStats) add(n node, depth uint32) { + d := int(depth) + switch (n).(type) { + case *shortNode: + s.level[d].short.Add(1) + case *fullNode: + s.level[d].full.Add(1) + case valueNode: + s.level[d].value.Add(1) + default: + panic(fmt.Sprintf("%T: invalid node: %v", n, n)) + } +} + +// addSize increases the raw byte-size tally at the specified depth. +func (s *LevelStats) addSize(depth uint32, size uint64) { + s.level[depth].size.Add(size) +} + +// AddLeaf records a leaf depth. Witness collection reuses the value-node bucket +// for leaf accounting. It panics if the depth is outside [0, 15]. +func (s *LevelStats) AddLeaf(depth int) { + s.level[depth].value.Add(1) +} + +// LeafDepths returns leaf counts grouped by depth. +func (s *LevelStats) LeafDepths() [trieStatLevels]int64 { + var leaves [trieStatLevels]int64 + for i := range s.level { + leaves[i] = int64(s.level[i].value.Load()) + } + return leaves +} + +// stat is a specific level's count of each node type. +type stat struct { + short atomic.Uint64 + full atomic.Uint64 + value atomic.Uint64 + size atomic.Uint64 +} + +// empty is a helper that returns whether there are any trie nodes at the level. +func (s *stat) empty() bool { + if s.full.Load() == 0 && s.short.Load() == 0 && s.value.Load() == 0 && s.size.Load() == 0 { + return true + } + return false +} + +// load is a helper that loads each node type's value. +func (s *stat) load() (uint64, uint64, uint64, uint64) { + return s.short.Load(), s.full.Load(), s.value.Load(), s.size.Load() +} + +// add is a helper that adds two level's stats together. +func (s *stat) add(other *stat) *stat { + s.short.Add(other.short.Load()) + s.full.Add(other.full.Load()) + s.value.Add(other.value.Load()) + s.size.Add(other.size.Load()) + return s +} diff --git a/trie/levelstats_test.go b/trie/levelstats_test.go new file mode 100644 index 0000000000..90581eb1ab --- /dev/null +++ b/trie/levelstats_test.go @@ -0,0 +1,37 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import "testing" + +func TestLevelStatsAddLeafDepthBounds(t *testing.T) { + stats := NewLevelStats() + stats.AddLeaf(15) + + if got := stats.LeafDepths()[15]; got != 1 { + t.Fatalf("leaf count at depth 15 = %d, want 1", got) + } +} + +func TestLevelStatsAddLeafPanicsOnDepth16(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Fatal("expected panic for depth >= 16") + } + }() + NewLevelStats().AddLeaf(16) +} diff --git a/trie/proof.go b/trie/proof.go index 1a06ed5d5e..58075daf9b 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -454,7 +454,7 @@ func hasRightElement(node node, key []byte) bool { // // The firstKey is paired with firstProof, not necessarily the same as keys[0] // (unless firstProof is an existent proof). Similarly, lastKey and lastProof -// are paired. +// are paired. The firstKey should be less than or equal to all keys in the list. // // Expect the normal case, this function can also be used to verify the following // range proofs: @@ -520,9 +520,14 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, valu } return false, nil } - var lastKey = keys[len(keys)-1] + // Short circuit if the key of first element is greater than firstKey. + // A nil firstKey slice is equivalent to an empty slice. + if bytes.Compare(firstKey, keys[0]) > 0 { + return false, errors.New("unexpected key-value pairs preceding the requested range") + } // Special case, there is only one element and two edge keys are same. // In this case, we can't construct two edge paths. So handle it here. + var lastKey = keys[len(keys)-1] if len(keys) == 1 && bytes.Equal(firstKey, lastKey) { root, val, err := proofToPath(rootHash, nil, firstKey, proof, false) if err != nil { @@ -577,7 +582,9 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, valu tr.root = nil } for index, key := range keys { - tr.Update(key, values[index]) + if err := tr.Update(key, values[index]); err != nil { + return false, err + } } if tr.Hash() != rootHash { return false, fmt.Errorf("invalid proof, want hash %x, got %x", rootHash, tr.Hash()) diff --git a/trie/trie_test.go b/trie/trie_test.go index 3423cde59c..3661933e22 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -880,6 +880,7 @@ func (b *spongeBatch) ValueSize() int { return 100 } func (b *spongeBatch) Write() error { return nil } func (b *spongeBatch) Reset() {} func (b *spongeBatch) Replay(w ethdb.KeyValueWriter) error { return nil } +func (b *spongeBatch) Close() {} // TestCommitSequence tests that the trie.Commit operation writes the elements // of the trie in the expected order. diff --git a/triedb/hashdb/database.go b/triedb/hashdb/database.go index 38392aa519..90d0514290 100644 --- a/triedb/hashdb/database.go +++ b/triedb/hashdb/database.go @@ -612,6 +612,9 @@ func (db *Database) Close() error { // NodeReader returns a reader for accessing trie nodes within the specified state. // An error will be returned if the specified state is not available. func (db *Database) NodeReader(root common.Hash) (database.NodeReader, error) { + if root == types.EmptyRootHash { + return &reader{db: db}, nil + } if _, err := db.node(root); err != nil { return nil, fmt.Errorf("state %#x is not available, %v", root, err) } diff --git a/triedb/pathdb/buffer.go b/triedb/pathdb/buffer.go index 853e1090b3..5d3099285f 100644 --- a/triedb/pathdb/buffer.go +++ b/triedb/pathdb/buffer.go @@ -180,6 +180,8 @@ func (b *buffer) flush(root common.Hash, db ethdb.KeyValueStore, freezers []ethd b.flushErr = err return } + batch.Close() + commitBytesMeter.Mark(int64(size)) commitNodesMeter.Mark(int64(nodes)) commitAccountsMeter.Mark(int64(accounts)) diff --git a/triedb/pathdb/history.go b/triedb/pathdb/history.go index 820c3c03bf..0a9f7091fa 100644 --- a/triedb/pathdb/history.go +++ b/triedb/pathdb/history.go @@ -412,28 +412,34 @@ func repairHistory(db ethdb.Database, isVerkle bool, readOnly bool, stateID uint // Truncate excessive history entries in either the state history or // the trienode history, ensuring both histories remain aligned with // the state. - head, err := states.Ancients() + shead, err := states.Ancients() if err != nil { return nil, nil, err } - if stateID > head { - return nil, nil, fmt.Errorf("gap between state [#%d] and state history [#%d]", stateID, head) + if stateID > shead { // Gap is not permitted in the state history + return nil, nil, fmt.Errorf("gap between state [#%d] and state history [#%d]", stateID, shead) } + truncTo := min(shead, stateID) + if trienodes != nil { - th, err := trienodes.Ancients() + thead, err := trienodes.Ancients() if err != nil { return nil, nil, err } - if stateID > th { - return nil, nil, fmt.Errorf("gap between state [#%d] and trienode history [#%d]", stateID, th) - } - if th != head { - log.Info("Histories are not aligned with each other", "state", head, "trienode", th) - head = min(head, th) + if stateID <= thead { + truncTo = min(truncTo, thead) + } else { + if thead == 0 { + _, err = trienodes.TruncateTail(stateID) + if err != nil { + return nil, nil, err + } + log.Warn("Initialized trienode history") + } else { + return nil, nil, fmt.Errorf("gap between state [#%d] and trienode history [#%d]", stateID, thead) + } } } - head = min(head, stateID) - // Truncate the extra history elements above in freezer in case it's not // aligned with the state. It might happen after an unclean shutdown. truncate := func(store ethdb.AncientStore, typ historyType, nhead uint64) { @@ -448,7 +454,7 @@ func repairHistory(db ethdb.Database, isVerkle bool, readOnly bool, stateID uint log.Warn("Truncated extra histories", "typ", typ, "number", pruned) } } - truncate(states, typeStateHistory, head) - truncate(trienodes, typeTrienodeHistory, head) + truncate(states, typeStateHistory, truncTo) + truncate(trienodes, typeTrienodeHistory, truncTo) return states, trienodes, nil } diff --git a/triedb/pathdb/reader.go b/triedb/pathdb/reader.go index aaa64e902c..e3cfbcba8a 100644 --- a/triedb/pathdb/reader.go +++ b/triedb/pathdb/reader.go @@ -349,7 +349,7 @@ func (db *Database) HistoricNodeReader(root common.Hash) (*HistoricalNodeReader, // are not accessible. meta, err := readTrienodeMetadata(db.trienodeFreezer, *id+1) if err != nil { - return nil, err // e.g., the referred trienode history has been pruned + return nil, fmt.Errorf("state %#x is not available", root) // e.g., the referred trienode history has been pruned } if meta.parent != root { return nil, fmt.Errorf("state %#x is not canonincal", root) diff --git a/version/version.go b/version/version.go index bcb61f27b1..b9bc87c866 100644 --- a/version/version.go +++ b/version/version.go @@ -19,6 +19,6 @@ package version const ( Major = 1 // Major version component of the current release Minor = 17 // Minor version component of the current release - Patch = 0 // Patch version component of the current release + Patch = 2 // Patch version component of the current release Meta = "unstable" // Version metadata to append to the version string )