forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-25 14:16:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

resolve merge conflict

Browse source
This commit is contained in:
Sina Mahmoodi 2025-06-16 12:40:10 +02:00
commit 1c13317390
402 changed files with 26194 additions and 11984 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
.gitea/workflows/release-azure-cleanup.yml Normal file
View file

@ -0,0 +1,22 @@
on:
schedule:
- cron: '0 14 * * *'
workflow_dispatch:
jobs:
azure-cleanup:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.24
cache: false
- name: Run cleanup script
run: |
go run build/ci.go purge -store gethstore/builds -days 14
env:
AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}

36
.gitea/workflows/release-ppa.yml Normal file
View file

@ -0,0 +1,36 @@
on:
schedule:
- cron: '0 15 * * *'
push:
branches:
- "release/*"
workflow_dispatch:
jobs:
ppa:
name: PPA Upload
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.24
cache: false
- name: Install deb toolchain
run: |
apt-get update
apt-get -yq --no-install-suggests --no-install-recommends install devscripts debhelper dput fakeroot
- name: Add launchpad to known_hosts
run: |
echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
- name: Run ci.go
run: |
go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
env:
PPA_SIGNING_KEY: ${{ secrets.PPA_SIGNING_KEY }}
PPA_SSH_KEY: ${{ secrets.PPA_SSH_KEY }}

147
.gitea/workflows/release.yml Normal file
View file

@ -0,0 +1,147 @@
on:
push:
branches:
- "master"
tags:
- "v*"
jobs:
linux-intel:
name: Linux Build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.24
cache: false
- name: Install cross toolchain
run: |
apt-get update
apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib
- name: Build (amd64)
run: |
go run build/ci.go install -arch amd64 -dlgo
- name: Create/upload archive (amd64)
run: |
go run build/ci.go archive -arch amd64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
rm -f build/bin/*
env:
LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
- name: Build (386)
run: |
go run build/ci.go install -arch 386 -dlgo
- name: Create/upload archive (386)
run: |
go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
rm -f build/bin/*
env:
LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
linux-arm:
name: Linux Build (arm)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.24
cache: false
- name: Install cross toolchain
run: |
apt-get update
apt-get -yq --no-install-suggests --no-install-recommends install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
ln -s /usr/include/asm-generic /usr/include/asm
- name: Build (arm64)
run: |
go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
- name: Create/upload archive (arm64)
run: |
go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
rm -fr build/bin/*
env:
LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
- name: Run build (arm5)
run: |
go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
env:
GOARM: "5"
- name: Create/upload archive (arm5)
run: |
go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
env:
GOARM: "5"
LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
- name: Run build (arm6)
run: |
go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
env:
GOARM: "6"
- name: Create/upload archive (arm6)
run: |
go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
rm -fr build/bin/*
env:
GOARM: "6"
LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
- name: Run build (arm7)
run: |
go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
env:
GOARM: "7"
- name: Create/upload archive (arm7)
run: |
go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
rm -fr build/bin/*
env:
GOARM: "7"
LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
docker:
name: Docker Image
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: 1.24
cache: false
- name: Run docker build
env:
DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
run: |
go run build/ci.go dockerx -platform linux/amd64,linux/arm64,linux/riscv64 -upload

19
.github/CODEOWNERS vendored
View file

@ -9,12 +9,11 @@ beacon/light/ @zsfelfoldi
beacon/merkle/ @zsfelfoldi
beacon/types/ @zsfelfoldi @fjl
beacon/params/ @zsfelfoldi @fjl
cmd/clef/ @holiman
cmd/evm/ @holiman @MariusVanDerWijden @lightclient
core/state/ @rjl493456442 @holiman
crypto/ @gballet @jwasinger @holiman @fjl
core/ @holiman @rjl493456442
eth/ @holiman @rjl493456442
cmd/evm/ @MariusVanDerWijden @lightclient
core/state/ @rjl493456442
crypto/ @gballet @jwasinger @fjl
core/ @rjl493456442
eth/ @rjl493456442
eth/catalyst/ @MariusVanDerWijden @lightclient @fjl @jwasinger
eth/tracers/ @s1na
ethclient/ @fjl
@ -26,11 +25,9 @@ core/tracing/ @s1na
graphql/ @s1na
internal/ethapi/ @fjl @s1na @lightclient
internal/era/ @lightclient
metrics/ @holiman
miner/ @MariusVanDerWijden @holiman @fjl @rjl493456442
miner/ @MariusVanDerWijden @fjl @rjl493456442
node/ @fjl
p2p/ @fjl @zsfelfoldi
rlp/ @fjl
params/ @fjl @holiman @karalabe @gballet @rjl493456442 @zsfelfoldi
rpc/ @fjl @holiman
signer/ @holiman
params/ @fjl @karalabe @gballet @rjl493456442 @zsfelfoldi
rpc/ @fjl

13
.gitignore vendored
View file

@ -43,3 +43,16 @@ profile.cov
.vscode
tests/spec-tests/
# binaries
cmd/abidump/abidump
cmd/abigen/abigen
cmd/blsync/blsync
cmd/clef/clef
cmd/devp2p/devp2p
cmd/era/era
cmd/ethkey/ethkey
cmd/evm/evm
cmd/geth/geth
cmd/rlpdump/rlpdump
cmd/workload/workload

131
.golangci.yml
View file

@ -1,31 +1,25 @@
# This file configures github.com/golangci/golangci-lint.
version: '2'
run:
timeout: 20m
tests: true
linters:
disable-all: true
default: none
enable:
- goimports
- gosimple
- govet
- ineffassign
- misspell
- unconvert
- typecheck
- unused
- staticcheck
- bidichk
- durationcheck
- copyloopvar
- whitespace
- revive # only certain checks enabled
- durationcheck
- gocheckcompilerdirectives
- reassign
- govet
- ineffassign
- mirror
- misspell
- reassign
- revive # only certain checks enabled
- staticcheck
- unconvert
- unused
- usetesting
- whitespace
### linters we tried and will not be using:
###
# - structcheck # lots of false positives
@ -36,44 +30,67 @@ linters:
# - exhaustive # silly check
# - makezero # false positives
# - nilerr # several intentional
linters-settings:
gofmt:
simplify: true
revive:
enable-all-rules: false
# here we enable specific useful rules
# see https://golangci-lint.run/usage/linters/#revive for supported rules
settings:
staticcheck:
checks:
# disable Quickfixes
- -QF1*
revive:
enable-all-rules: false
# here we enable specific useful rules
# see https://golangci-lint.run/usage/linters/#revive for supported rules
rules:
- name: receiver-naming
severity: warning
disabled: false
exclude:
- ''
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- std-error-handling
rules:
- name: receiver-naming
severity: warning
disabled: false
exclude: [""]
issues:
# default is true. Enables skipping of directories:
# vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
exclude-dirs-use-default: true
exclude-files:
- core/genesis_alloc.go
exclude-rules:
- path: crypto/bn256/cloudflare/optate.go
linters:
- deadcode
- staticcheck
- path: crypto/bn256/
linters:
- revive
- path: cmd/utils/flags.go
text: "SA1019: cfg.TxLookupLimit is deprecated: use 'TransactionHistory' instead."
- path: cmd/utils/flags.go
text: "SA1019: ethconfig.Defaults.TxLookupLimit is deprecated: use 'TransactionHistory' instead."
- path: internal/build/pgp.go
text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.'
- path: core/vm/contracts.go
text: 'SA1019: "golang.org/x/crypto/ripemd160" is deprecated: RIPEMD-160 is a legacy hash and should not be used for new applications.'
exclude:
- 'SA1019: event.TypeMux is deprecated: use Feed'
- 'SA1019: strings.Title is deprecated'
- 'SA1019: strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead.'
- 'SA1029: should not use built-in type string as key for value'
- linters:
- deadcode
- staticcheck
path: crypto/bn256/cloudflare/optate.go
- linters:
- revive
path: crypto/bn256/
- path: cmd/utils/flags.go
text: "SA1019: cfg.TxLookupLimit is deprecated: use 'TransactionHistory' instead."
- path: cmd/utils/flags.go
text: "SA1019: ethconfig.Defaults.TxLookupLimit is deprecated: use 'TransactionHistory' instead."
- path: internal/build/pgp.go
text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.'
- path: core/vm/contracts.go
text: 'SA1019: "golang.org/x/crypto/ripemd160" is deprecated: RIPEMD-160 is a legacy hash and should not be used for new applications.'
- path: (.+)\.go$
text: 'SA1019: event.TypeMux is deprecated: use Feed'
- path: (.+)\.go$
text: 'SA1019: strings.Title is deprecated'
- path: (.+)\.go$
text: 'SA1019: strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead.'
- path: (.+)\.go$
text: 'SA1029: should not use built-in type string as key for value'
paths:
- core/genesis_alloc.go
- third_party$
- builtin$
- examples$
formatters:
enable:
- goimports
settings:
gofmt:
simplify: true
exclusions:
generated: lax
paths:
- core/genesis_alloc.go
- third_party$
- builtin$
- examples$

7
.mailmap
View file

@ -50,6 +50,10 @@ Boqin Qin <bobbqqin@bupt.edu.cn> <Bobbqqin@gmail.com>
Casey Detrio <cdetrio@gmail.com>
Charlotte <tqpcharlie@proton.me>
Charlotte <tqpcharlie@proton.me> <Zachinquarantine@protonmail.com>
Charlotte <tqpcharlie@proton.me> <zachinquarantine@yahoo.com>
Cheng Li <lob4tt@gmail.com>
Chris Ziogas <ziogaschr@gmail.com>
@ -301,9 +305,6 @@ Yohann Léon <sybiload@gmail.com>
yzb <335357057@qq.com>
yzb <335357057@qq.com> <flyingyzb@gmail.com>
Zachinquarantine <Zachinquarantine@protonmail.com>
Zachinquarantine <Zachinquarantine@protonmail.com> <zachinquarantine@yahoo.com>
Ziyuan Zhong <zzy.albert@163.com>
Zsolt Felföldi <zsfelfoldi@gmail.com>

88
.travis.yml
View file

@ -2,16 +2,10 @@ language: go
go_import_path: github.com/ethereum/go-ethereum
sudo: false
jobs:
allow_failures:
- stage: build
os: osx
env:
- azure-osx
include:
# This builder create and push the Docker images for all architectures
- stage: build
if: type = push
if: type = push && tag ~= /^v[0-9]/
os: linux
arch: amd64
dist: focal
@ -27,9 +21,25 @@ jobs:
script:
- go run build/ci.go dockerx -platform "linux/amd64,linux/arm64,linux/riscv64" -hub ethereum/client-go -upload
# This builder does the Ubuntu PPA nightly uploads
- stage: build
if: type = push && tag ~= /^v[0-9]/
os: linux
dist: focal
go: 1.24.x
env:
- ubuntu-ppa
git:
submodules: false # avoid cloning ethereum/tests
before_install:
- sudo -E apt-get -yq --no-install-suggests --no-install-recommends install devscripts debhelper dput fakeroot
script:
- echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
- go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
# This builder does the Linux Azure uploads
- stage: build
if: type = push
if: type = push && tag ~= /^v[0-9]/
os: linux
dist: focal
sudo: required
@ -62,57 +72,6 @@ jobs:
- go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
- go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
# This builder does the OSX Azure uploads
- stage: build
if: type = push
os: osx
osx_image: xcode14.2
go: 1.23.1 # See https://github.com/ethereum/go-ethereum/pull/30478
env:
- azure-osx
git:
submodules: false # avoid cloning ethereum/tests
script:
- ln -sf /Users/travis/gopath/bin/go1.23.1 /usr/local/bin/go # Work around travis go-setup bug
- go run build/ci.go install -dlgo
- go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
- go run build/ci.go install -dlgo -arch arm64
- go run build/ci.go archive -arch arm64 -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
# These builders run the tests
- stage: build
if: type = push
os: linux
arch: amd64
dist: focal
go: 1.24.x
script:
- travis_wait 45 go run build/ci.go test $TEST_PACKAGES
- stage: build
if: type = push
os: linux
dist: focal
go: 1.23.x
script:
- travis_wait 45 go run build/ci.go test $TEST_PACKAGES
# This builder does the Ubuntu PPA nightly uploads
- stage: build
if: type = cron || (type = push && tag ~= /^v[0-9]/)
os: linux
dist: focal
go: 1.24.x
env:
- ubuntu-ppa
git:
submodules: false # avoid cloning ethereum/tests
before_install:
- sudo -E apt-get -yq --no-install-suggests --no-install-recommends install devscripts debhelper dput fakeroot
script:
- echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
- go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
# This builder does the Azure archive purges to avoid accumulating junk
- stage: build
if: type = cron
@ -125,14 +84,3 @@ jobs:
submodules: false # avoid cloning ethereum/tests
script:
- go run build/ci.go purge -store gethstore/builds -days 14
# This builder executes race tests
- stage: build
if: type = cron
os: linux
dist: focal
go: 1.24.x
env:
- racetests
script:
- travis_wait 60 go run build/ci.go test -race $TEST_PACKAGES

2
AUTHORS
View file

@ -123,6 +123,7 @@ Ceyhun Onur <ceyhun.onur@avalabs.org>
chabashilah <doumodoumo@gmail.com>
changhong <changhong.yu@shanbay.com>
Charles Cooper <cooper.charles.m@gmail.com>
Charlotte <tqpcharlie@proton.me>
Chase Wright <mysticryuujin@gmail.com>
Chawin Aiemvaravutigul <nick41746@hotmail.com>
Chen Quan <terasum@163.com>
@ -839,7 +840,6 @@ ywzqwwt <39263032+ywzqwwt@users.noreply.github.com>
yzb <335357057@qq.com>
zaccoding <zaccoding725@gmail.com>
Zach <zach.ramsay@gmail.com>
Zachinquarantine <Zachinquarantine@protonmail.com>
zah <zahary@gmail.com>
Zahoor Mohamed <zahoor@zahoor.in>
Zak Cole <zak@beattiecole.com>

8
Makefile
View file

@ -2,7 +2,7 @@
# with Go source code. If you know what GOPATH is then you probably
# don't need to bother with make.
.PHONY: geth all test lint fmt clean devtools help
.PHONY: geth evm all test lint fmt clean devtools help
GOBIN = ./build/bin
GO ?= latest
@ -14,6 +14,12 @@ geth:
@echo "Done building."
@echo "Run \"$(GOBIN)/geth\" to launch geth."
#? evm: Build evm.
evm:
$(GORUN) build/ci.go install ./cmd/evm
@echo "Done building."
@echo "Run \"$(GOBIN)/evm\" to launch evm."
#? all: Build all packages and executables.
all:
$(GORUN) build/ci.go install

2
README.md
View file

@ -162,7 +162,7 @@ accessible from the outside.
As a developer, sooner rather than later you'll want to start interacting with `geth` and the
Ethereum network via your own programs and not manually through the console. To aid
this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://ethereum.github.io/execution-apis/api-documentation/)
this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://ethereum.org/en/developers/docs/apis/json-rpc/)
and [`geth` specific APIs](https://geth.ethereum.org/docs/interacting-with-geth/rpc)).
These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based
platforms, and named pipes on Windows).

60
accounts/abi/abigen/bind_test.go
View file

@ -541,7 +541,7 @@ var bindTests = []struct {
struct A {
bytes32 B;
}
function F() public view returns (A[] memory a, uint256[] memory c, bool[] memory d) {
A[] memory a = new A[](2);
a[0].B = bytes32(uint256(1234) << 96);
@ -549,7 +549,7 @@ var bindTests = []struct {
bool[] memory d;
return (a, c, d);
}
function G() public view returns (A[] memory a) {
A[] memory a = new A[](2);
a[0].B = bytes32(uint256(1234) << 96);
@ -571,10 +571,10 @@ var bindTests = []struct {
// Generate a new random account and a funded simulator
key, _ := crypto.GenerateKey()
auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000)
defer sim.Close()
// Deploy a structs method invoker contract and execute its default method
_, _, structs, err := DeployStructs(auth, sim)
if err != nil {
@ -939,6 +939,7 @@ var bindTests = []struct {
if _, err := eventer.RaiseSimpleEvent(auth, common.Address{byte(j)}, [32]byte{byte(j)}, true, big.NewInt(int64(10*i+j))); err != nil {
t.Fatalf("block %d, event %d: raise failed: %v", i, j, err)
}
time.Sleep(time.Millisecond * 200)
}
sim.Commit()
}
@ -1495,7 +1496,7 @@ var bindTests = []struct {
if n != 3 {
t.Fatalf("Invalid bar0 event")
}
case <-time.NewTimer(3 * time.Second).C:
case <-time.NewTimer(10 * time.Second).C:
t.Fatalf("Wait bar0 event timeout")
}
@ -1506,7 +1507,7 @@ var bindTests = []struct {
if n != 1 {
t.Fatalf("Invalid bar event")
}
case <-time.NewTimer(3 * time.Second).C:
case <-time.NewTimer(10 * time.Second).C:
t.Fatalf("Wait bar event timeout")
}
close(stopCh)
@ -1701,13 +1702,13 @@ var bindTests = []struct {
`NewFallbacks`,
`
pragma solidity >=0.6.0 <0.7.0;
contract NewFallbacks {
event Fallback(bytes data);
fallback() external {
emit Fallback(msg.data);
}
event Received(address addr, uint value);
receive() external payable {
emit Received(msg.sender, msg.value);
@ -1719,7 +1720,7 @@ var bindTests = []struct {
`
"bytes"
"math/big"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
"github.com/ethereum/go-ethereum/core/types"
@ -1728,22 +1729,22 @@ var bindTests = []struct {
`
key, _ := crypto.GenerateKey()
addr := crypto.PubkeyToAddress(key.PublicKey)
sim := backends.NewSimulatedBackend(types.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 1000000)
defer sim.Close()
opts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337))
_, _, c, err := DeployNewFallbacks(opts, sim)
if err != nil {
t.Fatalf("Failed to deploy contract: %v", err)
}
sim.Commit()
// Test receive function
opts.Value = big.NewInt(100)
c.Receive(opts)
sim.Commit()
var gotEvent bool
iter, _ := c.FilterReceived(nil)
defer iter.Close()
@ -1760,14 +1761,14 @@ var bindTests = []struct {
if !gotEvent {
t.Fatal("Expect to receive event emitted by receive")
}
// Test fallback function
gotEvent = false
opts.Value = nil
calldata := []byte{0x01, 0x02, 0x03}
c.Fallback(opts, calldata)
sim.Commit()
iter2, _ := c.FilterFallback(nil)
defer iter2.Close()
for iter2.Next() {
@ -1806,7 +1807,9 @@ var bindTests = []struct {
[]string{"608060405234801561001057600080fd5b50610113806100206000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806324ec1d3f14602d575b600080fd5b60336035565b005b7fb4b2ff75e30cb4317eaae16dd8a187dd89978df17565104caa6c2797caae27d460405180604001604052806001815260200160028152506040516078919060ba565b60405180910390a1565b6040820160008201516096600085018260ad565b50602082015160a7602085018260ad565b50505050565b60b48160d3565b82525050565b600060408201905060cd60008301846082565b92915050565b600081905091905056fea26469706673582212208823628796125bf9941ce4eda18da1be3cf2931b231708ab848e1bd7151c0c9a64736f6c63430008070033"},
[]string{`[{"anonymous":false,"inputs":[{"components":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"uint256","name":"b","type":"uint256"}],"indexed":false,"internalType":"struct Test.MyStruct","name":"s","type":"tuple"}],"name":"StructEvent","type":"event"},{"inputs":[],"name":"TestEvent","outputs":[],"stateMutability":"nonpayable","type":"function"}]`},
`
"context"
"math/big"
"time"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
@ -1828,12 +1831,23 @@ var bindTests = []struct {
}
sim.Commit()
_, err = d.TestEvent(user)
tx, err := d.TestEvent(user)
if err != nil {
t.Fatalf("Failed to call contract %v", err)
}
sim.Commit()
// Wait for the transaction to be mined
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
receipt, err := bind.WaitMined(ctx, sim, tx)
if err != nil {
t.Fatalf("Failed to wait for tx to be mined: %v", err)
}
if receipt.Status != types.ReceiptStatusSuccessful {
t.Fatal("Transaction failed")
}
it, err := d.FilterStructEvent(nil)
if err != nil {
t.Fatalf("Failed to filter contract event %v", err)
@ -1862,7 +1876,7 @@ var bindTests = []struct {
`NewErrors`,
`
pragma solidity >0.8.4;
contract NewErrors {
error MyError(uint256);
error MyError1(uint256);
@ -1878,7 +1892,7 @@ var bindTests = []struct {
`
"context"
"math/big"
"github.com/ethereum/go-ethereum/accounts/abi/bind"
"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
"github.com/ethereum/go-ethereum/core/types"
@ -1892,7 +1906,7 @@ var bindTests = []struct {
sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil)
)
defer sim.Close()
_, tx, contract, err := DeployNewErrors(user, sim)
if err != nil {
t.Fatal(err)
@ -1917,12 +1931,12 @@ var bindTests = []struct {
name: `ConstructorWithStructParam`,
contract: `
pragma solidity >=0.8.0 <0.9.0;
contract ConstructorWithStructParam {
struct StructType {
uint256 field;
}
constructor(StructType memory st) {}
}
`,
@ -1951,7 +1965,7 @@ var bindTests = []struct {
t.Fatalf("DeployConstructorWithStructParam() got err %v; want nil err", err)
}
sim.Commit()
if _, err = bind.WaitDeployed(context.Background(), sim, tx); err != nil {
t.Logf("Deployment tx: %+v", tx)
t.Errorf("bind.WaitDeployed(nil, %T, <deployment tx>) got err %v; want nil err", sim, err)
@ -2000,7 +2014,7 @@ var bindTests = []struct {
t.Fatalf("DeployNameConflict() got err %v; want nil err", err)
}
sim.Commit()
if _, err = bind.WaitDeployed(context.Background(), sim, tx); err != nil {
t.Logf("Deployment tx: %+v", tx)
t.Errorf("bind.WaitDeployed(nil, %T, <deployment tx>) got err %v; want nil err", sim, err)

116
accounts/abi/abigen/testdata/v2/structs-abi.go.txt vendored
View file

@ -1,116 +0,0 @@
// Code generated via abigen V2 - DO NOT EDIT.
// This file is a generated binding and any manual changes will be lost.
package v1bindtests
import (
"bytes"
"errors"
"math/big"
"github.com/ethereum/go-ethereum/accounts/abi"
"github.com/ethereum/go-ethereum/accounts/abi/bind/v2"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
// Reference imports to suppress errors if they are not otherwise used.
var (
_ = bytes.Equal
_ = errors.New
_ = big.NewInt
_ = common.Big1
_ = types.BloomLookup
_ = abi.ConvertType
)
// Struct0 is an auto generated low-level Go binding around an user-defined struct.
type Struct0 struct {
B [32]byte
}
// StructsMetaData contains all meta data concerning the Structs contract.
var StructsMetaData = bind.MetaData{
ABI: "[{\"inputs\":[],\"name\":\"F\",\"outputs\":[{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"B\",\"type\":\"bytes32\"}],\"internalType\":\"structStructs.A[]\",\"name\":\"a\",\"type\":\"tuple[]\"},{\"internalType\":\"uint256[]\",\"name\":\"c\",\"type\":\"uint256[]\"},{\"internalType\":\"bool[]\",\"name\":\"d\",\"type\":\"bool[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"G\",\"outputs\":[{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"B\",\"type\":\"bytes32\"}],\"internalType\":\"structStructs.A[]\",\"name\":\"a\",\"type\":\"tuple[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"}]",
ID: "Structs",
}
// Structs is an auto generated Go binding around an Ethereum contract.
type Structs struct {
abi abi.ABI
}
// NewStructs creates a new instance of Structs.
func NewStructs() *Structs {
parsed, err := StructsMetaData.ParseABI()
if err != nil {
panic(errors.New("invalid ABI: " + err.Error()))
}
return &Structs{abi: *parsed}
}
// Instance creates a wrapper for a deployed contract instance at the given address.
<<<<<<< HEAD
// Use this to create the instance object passed to abigen v2 library functions Call, Transact, etc.
func (c *Structs) Instance(backend bind.ContractBackend, addr common.Address) *bind.BoundContract {
return bind.NewBoundContract(addr, c.abi, backend, backend, backend)
=======
// Use this to create the instance object passed to abigen v2 library functions Call,
// Transact, etc.
func (c *Structs) Instance(backend bind.ContractBackend, addr common.Address) bind.BoundContract {
return bind.NewBoundContract(backend, addr, c.abi)
>>>>>>> 854c25e086 (accounts/abi/abigen: improve v2 template)
}
// F is the Go binding used to pack the parameters required for calling
// the contract method 0x28811f59.
//
// Solidity: function F() view returns((bytes32)[] a, uint256[] c, bool[] d)
func (structs *Structs) PackF() ([]byte, error) {
return structs.abi.Pack("F")
}
// FOutput serves as a container for the return parameters of contract
// method F.
type FOutput struct {
A []Struct0
C []*big.Int
D []bool
}
// UnpackF is the Go binding that unpacks the parameters returned
// from invoking the contract method with ID 0x28811f59.
//
// Solidity: function F() view returns((bytes32)[] a, uint256[] c, bool[] d)
func (structs *Structs) UnpackF(data []byte) (*FOutput, error) {
out, err := structs.abi.Unpack("F", data)
if err != nil {
return nil, err
}
ret := new(FOutput)
ret.A = *abi.ConvertType(out[0], new([]Struct0)).(*[]Struct0)
ret.C = *abi.ConvertType(out[1], new([]*big.Int)).(*[]*big.Int)
ret.D = *abi.ConvertType(out[2], new([]bool)).(*[]bool)
return ret, nil
}
// G is the Go binding used to pack the parameters required for calling
// the contract method 0x6fecb623.
//
// Solidity: function G() view returns((bytes32)[] a)
func (structs *Structs) PackG() ([]byte, error) {
return structs.abi.Pack("G")
}
// UnpackG is the Go binding that unpacks the parameters returned
// from invoking the contract method with ID 0x6fecb623.
//
// Solidity: function G() view returns((bytes32)[] a)
func (structs *Structs) UnpackG(data []byte) (*[]Struct0, error) {
out, err := structs.abi.Unpack("G", data)
if err != nil {
return nil, err
}
out0 := *abi.ConvertType(out[0], new([]Struct0)).(*[]Struct0)
return &out0, nil
}

15
accounts/abi/bind/v2/base.go
View file

@ -383,13 +383,14 @@ func (c *BoundContract) estimateGasLimit(opts *TransactOpts, contract *common.Ad
}
}
msg := ethereum.CallMsg{
From: opts.From,
To: contract,
GasPrice: gasPrice,
GasTipCap: gasTipCap,
GasFeeCap: gasFeeCap,
Value: value,
Data: input,
From: opts.From,
To: contract,
GasPrice: gasPrice,
GasTipCap: gasTipCap,
GasFeeCap: gasFeeCap,
Value: value,
Data: input,
AccessList: opts.AccessList,
}
return c.transactor.EstimateGas(ensureContext(opts.Context), msg)
}

19
accounts/abi/error_handling.go
View file

@ -23,15 +23,16 @@ import (
)
var (
errBadBool = errors.New("abi: improperly encoded boolean value")
errBadUint8 = errors.New("abi: improperly encoded uint8 value")
errBadUint16 = errors.New("abi: improperly encoded uint16 value")
errBadUint32 = errors.New("abi: improperly encoded uint32 value")
errBadUint64 = errors.New("abi: improperly encoded uint64 value")
errBadInt8 = errors.New("abi: improperly encoded int8 value")
errBadInt16 = errors.New("abi: improperly encoded int16 value")
errBadInt32 = errors.New("abi: improperly encoded int32 value")
errBadInt64 = errors.New("abi: improperly encoded int64 value")
errBadBool = errors.New("abi: improperly encoded boolean value")
errBadUint8 = errors.New("abi: improperly encoded uint8 value")
errBadUint16 = errors.New("abi: improperly encoded uint16 value")
errBadUint32 = errors.New("abi: improperly encoded uint32 value")
errBadUint64 = errors.New("abi: improperly encoded uint64 value")
errBadInt8 = errors.New("abi: improperly encoded int8 value")
errBadInt16 = errors.New("abi: improperly encoded int16 value")
errBadInt32 = errors.New("abi: improperly encoded int32 value")
errBadInt64 = errors.New("abi: improperly encoded int64 value")
errInvalidSign = errors.New("abi: negatively-signed value cannot be packed into uint parameter")
)
// formatSliceString formats the reflection kind with the given slice size

11
accounts/abi/pack.go
View file

@ -37,7 +37,16 @@ func packBytesSlice(bytes []byte, l int) []byte {
// t.
func packElement(t Type, reflectValue reflect.Value) ([]byte, error) {
switch t.T {
case IntTy, UintTy:
case UintTy:
// make sure to not pack a negative value into a uint type.
if reflectValue.Kind() == reflect.Ptr {
val := new(big.Int).Set(reflectValue.Interface().(*big.Int))
if val.Sign() == -1 {
return nil, errInvalidSign
}
}
return packNum(reflectValue), nil
case IntTy:
return packNum(reflectValue), nil
case StringTy:
return packBytesSlice([]byte(reflectValue.String()), reflectValue.Len()), nil

5
accounts/abi/pack_test.go
View file

@ -177,6 +177,11 @@ func TestMethodPack(t *testing.T) {
if !bytes.Equal(packed, sig) {
t.Errorf("expected %x got %x", sig, packed)
}
// test that we can't pack a negative value for a parameter that is specified as a uint
if _, err := abi.Pack("send", big.NewInt(-1)); err == nil {
t.Fatal("expected error when trying to pack negative big.Int into uint256 value")
}
}
func TestPackNumber(t *testing.T) {

66
accounts/abi/unpack_test.go
View file

@ -1014,128 +1014,134 @@ func TestPackAndUnpackIncompatibleNumber(t *testing.T) {
cases := []struct {
decodeType string
inputValue *big.Int
err error
unpackErr error
packErr error
expectValue interface{}
}{
{
decodeType: "uint8",
inputValue: big.NewInt(math.MaxUint8 + 1),
err: errBadUint8,
unpackErr: errBadUint8,
},
{
decodeType: "uint8",
inputValue: big.NewInt(math.MaxUint8),
err: nil,
unpackErr: nil,
expectValue: uint8(math.MaxUint8),
},
{
decodeType: "uint16",
inputValue: big.NewInt(math.MaxUint16 + 1),
err: errBadUint16,
unpackErr: errBadUint16,
},
{
decodeType: "uint16",
inputValue: big.NewInt(math.MaxUint16),
err: nil,
unpackErr: nil,
expectValue: uint16(math.MaxUint16),
},
{
decodeType: "uint32",
inputValue: big.NewInt(math.MaxUint32 + 1),
err: errBadUint32,
unpackErr: errBadUint32,
},
{
decodeType: "uint32",
inputValue: big.NewInt(math.MaxUint32),
err: nil,
unpackErr: nil,
expectValue: uint32(math.MaxUint32),
},
{
decodeType: "uint64",
inputValue: maxU64Plus1,
err: errBadUint64,
unpackErr: errBadUint64,
},
{
decodeType: "uint64",
inputValue: maxU64,
err: nil,
unpackErr: nil,
expectValue: uint64(math.MaxUint64),
},
{
decodeType: "uint256",
inputValue: maxU64Plus1,
err: nil,
unpackErr: nil,
expectValue: maxU64Plus1,
},
{
decodeType: "int8",
inputValue: big.NewInt(math.MaxInt8 + 1),
err: errBadInt8,
unpackErr: errBadInt8,
},
{
decodeType: "int8",
inputValue: big.NewInt(math.MinInt8 - 1),
err: errBadInt8,
packErr: errInvalidSign,
},
{
decodeType: "int8",
inputValue: big.NewInt(math.MaxInt8),
err: nil,
unpackErr: nil,
expectValue: int8(math.MaxInt8),
},
{
decodeType: "int16",
inputValue: big.NewInt(math.MaxInt16 + 1),
err: errBadInt16,
unpackErr: errBadInt16,
},
{
decodeType: "int16",
inputValue: big.NewInt(math.MinInt16 - 1),
err: errBadInt16,
packErr: errInvalidSign,
},
{
decodeType: "int16",
inputValue: big.NewInt(math.MaxInt16),
err: nil,
unpackErr: nil,
expectValue: int16(math.MaxInt16),
},
{
decodeType: "int32",
inputValue: big.NewInt(math.MaxInt32 + 1),
err: errBadInt32,
unpackErr: errBadInt32,
},
{
decodeType: "int32",
inputValue: big.NewInt(math.MinInt32 - 1),
err: errBadInt32,
packErr: errInvalidSign,
},
{
decodeType: "int32",
inputValue: big.NewInt(math.MaxInt32),
err: nil,
unpackErr: nil,
expectValue: int32(math.MaxInt32),
},
{
decodeType: "int64",
inputValue: new(big.Int).Add(big.NewInt(math.MaxInt64), big.NewInt(1)),
err: errBadInt64,
unpackErr: errBadInt64,
},
{
decodeType: "int64",
inputValue: new(big.Int).Sub(big.NewInt(math.MinInt64), big.NewInt(1)),
err: errBadInt64,
packErr: errInvalidSign,
},
{
decodeType: "int64",
inputValue: big.NewInt(math.MaxInt64),
err: nil,
unpackErr: nil,
expectValue: int64(math.MaxInt64),
},
}
for i, testCase := range cases {
packed, err := encodeABI.Pack(testCase.inputValue)
if err != nil {
panic(err)
if testCase.packErr != nil {
if err == nil {
t.Fatalf("expected packing of testcase input value to fail")
}
if err != testCase.packErr {
t.Fatalf("expected error '%v', got '%v'", testCase.packErr, err)
}
continue
}
if err != nil && err != testCase.packErr {
panic(fmt.Errorf("unexpected error packing test-case input: %v", err))
}
ty, err := NewType(testCase.decodeType, "", nil)
if err != nil {
@ -1145,8 +1151,8 @@ func TestPackAndUnpackIncompatibleNumber(t *testing.T) {
{Type: ty},
}
decoded, err := decodeABI.Unpack(packed)
if err != testCase.err {
t.Fatalf("Expected error %v, actual error %v. case %d", testCase.err, err, i)
if err != testCase.unpackErr {
t.Fatalf("Expected error %v, actual error %v. case %d", testCase.unpackErr, err, i)
}
if err != nil {
continue

13
accounts/usbwallet/trezor.go
View file

@ -25,6 +25,7 @@ import (
"errors"
"fmt"
"io"
"math"
"math/big"
"github.com/ethereum/go-ethereum/accounts"
@ -249,7 +250,11 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
}
}
// Extract the Ethereum signature and do a sanity validation
if len(response.GetSignatureR()) == 0 || len(response.GetSignatureS()) == 0 || response.GetSignatureV() == 0 {
if len(response.GetSignatureR()) == 0 || len(response.GetSignatureS()) == 0 {
return common.Address{}, nil, errors.New("reply lacks signature")
} else if response.GetSignatureV() == 0 && int(chainID.Int64()) <= (math.MaxUint32-36)/2 {
// for chainId >= (MaxUint32-36)/2, Trezor returns signature bit only
// https://github.com/trezor/trezor-mcu/pull/399
return common.Address{}, nil, errors.New("reply lacks signature")
}
signature := append(append(response.GetSignatureR(), response.GetSignatureS()...), byte(response.GetSignatureV()))
@ -261,7 +266,11 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
} else {
// Trezor backend does not support typed transactions yet.
signer = types.NewEIP155Signer(chainID)
signature[64] -= byte(chainID.Uint64()*2 + 35)
// if chainId is above (MaxUint32 - 36) / 2 then the final v values is returned
// directly. Otherwise, the returned value is 35 + chainid * 2.
if signature[64] > 1 && int(chainID.Int64()) <= (math.MaxUint32-36)/2 {
signature[64] -= byte(chainID.Uint64()*2 + 35)
}
}
// Inject the final signature into the transaction and sanity check the sender

10
beacon/blsync/client.go
View file

@ -23,9 +23,11 @@ import (
"github.com/ethereum/go-ethereum/beacon/light/sync"
"github.com/ethereum/go-ethereum/beacon/params"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
)
@ -46,7 +48,13 @@ func NewClient(config params.ClientConfig) *Client {
var (
db = memorydb.New()
committeeChain = light.NewCommitteeChain(db, &config.ChainConfig, config.Threshold, !config.NoFilter)
headTracker = light.NewHeadTracker(committeeChain, config.Threshold)
headTracker = light.NewHeadTracker(committeeChain, config.Threshold, func(checkpoint common.Hash) {
if saved, err := config.SaveCheckpointToFile(checkpoint); saved {
log.Debug("Saved beacon checkpoint", "file", config.CheckpointFile, "checkpoint", checkpoint)
} else if err != nil {
log.Error("Failed to save beacon checkpoint", "file", config.CheckpointFile, "checkpoint", checkpoint, "error", err)
}
})
)
headSync := sync.NewHeadSync(headTracker, committeeChain)

7
beacon/blsync/engineclient.go
View file

@ -26,6 +26,7 @@ import (
"github.com/ethereum/go-ethereum/beacon/params"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
ctypes "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
@ -104,7 +105,11 @@ func (ec *engineClient) callNewPayload(fork string, event types.ChainHeadEvent)
method = "engine_newPayloadV4"
parentBeaconRoot := event.BeaconHead.ParentRoot
blobHashes := collectBlobHashes(event.Block)
params = append(params, blobHashes, parentBeaconRoot, event.ExecRequests)
hexRequests := make([]hexutil.Bytes, len(event.ExecRequests))
for i := range event.ExecRequests {
hexRequests[i] = hexutil.Bytes(event.ExecRequests[i])
}
params = append(params, blobHashes, parentBeaconRoot, hexRequests)
case "deneb":
method = "engine_newPayloadV3"
parentBeaconRoot := event.BeaconHead.ParentRoot

11
beacon/engine/types.go
View file

@ -123,6 +123,11 @@ type BlobAndProofV1 struct {
Proof hexutil.Bytes `json:"proof"`
}
type BlobAndProofV2 struct {
Blob hexutil.Bytes `json:"blob"`
CellProofs []hexutil.Bytes `json:"proofs"`
}
// JSON type overrides for ExecutionPayloadEnvelope.
type executionPayloadEnvelopeMarshaling struct {
BlockValue *hexutil.Big
@ -131,7 +136,7 @@ type executionPayloadEnvelopeMarshaling struct {
type PayloadStatusV1 struct {
Status string `json:"status"`
Witness *hexutil.Bytes `json:"witness"`
Witness *hexutil.Bytes `json:"witness,omitempty"`
LatestValidHash *common.Hash `json:"latestValidHash"`
ValidationError *string `json:"validationError"`
}
@ -331,7 +336,9 @@ func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types.
for j := range sidecar.Blobs {
bundle.Blobs = append(bundle.Blobs, hexutil.Bytes(sidecar.Blobs[j][:]))
bundle.Commitments = append(bundle.Commitments, hexutil.Bytes(sidecar.Commitments[j][:]))
bundle.Proofs = append(bundle.Proofs, hexutil.Bytes(sidecar.Proofs[j][:]))
}
for _, proof := range sidecar.Proofs {
bundle.Proofs = append(bundle.Proofs, hexutil.Bytes(proof[:]))
}
}

56
beacon/engine/types_test.go Normal file
View file

@ -0,0 +1,56 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package engine
import (
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
func TestBlobs(t *testing.T) {
var (
emptyBlob = new(kzg4844.Blob)
emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob)
emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit)
emptyCellProof, _ = kzg4844.ComputeCellProofs(emptyBlob)
)
header := types.Header{}
block := types.NewBlock(&header, &types.Body{}, nil, nil)
sidecarWithoutCellProofs := &types.BlobTxSidecar{
Blobs: []kzg4844.Blob{*emptyBlob},
Commitments: []kzg4844.Commitment{emptyBlobCommit},
Proofs: []kzg4844.Proof{emptyBlobProof},
}
env := BlockToExecutableData(block, common.Big0, []*types.BlobTxSidecar{sidecarWithoutCellProofs}, nil)
if len(env.BlobsBundle.Proofs) != 1 {
t.Fatalf("Expect 1 proof in blobs bundle, got %v", len(env.BlobsBundle.Proofs))
}
sidecarWithCellProofs := &types.BlobTxSidecar{
Blobs: []kzg4844.Blob{*emptyBlob},
Commitments: []kzg4844.Commitment{emptyBlobCommit},
Proofs: emptyCellProof,
}
env = BlockToExecutableData(block, common.Big0, []*types.BlobTxSidecar{sidecarWithCellProofs}, nil)
if len(env.BlobsBundle.Proofs) != 128 {
t.Fatalf("Expect 128 proofs in blobs bundle, got %v", len(env.BlobsBundle.Proofs))
}
}

9
beacon/light/head_tracker.go
View file

@ -21,7 +21,9 @@ import (
"sync"
"time"
"github.com/ethereum/go-ethereum/beacon/params"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
)
@ -38,13 +40,15 @@ type HeadTracker struct {
hasFinalityUpdate bool
prefetchHead types.HeadInfo
changeCounter uint64
saveCheckpoint func(common.Hash)
}
// NewHeadTracker creates a new HeadTracker.
func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int) *HeadTracker {
func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int, saveCheckpoint func(common.Hash)) *HeadTracker {
return &HeadTracker{
committeeChain: committeeChain,
minSignerCount: minSignerCount,
saveCheckpoint: saveCheckpoint,
}
}
@ -100,6 +104,9 @@ func (h *HeadTracker) ValidateFinality(update types.FinalityUpdate) (bool, error
if replace {
h.finalityUpdate, h.hasFinalityUpdate = update, true
h.changeCounter++
if h.saveCheckpoint != nil && update.Finalized.Slot%params.EpochLength == 0 {
h.saveCheckpoint(update.Finalized.Hash())
}
}
return replace, err
}

1
beacon/params/checkpoint_holesky.hex Normal file
View file

@ -0,0 +1 @@
0xd60e5310c5d52ced44cfb13be4e9f22a1e6a6dc56964c3cccd429182d26d72d0

1
beacon/params/checkpoint_mainnet.hex Normal file
View file

@ -0,0 +1 @@
0x02f0bb348b0d45f95a9b7e2bb5705768ad06548876cee03d880a2c9dabb1ff88

1
beacon/params/checkpoint_sepolia.hex Normal file
View file

@ -0,0 +1 @@
0xa0dad451a230c01be6f2492980ec5bb412d8cf33351a75e8b172b5b84a5fd03a

34
beacon/params/config.go
View file

@ -54,6 +54,7 @@ type ChainConfig struct {
GenesisValidatorsRoot common.Hash // Root hash of the genesis validator set, used for signature domain calculation
Forks Forks
Checkpoint common.Hash
CheckpointFile string
}
// ForkAtEpoch returns the latest active fork at the given epoch.
@ -211,3 +212,36 @@ func (f Forks) Less(i, j int) bool {
}
return f[i].knownIndex < f[j].knownIndex
}
// SetCheckpointFile sets the checkpoint import/export file name and attempts to
// read the checkpoint from the file if it already exists. It returns true if
// a checkpoint has been loaded.
func (c *ChainConfig) SetCheckpointFile(checkpointFile string) (bool, error) {
c.CheckpointFile = checkpointFile
file, err := os.ReadFile(checkpointFile)
if os.IsNotExist(err) {
return false, nil // did not load checkpoint
}
if err != nil {
return false, fmt.Errorf("failed to read beacon checkpoint file: %v", err)
}
cp, err := hexutil.Decode(string(file))
if err != nil {
return false, fmt.Errorf("failed to decode hex string in beacon checkpoint file: %v", err)
}
if len(cp) != 32 {
return false, fmt.Errorf("invalid hex string length in beacon checkpoint file: %d", len(cp))
}
copy(c.Checkpoint[:len(cp)], cp)
return true, nil
}
// SaveCheckpointToFile saves the given checkpoint to file if a checkpoint
// import/export file has been specified.
func (c *ChainConfig) SaveCheckpointToFile(checkpoint common.Hash) (bool, error) {
if c.CheckpointFile == "" {
return false, nil
}
err := os.WriteFile(c.CheckpointFile, []byte(checkpoint.Hex()), 0600)
return err == nil, err
}

20
beacon/params/networks.go
View file

@ -17,25 +17,37 @@
package params
import (
_ "embed"
"github.com/ethereum/go-ethereum/common"
)
//go:embed checkpoint_mainnet.hex
var checkpointMainnet string
//go:embed checkpoint_sepolia.hex
var checkpointSepolia string
//go:embed checkpoint_holesky.hex
var checkpointHolesky string
var (
MainnetLightConfig = (&ChainConfig{
GenesisValidatorsRoot: common.HexToHash("0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95"),
GenesisTime: 1606824023,
Checkpoint: common.HexToHash("0x6509b691f4de4f7b083f2784938fd52f0e131675432b3fd85ea549af9aebd3d0"),
Checkpoint: common.HexToHash(checkpointMainnet),
}).
AddFork("GENESIS", 0, []byte{0, 0, 0, 0}).
AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}).
AddFork("BELLATRIX", 144896, []byte{2, 0, 0, 0}).
AddFork("CAPELLA", 194048, []byte{3, 0, 0, 0}).
AddFork("DENEB", 269568, []byte{4, 0, 0, 0})
AddFork("DENEB", 269568, []byte{4, 0, 0, 0}).
AddFork("ELECTRA", 364032, []byte{5, 0, 0, 0})
SepoliaLightConfig = (&ChainConfig{
GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"),
GenesisTime: 1655733600,
Checkpoint: common.HexToHash("0x456e85f5608afab3465a0580bff8572255f6d97af0c5f939e3f7536b5edb2d3f"),
Checkpoint: common.HexToHash(checkpointSepolia),
}).
AddFork("GENESIS", 0, []byte{144, 0, 0, 105}).
AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}).
@ -47,7 +59,7 @@ var (
HoleskyLightConfig = (&ChainConfig{
GenesisValidatorsRoot: common.HexToHash("0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1"),
GenesisTime: 1695902400,
Checkpoint: common.HexToHash("0x6456a1317f54d4b4f2cb5bc9d153b5af0988fe767ef0609f0236cf29030bcff7"),
Checkpoint: common.HexToHash(checkpointHolesky),
}).
AddFork("GENESIS", 0, []byte{1, 1, 112, 0}).
AddFork("ALTAIR", 0, []byte{2, 1, 112, 0}).

206
build/checksums.txt
View file

@ -1,113 +1,113 @@
# This file contains sha256 checksums of optional build dependencies.
# version:spec-tests pectra-devnet-6@v1.0.0
# version:spec-tests v4.5.0
# https://github.com/ethereum/execution-spec-tests/releases
# https://github.com/ethereum/execution-spec-tests/releases/download/pectra-devnet-6%40v1.0.0/
b69211752a3029083c020dc635fe12156ca1a6725a08559da540a0337586a77e fixtures_pectra-devnet-6.tar.gz
# https://github.com/ethereum/execution-spec-tests/releases/download/v4.5.0/
58afb92a0075a2cb7c4dec1281f7cb88b21b02afbedad096b580f3f8cc14c54c fixtures_develop.tar.gz
# version:golang 1.24.1
# version:golang 1.24.3
# https://go.dev/dl/
8244ebf46c65607db10222b5806aeb31c1fcf8979c1b6b12f60c677e9a3c0656 go1.24.1.src.tar.gz
8d627dc163a4bffa2b1887112ad6194af175dce108d606ed1714a089fb806033 go1.24.1.aix-ppc64.tar.gz
addbfce2056744962e2d7436313ab93486660cf7a2e066d171b9d6f2da7c7abe go1.24.1.darwin-amd64.tar.gz
58d529334561cff11087cd4ab18fe0b46d8d5aad88f45c02b9645f847e014512 go1.24.1.darwin-amd64.pkg
295581b5619acc92f5106e5bcb05c51869337eb19742fdfa6c8346c18e78ff88 go1.24.1.darwin-arm64.tar.gz
78b0fc8ddc344eb499f1a952c687cb84cbd28ba2b739cfa0d4eb042f07e44e82 go1.24.1.darwin-arm64.pkg
e70053f56f7eb93806d80cbd5726f78509a0a467602f7bea0e2c4ee8ed7c3968 go1.24.1.dragonfly-amd64.tar.gz
3595e2674ed8fe72e604ca59c964d3e5277aafb08475c2b1aaca2d2fd69c24fc go1.24.1.freebsd-386.tar.gz
47d7de8bb64d5c3ee7b6723aa62d5ecb11e3568ef2249bbe1d4bbd432d37c00c go1.24.1.freebsd-amd64.tar.gz
04eec3bcfaa14c1370cdf98e8307fac7e4853496c3045afb9c3124a29cbca205 go1.24.1.freebsd-arm.tar.gz
51aa70146e40cfdc20927424083dc86e6223f85dc08089913a1651973b55665b go1.24.1.freebsd-arm64.tar.gz
3c131d8e3fc285a1340f87813153e24226d3ddbd6e54f3facbd6e4c46a84655e go1.24.1.freebsd-riscv64.tar.gz
201d09da737ba39d5367f87d4e8b31edaeeb3dc9b9c407cb8cfb40f90c5a727a go1.24.1.illumos-amd64.tar.gz
8c530ecedbc17e42ce10177bea07ccc96a3e77c792ea1ea72173a9675d16ffa5 go1.24.1.linux-386.tar.gz
cb2396bae64183cdccf81a9a6df0aea3bce9511fc21469fb89a0c00470088073 go1.24.1.linux-amd64.tar.gz
8df5750ffc0281017fb6070fba450f5d22b600a02081dceef47966ffaf36a3af go1.24.1.linux-arm64.tar.gz
6d95f8d7884bfe2364644c837f080f2b585903d0b771eb5b06044e226a4f120a go1.24.1.linux-armv6l.tar.gz
19304a4a56e46d04604547d2d83235dc4f9b192c79832560ce337d26cc7b835a go1.24.1.linux-loong64.tar.gz
6347be77fa5359c12a5308c8ab87147c1fc4717b0c216493d1706c3b9fcde22d go1.24.1.linux-mips.tar.gz
1647df415f7030b82d4105670192aa7e8910e18563bb0d505192d72800cc2d21 go1.24.1.linux-mips64.tar.gz
762da594e4ec0f9cf6defae6ef971f5f7901203ee6a2d979e317adec96657317 go1.24.1.linux-mips64le.tar.gz
9d8133c7b23a557399fab870b5cf464079c2b623a43b214a7567cf11c254a444 go1.24.1.linux-mipsle.tar.gz
132f10999abbaccbada47fa85462db30c423955913b14d6c692de25f4636c766 go1.24.1.linux-ppc64.tar.gz
0fb522efcefabae6e37e69bdc444094e75bfe824ea6d4cc3cbc70c7ae1b16858 go1.24.1.linux-ppc64le.tar.gz
eaef4323d5467ff97fb1979c8491764060dade19f02f3275a9313f9a0da3b9c0 go1.24.1.linux-riscv64.tar.gz
6c05e14d8f11094cb56a1c50f390b6b658bed8a7cbd8d1a57e926581b7eabfce go1.24.1.linux-s390x.tar.gz
5dbb287d343ea00d58a70b11629f32ee716dc50a6875c459ea2018df0f294cd8 go1.24.1.netbsd-386.tar.gz
617aa3faee50ce84c343db0888e9a210c310a7203666b4ed620f31030c9fb32f go1.24.1.netbsd-amd64.tar.gz
59a928b7080c4a6ac985946274b7c65ce1cecc0b468ecd992d17b7c12fab9296 go1.24.1.netbsd-arm.tar.gz
28daa8d0feb4aef2af60cefa3305bb9314de7e8a05cbca41ac548964cdfe89b7 go1.24.1.netbsd-arm64.tar.gz
b7382b2f5d99813aeac14db482faa3bfbd47a68880b607fa2a7e669e26bab9cd go1.24.1.openbsd-386.tar.gz
2513b6537c45deead5e641c7ce7502913e7d5e6f0b21c52542fb11f81578690f go1.24.1.openbsd-amd64.tar.gz
853c1917d4fc7b144c55a02842aa48542d5cc798dde8db96dc0fdbc263200e04 go1.24.1.openbsd-arm.tar.gz
6bc207b91e6f6ae3347fb54616a8fb2f5c11983713846a4cef111ff3f4f94d14 go1.24.1.openbsd-arm64.tar.gz
4279260e2f2b94ee94e81470d13db7367f4393b061fee60985528fa0fa430df4 go1.24.1.openbsd-ppc64.tar.gz
6fc4023a0a339ee0778522364a127d94c78e62122288d47d820dba703f81dc07 go1.24.1.openbsd-riscv64.tar.gz
b5eb9fafd77146e7e1f748acfd95559580ecc8d2f15abf432a20f58c929c7cd2 go1.24.1.plan9-386.tar.gz
24dcad6361b141fc8cced15b092351e12a99d2e58d7013204a3013c50daf9fdd go1.24.1.plan9-amd64.tar.gz
a026ac3b55aa1e6fdc2aaab30207a117eafbe965ed81d3aa0676409f280ddc37 go1.24.1.plan9-arm.tar.gz
8e4f6a77388dc6e5aa481efd5abdb3b9f5c9463bb82f4db074494e04e5c84992 go1.24.1.solaris-amd64.tar.gz
b799f4ab264eef12a014c759383ed934056608c483e0f73e34ea6caf9f1df5f9 go1.24.1.windows-386.zip
db128981033ac82a64688a123f631e61297b6b8f52ca913145e57caa8ce94cc3 go1.24.1.windows-386.msi
95666b551453209a2b8869d29d177285ff9573af10f085d961d7ae5440f645ce go1.24.1.windows-amd64.zip
5968e7adcf26e68a54f1cd41ad561275a670a8e2ca5263bc375b524638557dfb go1.24.1.windows-amd64.msi
e28c4e6d0b913955765b46157ab88ae59bb636acaa12d7bec959aa6900f1cebd go1.24.1.windows-arm64.zip
6d352c1f154a102a5b90c480cc64bab205ccf2681e34e78a3a4d3f1ddfbc81e4 go1.24.1.windows-arm64.msi
229c08b600b1446798109fae1f569228102c8473caba8104b6418cb5bc032878 go1.24.3.src.tar.gz
6f6901497547db3b77c14f7f953fbcef9fa5fb84199ee2ee14a5686e66bed5a6 go1.24.3.aix-ppc64.tar.gz
a05fa7e4043a4fec66897135219e3b8ab2202b5ef351c60c2fbb531dfb8f2900 go1.24.3.darwin-amd64.pkg
13e6fe3fcf65689d77d40e633de1e31c6febbdbcb846eb05fc2434ed2213e92b go1.24.3.darwin-amd64.tar.gz
97055ff4214043b39dc32e043fdd5c565df7c0a4e2fc0174e779a134c347ae0e go1.24.3.darwin-arm64.pkg
64a3fa22142f627e78fac3018ce3d4aeace68b743eff0afda8aae0411df5e4fb go1.24.3.darwin-arm64.tar.gz
32de3fd44d5055973978436a7f1f0ffbaae85c1b603ec6105e5c38d8a674c721 go1.24.3.dragonfly-amd64.tar.gz
9fe6101b3797919bd7337ee5ce591954f85d59db7ae88983904db29fd64c3dd1 go1.24.3.freebsd-386.tar.gz
6ccf4cca287e90cc28cd7954b6172f5d177a17e20b072b65f7f39636c325e2fb go1.24.3.freebsd-amd64.tar.gz
ce45ebf389066f82a7b056b66dd650efb51fde6f8bf92a2a3ab6990f02788ebf go1.24.3.freebsd-arm.tar.gz
8f6494a12a874d0ea57c67987829359e016960ce3ba0673273609d6ac2af589a go1.24.3.freebsd-arm64.tar.gz
f9db392560cf0851f0bc8f2190e1978e01b4603038c27fecfc8658a695b71616 go1.24.3.freebsd-riscv64.tar.gz
01717fff64c5d98457272002fa825d0a15e307bf6e189f2b0c23817fa033b61c go1.24.3.illumos-amd64.tar.gz
41b1051063e68cbd2b919bf12326764fe33937cf1d32b5c529dd1a4f43dce578 go1.24.3.linux-386.tar.gz
3333f6ea53afa971e9078895eaa4ac7204a8c6b5c68c10e6bc9a33e8e391bdd8 go1.24.3.linux-amd64.tar.gz
a463cb59382bd7ae7d8f4c68846e73c4d589f223c589ac76871b66811ded7836 go1.24.3.linux-arm64.tar.gz
17a392d7e826625dd12a32099df0b00b85c32d8132ed86fe917183ee5c3f88ed go1.24.3.linux-armv6l.tar.gz
e4b003c04c902edc140153d279b42167f1ad7c229f48f1f729bbef5e65e88d1f go1.24.3.linux-loong64.tar.gz
1c79d89edf835edf9d4336ccea7cb89bc5c0ca82b12b36b218d599a5400d60fe go1.24.3.linux-mips.tar.gz
0b64fe147d69f4d681d8e8a035c760477531432f83d831f18d37cb9bf3652488 go1.24.3.linux-mips64.tar.gz
396b784c255b64512dc00c302c053e43a3cbfc77518664c6ac5569aafad4d1e6 go1.24.3.linux-mips64le.tar.gz
93898313887f14e8efbe9d7386d5da4792b2d6c492bee562993fd4c9daa75c6d go1.24.3.linux-mipsle.tar.gz
873ae3a6a6655a7b6f820e095d9965507e8dfd3cf76bc92d75c564ecbca385f6 go1.24.3.linux-ppc64.tar.gz
341a749d168f47b1d4dad25e32cae70849b7ceed7c290823b853c9e6b0df0856 go1.24.3.linux-ppc64le.tar.gz
fa482f53ccb4ba280316b8c5751ea67291507280d9166f2a38fe4d9b5d5fb64b go1.24.3.linux-riscv64.tar.gz
a87b0c2a079a0bece1620fb29a00e02b4dba17507850f837e754af7d57cda282 go1.24.3.linux-s390x.tar.gz
63155382308db1306200aff7821aa26bf2a2dda23537dd637a9704b485b6ddf0 go1.24.3.netbsd-386.tar.gz
fe2c5c79482958b867c08a4fc2a10a998de9c0206b08d5b3ebcb2232e8d2777c go1.24.3.netbsd-amd64.tar.gz
e8ff77aef21521b5dd94e44282a3243309b80717414cf12f72835a45886a049f go1.24.3.netbsd-arm.tar.gz
b337fbaf82822685940ffaa76fbcf4be5d2f0258bc819cd80bc408b491f45c04 go1.24.3.netbsd-arm64.tar.gz
c1bb9dd8418480aa7f65452b08de3759da3bf89702be71b5a9fc084836b24ad5 go1.24.3.openbsd-386.tar.gz
531218de748b0caaf6d1ad18921206fc12baaa89bf483a0a5e60a571c206fe6f go1.24.3.openbsd-amd64.tar.gz
bcd0dc959986fc346969b5d4111c3c8031882d8bf8d87a2c2ecf1328962a91f2 go1.24.3.openbsd-arm.tar.gz
00ee6f8f1c41fd2e28ad386bd7e39acce7cab84af6de835855b29d1c597335c4 go1.24.3.openbsd-arm64.tar.gz
9f4ec0a9203ed3c54ce1a2a390ad3d45838cdb7efd85baeff857e37dfde04edd go1.24.3.openbsd-ppc64.tar.gz
da4d6f80e2373250d8c31c32dcd1e08775c327c0d610923604660cc0e07e8cba go1.24.3.openbsd-riscv64.tar.gz
f5d02149132eedda6c2d46b360d7da462b8a5f9e3f8567db100c2d7bff0ddcd7 go1.24.3.plan9-386.tar.gz
175f3d79f4762a3c545d2c6393bf6b8bac24e838026869dafab06b930735c94f go1.24.3.plan9-amd64.tar.gz
d1e4ac15095da1611659261c2228c2058756cf87d61d9fad262f76755ef26849 go1.24.3.plan9-arm.tar.gz
e644220a6ced3c07a7acc1364193cb709a97737dd8b6792a07a8ec6d9996713e go1.24.3.solaris-amd64.tar.gz
0d7e7dc0a31ba0cdd487415709d03b02fc9490ef111e8dfd22788a6d63316f37 go1.24.3.windows-386.msi
c27c463a61ab849266baa0c17a6c5c4256a574ab642f609ba25c96ec965dc184 go1.24.3.windows-386.zip
d5b7637e7e138be877d96a4501709d480e050d86a8f402bc950e72112b5aedc5 go1.24.3.windows-amd64.msi
be9787cb08998b1860fe3513e48a5fe5b96302d358a321b58e651184fa9638b3 go1.24.3.windows-amd64.zip
7efde2e5e8468e9caf2c7fc94f4da78a726a5031a1ed63acff7899527cdddff6 go1.24.3.windows-arm64.msi
eec9fa736056b54dd88ecb669db2bfad39b0c48f6f9080f036dfa1ca42dc4bb5 go1.24.3.windows-arm64.zip
# version:golangci 1.64.6
# version:golangci 2.0.2
# https://github.com/golangci/golangci-lint/releases/
# https://github.com/golangci/golangci-lint/releases/download/v1.64.6/
08f9459e7125fed2612abd71596e04d172695921aff82120d1c1e5e9b6fff2a3 golangci-lint-1.64.6-darwin-amd64.tar.gz
8c10d0c7c3935b8c2269d628b6a06a8f48d8fb4cc31af02fe4ce0cd18b0704c1 golangci-lint-1.64.6-darwin-arm64.tar.gz
c07fcabb55deb8d2c4d390025568e76162d7f91b1d14bd2311be45d8d440a624 golangci-lint-1.64.6-freebsd-386.tar.gz
8ed1ef1102e1a42983ffcaae8e06de6a540334c3a69e205c610b8a7c92d469cd golangci-lint-1.64.6-freebsd-amd64.tar.gz
8f8dda66d1b4a85cc8a1daf1b69af6559c3eb0a41dd8033148a9ad85cfc0e1d9 golangci-lint-1.64.6-freebsd-armv6.tar.gz
59e8ca1fa254661b2a55e96515b14a10cd02221d443054deac5b28c3c3e12d6b golangci-lint-1.64.6-freebsd-armv7.tar.gz
e3d323d5f132e9b7593141bfe1d19c8460e65a55cea1008ec96945fed563f981 golangci-lint-1.64.6-illumos-amd64.tar.gz
5ce910f2a864c5fbeb32a30cbd506e1b2ef215f7a0f422cd5be6370b13db6f03 golangci-lint-1.64.6-linux-386.deb
2caab682a26b9a5fb94ba24e3a7e1404fa9eab2c12e36ae1c5548d80a1be190c golangci-lint-1.64.6-linux-386.rpm
2d82d0a4716e6d9b70c95103054855cb4b5f20f7bbdee42297f0189955bd14b6 golangci-lint-1.64.6-linux-386.tar.gz
9cd82503e9841abcaa57663fc899587fe90363c26d86a793a98d3080fd25e907 golangci-lint-1.64.6-linux-amd64.deb
981aaca5e5202d4fbb162ec7080490eb67ef5d32add5fb62fb02210597acc9da golangci-lint-1.64.6-linux-amd64.rpm
71e290acbacb7b3ba4f68f0540fb74dc180c4336ac8a6f3bdcd7fcc48b15148d golangci-lint-1.64.6-linux-amd64.tar.gz
718016bb06c1f598a8d23c7964e2643de621dbe5808688cb38857eb0bb773c84 golangci-lint-1.64.6-linux-arm64.deb
ddc0e7b4b10b47cf894aea157e89e3674bbb60f8f5c480110c21c49cc2c1634d golangci-lint-1.64.6-linux-arm64.rpm
99a7ff13dec7a8781a68408b6ecb8a1c5e82786cba3189eaa91d5cdcc24362ce golangci-lint-1.64.6-linux-arm64.tar.gz
90e360f89c244394912b8709fb83a900b6d56cf19141df2afaf9e902ef3057b1 golangci-lint-1.64.6-linux-armv6.deb
46546ff7c98d873ffcdbee06b39dc1024fc08db4fbf1f6309360a44cf976b5c2 golangci-lint-1.64.6-linux-armv6.rpm
e45c1a42868aca0b0ee54d14fb89da216f3b4409367cd7bce3b5f36788b4fc92 golangci-lint-1.64.6-linux-armv6.tar.gz
3cf6ddbbbf358db3de4b64a24f9683bbe2da3f003cfdee86cb610124b57abafb golangci-lint-1.64.6-linux-armv7.deb
508b6712710da10f11aab9ea5e63af415c932dfe7fff718e649d3100b838f797 golangci-lint-1.64.6-linux-armv7.rpm
da9a8bbee86b4dfee73fbc17e0856ec84c5b04919eb09bf3dd5904c39ce41753 golangci-lint-1.64.6-linux-armv7.tar.gz
ad496a58284e1e9c8ac6f993fec429dcd96c85a8c4715dbb6530a5af0dae7fbd golangci-lint-1.64.6-linux-loong64.deb
3bd70fa737b224750254dce616d9a188570e49b894b0cdb2fd04155e2c061350 golangci-lint-1.64.6-linux-loong64.rpm
a535af973499729f2215a84303eb0de61399057aad6901ddea1b4f73f68d5f2c golangci-lint-1.64.6-linux-loong64.tar.gz
6ad2a1cd37ca30303a488abfca2de52aff57519901c6d8d2608656fe9fb05292 golangci-lint-1.64.6-linux-mips64.deb
5f18369f0ca010a02c267352ebe6e3e0514c6debff49899c9e5520906c0da287 golangci-lint-1.64.6-linux-mips64.rpm
3449d6c13307b91b0e8817f8911c07c3412cdb6931b8d101e42db3e9762e91ad golangci-lint-1.64.6-linux-mips64.tar.gz
d4712a348f65dcaf9f6c58f1cfd6d0984d54a902873dca5e76f0d686f5c59499 golangci-lint-1.64.6-linux-mips64le.deb
57cea4538894558cb8c956d6b69c5509e4304546abe4a467478fc9ada0c736d6 golangci-lint-1.64.6-linux-mips64le.rpm
bc030977d44535b2152fddb2732f056d193c043992fe638ddecea21a40ef09fe golangci-lint-1.64.6-linux-mips64le.tar.gz
1ceb4e492efa527d246c61798c581f49113756fab8c39bb3eefdb1fa97041f92 golangci-lint-1.64.6-linux-ppc64le.deb
454e1c2b3583a8644e0c33a970fb4ce35b8f11848e1a770d9095d99d159ad0ab golangci-lint-1.64.6-linux-ppc64le.rpm
fddf30d35923eb6c7bb57520d8645768f802bf86c4cbf5a3a13049efb9e71b82 golangci-lint-1.64.6-linux-ppc64le.tar.gz
bd75f0dd6f65bee5821c433803b28f3c427ef3582764c3d0e7f7fae1c9d468b6 golangci-lint-1.64.6-linux-riscv64.deb
58457207c225cbd5340c8dcb75d9a44aa890d604e28464115a3a9762febaed04 golangci-lint-1.64.6-linux-riscv64.rpm
82639518a613a6705b7e2de5b28c278e875d782a5c97e9c1b2ac10b4ecd7852f golangci-lint-1.64.6-linux-riscv64.tar.gz
131d69204f8ced200b1437731e987cc886edac30dc87e6e1dcbd4f833d351713 golangci-lint-1.64.6-linux-s390x.deb
ca6caf28ca7a1df7cff720f8fd6ac4b8f2eac10c8cbfe7d2eade70876aded570 golangci-lint-1.64.6-linux-s390x.rpm
ed966d28a6512bc2b1120029a9f78ed77f2015e330b589a731d67d59be30b0b1 golangci-lint-1.64.6-linux-s390x.tar.gz
b181132b41943fc77ace7f9f5523085d12d3613f27774a0e35e17dd5e65ba589 golangci-lint-1.64.6-netbsd-386.tar.gz
f7b81c426006e3c699dc8665797a462aecba8c2cd23ac4971472e55184d95bc9 golangci-lint-1.64.6-netbsd-amd64.tar.gz
663d6fb4c3bef57b8aacdb1b1a4634075e55d294ed170724b443374860897ca6 golangci-lint-1.64.6-netbsd-arm64.tar.gz
8c7a76ee822568cc19352bbb9b2b0863dc5e185eb07782adbbaf648afd02ebcd golangci-lint-1.64.6-netbsd-armv6.tar.gz
0ce26d56ce78e516529e087118ba3f1bcd71d311b4c5b2bde6ec24a6e8966d85 golangci-lint-1.64.6-netbsd-armv7.tar.gz
135d5d998168683f46e4fab308cef5aa3c282025b7de6b258f976aadfb820db7 golangci-lint-1.64.6-source.tar.gz
ccdb0cc249531a205304a76308cfa7cda830083d083d557884416a2461148263 golangci-lint-1.64.6-windows-386.zip
2d88f282e08e1853c70bc7c914b5f58beaa6509903d56098aeb9bc3df30ea9d5 golangci-lint-1.64.6-windows-amd64.zip
6a3c6e7afc6916392679d7d6b95ac239827644e3e50ec4e8ca6ab1e4e6db65fe golangci-lint-1.64.6-windows-arm64.zip
9c9c1ef9687651566987f93e76252f7526c386901d7d8aeee044ca88115da4b1 golangci-lint-1.64.6-windows-armv6.zip
4f0df114155791799dfde8cd8cb6307fecfb17fed70b44205486ec925e2f39cf golangci-lint-1.64.6-windows-armv7.zip
# https://github.com/golangci/golangci-lint/releases/download/v2.0.2/
a88cbdc86b483fe44e90bf2dcc3fec2af8c754116e6edf0aa6592cac5baa7a0e golangci-lint-2.0.2-darwin-amd64.tar.gz
664550e7954f5f4451aae99b4f7382c1a47039c66f39ca605f5d9af1a0d32b49 golangci-lint-2.0.2-darwin-arm64.tar.gz
bda0f0f27d300502faceda8428834a76ca25986f6d9fc2bd41d201c3ed73f08e golangci-lint-2.0.2-freebsd-386.tar.gz
1cbd0c7ade3fb027d61d38a646ec1b51be5846952b4b04a5330e7f4687f2270c golangci-lint-2.0.2-freebsd-amd64.tar.gz
1e828a597726198b2e35acdbcc5f3aad85244d79846d2d2bdb05241c5a535f9e golangci-lint-2.0.2-freebsd-armv6.tar.gz
848b49315dc5cddd0c9ce35e96ab33d584db0ea8fb57bcbf9784f1622bec0269 golangci-lint-2.0.2-freebsd-armv7.tar.gz
cabf9a6beab574c7f98581eb237919e580024759e3cdc05c4d516b044dce6770 golangci-lint-2.0.2-illumos-amd64.tar.gz
2fde80d15ed6527791f106d606120620e913c3a663c90a8596861d0a4461169e golangci-lint-2.0.2-linux-386.deb
804bc6e350a8c613aaa0a33d8d45414a80157b0ba1b2c2335ac859f85ad98ebd golangci-lint-2.0.2-linux-386.rpm
e64beb72fecf581e57d88ae3adb1c9d4bf022694b6bd92e3c8e460910bbdc37d golangci-lint-2.0.2-linux-386.tar.gz
9c55aed174d7a52bb1d4006b36e7edee9023631f6b814a80cb39c9860d6f75c3 golangci-lint-2.0.2-linux-amd64.deb
c55a2ef741a687b4c679696931f7fd4a467babd64c9457cf17bb9632fd1cecd1 golangci-lint-2.0.2-linux-amd64.rpm
89cc8a7810dc63b9a37900da03e37c3601caf46d42265d774e0f1a5d883d53e2 golangci-lint-2.0.2-linux-amd64.tar.gz
a3e78583c4e7ea1b63e82559f126bb3a5b12788676f158526752d53e67824b99 golangci-lint-2.0.2-linux-arm64.deb
bd5dd52b5c9f18aa7a2904eda9a9f91c628e98623fe70b7afcbb847e2de84422 golangci-lint-2.0.2-linux-arm64.rpm
789d5b91219ac68c2336f77d41cd7e33a910420594780f455893f8453d09595b golangci-lint-2.0.2-linux-arm64.tar.gz
534cd4c464a66178714ed68152c1ed7aa73e5700bf409e4ed1a8363adf96afca golangci-lint-2.0.2-linux-armv6.deb
cf7d02905a5fc80b96c9a64621693b4cc7337b1ce29986c19fd72608dafe66c5 golangci-lint-2.0.2-linux-armv6.rpm
a0d81cb527d8fe878377f2356b5773e219b0b91832a6b59e7b9bcf9a90fe0b0e golangci-lint-2.0.2-linux-armv6.tar.gz
dedd5be7fff8cba8fe15b658a59347ea90d7d02a9fff87f09c7687e6da05a8b6 golangci-lint-2.0.2-linux-armv7.deb
85521b6f3ad2f5a2bc9bfe14b9b08623f764964048f75ed6dfcfaf8eb7d57cc1 golangci-lint-2.0.2-linux-armv7.rpm
96471046c7780dda4ea680f65e92c2ef56ff58d40bcffaf6cfe9d6d48e3c27aa golangci-lint-2.0.2-linux-armv7.tar.gz
815d914a7738e4362466b2d11004e8618b696b49e8ace13df2c2b25f28fb1e17 golangci-lint-2.0.2-linux-loong64.deb
f16381e3d8a0f011b95e086d83d620248432b915d01f4beab4d29cfe4dc388b0 golangci-lint-2.0.2-linux-loong64.rpm
1bd8d7714f9c92db6a0f23bae89f39c85ba047bec8eeb42b8748d30ae3228d18 golangci-lint-2.0.2-linux-loong64.tar.gz
ea6e9d4aabb526aa298e47e8b026d8893d918c5eb919ba0ab403e315def74cc5 golangci-lint-2.0.2-linux-mips64.deb
519d8d53af83fdc9c25cc3fba8b663331ac22ef68131d4b0084cb6f425b6f79a golangci-lint-2.0.2-linux-mips64.rpm
80d655a0a1ac1b19dcef4b58fa2a7dadb646cc50ad08d460b5c53cdb421165e4 golangci-lint-2.0.2-linux-mips64.tar.gz
aa0e75384bb482c865d4dfc95d23ceb25666bf20461b67a832f0eea6670312ec golangci-lint-2.0.2-linux-mips64le.deb
f2a8b500fb69bdea1b01df6267aaa5218fa4a58aeb781c1a20d0d802fe465a52 golangci-lint-2.0.2-linux-mips64le.rpm
e66a0c0c9a275f02d27a7caa9576112622306f001d73dfc082cf1ae446fc1242 golangci-lint-2.0.2-linux-mips64le.tar.gz
e85ad51aac6428be2d8a37000d053697371a538a5bcbc1644caa7c5e77f6d0af golangci-lint-2.0.2-linux-ppc64le.deb
906798365eac1944af2a9b9a303e6fd49ec9043307bc681b7a96277f7f8beea5 golangci-lint-2.0.2-linux-ppc64le.rpm
f7f1a271b0af274d6c9ce000f5dc6e1fb194350c67bcc62494f96f791882ba92 golangci-lint-2.0.2-linux-ppc64le.tar.gz
eea8bf643a42bf05de9780530db22923e5ab0d588f0e173594dc6518f2a25d2a golangci-lint-2.0.2-linux-riscv64.deb
4ff40f9fe2954400836e2a011ba4744d00ffab5068a51368552dfce6aba3b81b golangci-lint-2.0.2-linux-riscv64.rpm
531d8f225866674977d630afbf0533eb02f9bec607fb13895f7a2cd7b2e0a648 golangci-lint-2.0.2-linux-riscv64.tar.gz
6f827647046c603f40d97ea5aadc6f48cd0bb5d19f7a3d56500c3b833d2a0342 golangci-lint-2.0.2-linux-s390x.deb
387a090e9576d19ca86aac738172e58e07c19f2784a13bb387f4f0d75fb9c8d3 golangci-lint-2.0.2-linux-s390x.rpm
57de1fb7722a9feb2d11ed0a007a93959d05b9db5929a392abc222e30012467e golangci-lint-2.0.2-linux-s390x.tar.gz
ed95e0492ea86bf79eb661f0334474b2a4255093685ff587eccd797c5a54db7e golangci-lint-2.0.2-netbsd-386.tar.gz
eab81d729778166415d349a80e568b2f2b3a781745a9be3212a92abb1e732daf golangci-lint-2.0.2-netbsd-amd64.tar.gz
d20add73f7c2de2c3b01ed4fd7b63ffcf0a6597d5ea228d1699e92339a3cd047 golangci-lint-2.0.2-netbsd-arm64.tar.gz
4e4f44e6057879cd62424ff1800a767d25a595c0e91d6d48809eea9186b4c739 golangci-lint-2.0.2-netbsd-armv6.tar.gz
51ec17b16d8743ae4098a0171f04f0ed4d64561e3051b982778b0e6c306a1b03 golangci-lint-2.0.2-netbsd-armv7.tar.gz
5482cf27b93fae1765c70ee2a95d4074d038e9dee61bdd61d017ce8893d3a4a8 golangci-lint-2.0.2-source.tar.gz
a35d8fdf3e14079a10880dbbb7586b46faec89be96f086b244b3e565aac80313 golangci-lint-2.0.2-windows-386.zip
fe4b946cc01366b989001215687003a9c4a7098589921f75e6228d6d8cffc15c golangci-lint-2.0.2-windows-amd64.zip
646bd9250ef8c771d85cd22fe8e6f2397ae39599179755e3bbfa9ef97ad44090 golangci-lint-2.0.2-windows-arm64.zip
ce1dc0bad6f8a61d64e6b3779eeb773479c175125d6f686b0e67ef9c8432d16e golangci-lint-2.0.2-windows-armv6.zip
92684a48faabe792b11ac27ca8b25551eff940b0a1e84ad7244e98b4994962db golangci-lint-2.0.2-windows-armv7.zip
# This is the builder on PPA that will build Go itself (inception-y), don't modify!
#

57
build/ci.go
View file

@ -59,6 +59,7 @@ import (
"github.com/cespare/cp"
"github.com/ethereum/go-ethereum/crypto/signify"
"github.com/ethereum/go-ethereum/internal/build"
"github.com/ethereum/go-ethereum/internal/download"
"github.com/ethereum/go-ethereum/internal/version"
)
@ -190,7 +191,7 @@ func doInstall(cmdline []string) {
// Configure the toolchain.
tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
if *dlgo {
csdb := build.MustLoadChecksums("build/checksums.txt")
csdb := download.MustLoadChecksums("build/checksums.txt")
tc.Root = build.DownloadGo(csdb)
}
// Disable CLI markdown doc generation in release builds.
@ -285,7 +286,7 @@ func doTest(cmdline []string) {
flag.CommandLine.Parse(cmdline)
// Get test fixtures.
csdb := build.MustLoadChecksums("build/checksums.txt")
csdb := download.MustLoadChecksums("build/checksums.txt")
downloadSpecTestFixtures(csdb, *cachedir)
// Configure the toolchain.
@ -329,16 +330,11 @@ func doTest(cmdline []string) {
}
// downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string {
executionSpecTestsVersion, err := build.Version(csdb, "spec-tests")
if err != nil {
log.Fatal(err)
}
func downloadSpecTestFixtures(csdb *download.ChecksumDB, cachedir string) string {
ext := ".tar.gz"
base := "fixtures_pectra-devnet-6" // TODO(s1na) rename once the version becomes part of the filename
url := fmt.Sprintf("https://github.com/ethereum/execution-spec-tests/releases/download/%s/%s%s", executionSpecTestsVersion, base, ext)
base := "fixtures_develop"
archivePath := filepath.Join(cachedir, base+ext)
if err := csdb.DownloadFile(url, archivePath); err != nil {
if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
log.Fatal(err)
}
if err := build.ExtractArchive(archivePath, executionSpecTestsDir); err != nil {
@ -444,14 +440,13 @@ func doLint(cmdline []string) {
// downloadLinter downloads and unpacks golangci-lint.
func downloadLinter(cachedir string) string {
csdb := build.MustLoadChecksums("build/checksums.txt")
version, err := build.Version(csdb, "golangci")
csdb := download.MustLoadChecksums("build/checksums.txt")
version, err := csdb.FindVersion("golangci")
if err != nil {
log.Fatal(err)
}
arch := runtime.GOARCH
ext := ".tar.gz"
if runtime.GOOS == "windows" {
ext = ".zip"
}
@ -459,9 +454,8 @@ func downloadLinter(cachedir string) string {
arch += "v" + os.Getenv("GOARM")
}
base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch)
url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext)
archivePath := filepath.Join(cachedir, base+ext)
if err := csdb.DownloadFile(url, archivePath); err != nil {
if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
log.Fatal(err)
}
if err := build.ExtractArchive(archivePath, cachedir); err != nil {
@ -497,8 +491,8 @@ func protocArchiveBaseName() (string, error) {
// in the generate command. It returns the full path of the directory
// containing the 'protoc-gen-go' executable.
func downloadProtocGenGo(cachedir string) string {
csdb := build.MustLoadChecksums("build/checksums.txt")
version, err := build.Version(csdb, "protoc-gen-go")
csdb := download.MustLoadChecksums("build/checksums.txt")
version, err := csdb.FindVersion("protoc-gen-go")
if err != nil {
log.Fatal(err)
}
@ -510,10 +504,8 @@ func downloadProtocGenGo(cachedir string) string {
archiveName += ".tar.gz"
}
url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf-go/releases/download/v%s/%s", version, archiveName)
archivePath := path.Join(cachedir, archiveName)
if err := csdb.DownloadFile(url, archivePath); err != nil {
if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
log.Fatal(err)
}
extractDest := filepath.Join(cachedir, baseName)
@ -531,8 +523,8 @@ func downloadProtocGenGo(cachedir string) string {
// files as a CI step. It returns the full path to the directory containing
// the protoc executable.
func downloadProtoc(cachedir string) string {
csdb := build.MustLoadChecksums("build/checksums.txt")
version, err := build.Version(csdb, "protoc")
csdb := download.MustLoadChecksums("build/checksums.txt")
version, err := csdb.FindVersion("protoc")
if err != nil {
log.Fatal(err)
}
@ -543,10 +535,8 @@ func downloadProtoc(cachedir string) string {
fileName := fmt.Sprintf("protoc-%s-%s", version, baseName)
archiveFileName := fileName + ".zip"
url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf/releases/download/v%s/%s", version, archiveFileName)
archivePath := filepath.Join(cachedir, archiveFileName)
if err := csdb.DownloadFile(url, archivePath); err != nil {
if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
log.Fatal(err)
}
extractDest := filepath.Join(cachedir, fileName)
@ -826,18 +816,17 @@ func doDebianSource(cmdline []string) {
// downloadGoBootstrapSources downloads the Go source tarball(s) that will be used
// to bootstrap the builder Go.
func downloadGoBootstrapSources(cachedir string) []string {
csdb := build.MustLoadChecksums("build/checksums.txt")
csdb := download.MustLoadChecksums("build/checksums.txt")
var bundles []string
for _, booter := range []string{"ppa-builder-1.19", "ppa-builder-1.21", "ppa-builder-1.23"} {
gobootVersion, err := build.Version(csdb, booter)
gobootVersion, err := csdb.FindVersion(booter)
if err != nil {
log.Fatal(err)
}
file := fmt.Sprintf("go%s.src.tar.gz", gobootVersion)
url := "https://dl.google.com/go/" + file
dst := filepath.Join(cachedir, file)
if err := csdb.DownloadFile(url, dst); err != nil {
if err := csdb.DownloadFileFromKnownURL(dst); err != nil {
log.Fatal(err)
}
bundles = append(bundles, dst)
@ -847,15 +836,14 @@ func downloadGoBootstrapSources(cachedir string) []string {
// downloadGoSources downloads the Go source tarball.
func downloadGoSources(cachedir string) string {
csdb := build.MustLoadChecksums("build/checksums.txt")
dlgoVersion, err := build.Version(csdb, "golang")
csdb := download.MustLoadChecksums("build/checksums.txt")
dlgoVersion, err := csdb.FindVersion("golang")
if err != nil {
log.Fatal(err)
}
file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion)
url := "https://dl.google.com/go/" + file
dst := filepath.Join(cachedir, file)
if err := csdb.DownloadFile(url, dst); err != nil {
if err := csdb.DownloadFileFromKnownURL(dst); err != nil {
log.Fatal(err)
}
return dst
@ -1181,5 +1169,6 @@ func doPurge(cmdline []string) {
}
func doSanityCheck() {
build.DownloadAndVerifyChecksums(build.MustLoadChecksums("build/checksums.txt"))
csdb := download.MustLoadChecksums("build/checksums.txt")
csdb.DownloadAndVerifyAll()
}

1
cmd/blsync/main.go
View file

@ -43,6 +43,7 @@ func main() {
utils.BeaconGenesisRootFlag,
utils.BeaconGenesisTimeFlag,
utils.BeaconCheckpointFlag,
utils.BeaconCheckpointFileFlag,
//TODO datadir for optional permanent database
utils.MainnetFlag,
utils.SepoliaFlag,

6
cmd/clef/README.md
View file

@ -1,6 +1,6 @@
# Clef
Clef can be used to sign transactions and data and is meant as a(n eventual) replacement for Geth's account management. This allows DApps to not depend on Geth's account management. When a DApp wants to sign data (or a transaction), it can send the content to Clef, which will then provide the user with context and asks for permission to sign the content. If the users grants the signing request, Clef will send the signature back to the DApp.
Clef can be used to sign transactions and data and is meant as a(n eventual) replacement for Geth's account management. This allows DApps to not depend on Geth's account management. When a DApp wants to sign data (or a transaction), it can send the content to Clef, which will then provide the user with context and ask for permission to sign the content. If the user grants the signing request, Clef will send the signature back to the DApp.
This setup allows a DApp to connect to a remote Ethereum node and send transactions that are locally signed. This can help in situations when a DApp is connected to an untrusted remote Ethereum node, because a local one is not available, not synchronized with the chain, or is a node that has no built-in (or limited) account management.
@ -123,7 +123,7 @@ The External API is **untrusted**: it does not accept credentials, nor does it e
Clef has one native console-based UI, for operation without any standalone tools. However, there is also an API to communicate with an external UI. To enable that UI, the signer needs to be executed with the `--stdio-ui` option, which allocates `stdin` / `stdout` for the UI API.
An example (insecure) proof-of-concept of has been implemented in `pythonsigner.py`.
An example (insecure) proof-of-concept has been implemented in `pythonsigner.py`.
The model is as follows:
@ -335,7 +335,7 @@ Bash example:
#### Arguments
- content type [string]: type of signed data
- `text/validator`: hex data with custom validator defined in a contract
- `text/validator`: hex data with a custom validator defined in a contract
- `application/clique`: [clique](https://github.com/ethereum/EIPs/issues/225) headers
- `text/plain`: simple hex data validated by `account_ecRecover`
- account [address]: account to sign with

2
cmd/clef/rules.md
View file

@ -2,7 +2,7 @@
The `signer` binary contains a ruleset engine, implemented with [OttoVM](https://github.com/robertkrimen/otto)
It enables usecases like the following:
It enables use cases like the following:
* I want to auto-approve transactions with contract `CasinoDapp`, with up to `0.05 ether` in value to maximum `1 ether` per 24h period
* I want to auto-approve transaction to contract `EthAlarmClock` with `data`=`0xdeadbeef`, if `value=0`, `gas < 44k` and `gasPrice < 40Gwei`

40
cmd/devp2p/internal/ethtest/conn.go
View file

@ -66,10 +66,9 @@ func (s *Suite) dialAs(key *ecdsa.PrivateKey) (*Conn, error) {
return nil, err
}
conn.caps = []p2p.Cap{
{Name: "eth", Version: 67},
{Name: "eth", Version: 68},
{Name: "eth", Version: 69},
}
conn.ourHighestProtoVersion = 68
conn.ourHighestProtoVersion = 69
return &conn, nil
}
@ -156,7 +155,7 @@ func (c *Conn) ReadEth() (any, error) {
var msg any
switch int(code) {
case eth.StatusMsg:
msg = new(eth.StatusPacket)
msg = new(eth.StatusPacket69)
case eth.GetBlockHeadersMsg:
msg = new(eth.GetBlockHeadersPacket)
case eth.BlockHeadersMsg:
@ -229,9 +228,21 @@ func (c *Conn) ReadSnap() (any, error) {
}
}
// dialAndPeer creates a peer connection and runs the handshake.
func (s *Suite) dialAndPeer(status *eth.StatusPacket69) (*Conn, error) {
c, err := s.dial()
if err != nil {
return nil, err
}
if err = c.peer(s.chain, status); err != nil {
c.Close()
}
return c, err
}
// peer performs both the protocol handshake and the status message
// exchange with the node in order to peer with it.
func (c *Conn) peer(chain *Chain, status *eth.StatusPacket) error {
func (c *Conn) peer(chain *Chain, status *eth.StatusPacket69) error {
if err := c.handshake(); err != nil {
return fmt.Errorf("handshake failed: %v", err)
}
@ -304,7 +315,7 @@ func (c *Conn) negotiateEthProtocol(caps []p2p.Cap) {
}
// statusExchange performs a `Status` message exchange with the given node.
func (c *Conn) statusExchange(chain *Chain, status *eth.StatusPacket) error {
func (c *Conn) statusExchange(chain *Chain, status *eth.StatusPacket69) error {
loop:
for {
code, data, err := c.Read()
@ -313,12 +324,16 @@ loop:
}
switch code {
case eth.StatusMsg + protoOffset(ethProto):
msg := new(eth.StatusPacket)
msg := new(eth.StatusPacket69)
if err := rlp.DecodeBytes(data, &msg); err != nil {
return fmt.Errorf("error decoding status packet: %w", err)
}
if have, want := msg.Head, chain.blocks[chain.Len()-1].Hash(); have != want {
return fmt.Errorf("wrong head block in status, want: %#x (block %d) have %#x",
if have, want := msg.LatestBlock, chain.blocks[chain.Len()-1].NumberU64(); have != want {
return fmt.Errorf("wrong head block in status, want: %d, have %d",
want, have)
}
if have, want := msg.LatestBlockHash, chain.blocks[chain.Len()-1].Hash(); have != want {
return fmt.Errorf("wrong head block in status, want: %#x (block %d) have %#x",
want, chain.blocks[chain.Len()-1].NumberU64(), have)
}
if have, want := msg.ForkID, chain.ForkID(); !reflect.DeepEqual(have, want) {
@ -348,13 +363,14 @@ loop:
}
if status == nil {
// default status message
status = &eth.StatusPacket{
status = &eth.StatusPacket69{
ProtocolVersion: uint32(c.negotiatedProtoVersion),
NetworkID: chain.config.ChainID.Uint64(),
TD: chain.TD(),
Head: chain.blocks[chain.Len()-1].Hash(),
Genesis: chain.blocks[0].Hash(),
ForkID: chain.ForkID(),
EarliestBlock: 0,
LatestBlock: chain.blocks[chain.Len()-1].NumberU64(),
LatestBlockHash: chain.blocks[chain.Len()-1].Hash(),
}
}
if err := c.Write(ethProto, eth.StatusMsg, status); err != nil {

2
cmd/devp2p/internal/ethtest/protocol.go
View file

@ -32,7 +32,7 @@ const (
// Unexported devp2p protocol lengths from p2p package.
const (
baseProtoLen = 16
ethProtoLen = 17
ethProtoLen = 18
snapProtoLen = 8
)

232
cmd/devp2p/internal/ethtest/suite.go
View file

@ -68,15 +68,19 @@ func (s *Suite) EthTests() []utesting.Test {
return []utesting.Test{
// status
{Name: "Status", Fn: s.TestStatus},
{Name: "MaliciousHandshake", Fn: s.TestMaliciousHandshake},
{Name: "BlockRangeUpdateExpired", Fn: s.TestBlockRangeUpdateHistoryExp},
{Name: "BlockRangeUpdateFuture", Fn: s.TestBlockRangeUpdateFuture},
{Name: "BlockRangeUpdateInvalid", Fn: s.TestBlockRangeUpdateInvalid},
// get block headers
{Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders},
{Name: "GetNonexistentBlockHeaders", Fn: s.TestGetNonexistentBlockHeaders},
{Name: "SimultaneousRequests", Fn: s.TestSimultaneousRequests},
{Name: "SameRequestID", Fn: s.TestSameRequestID},
{Name: "ZeroRequestID", Fn: s.TestZeroRequestID},
// get block bodies
// get history
{Name: "GetBlockBodies", Fn: s.TestGetBlockBodies},
// // malicious handshakes + status
{Name: "MaliciousHandshake", Fn: s.TestMaliciousHandshake},
{Name: "GetReceipts", Fn: s.TestGetReceipts},
// test transactions
{Name: "LargeTxRequest", Fn: s.TestLargeTxRequest, Slow: true},
{Name: "Transaction", Fn: s.TestTransaction},
@ -100,15 +104,11 @@ func (s *Suite) SnapTests() []utesting.Test {
func (s *Suite) TestStatus(t *utesting.T) {
t.Log(`This test is just a sanity check. It performs an eth protocol handshake.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
t.Fatal("peering failed:", err)
}
conn.Close()
}
// headersMatch returns whether the received headers match the given request
@ -118,15 +118,12 @@ func headersMatch(expected []*types.Header, headers []*types.Header) bool {
func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
t.Log(`This test requests block headers from the node.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err = conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Send headers request.
req := &eth.GetBlockHeadersPacket{
RequestId: 33,
@ -158,18 +155,51 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
}
}
func (s *Suite) TestGetNonexistentBlockHeaders(t *utesting.T) {
t.Log(`This test sends GetBlockHeaders requests for nonexistent blocks (using max uint64 value)
to check if the node disconnects after receiving multiple invalid requests.`)
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Create request with max uint64 value for a nonexistent block
badReq := &eth.GetBlockHeadersPacket{
GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
Origin: eth.HashOrNumber{Number: ^uint64(0)},
Amount: 1,
Skip: 0,
Reverse: false,
},
}
// Send request 10 times. Some clients are lient on the first few invalids.
for i := 0; i < 10; i++ {
badReq.RequestId = uint64(i)
if err := conn.Write(ethProto, eth.GetBlockHeadersMsg, badReq); err != nil {
if err == errDisc {
t.Fatalf("peer disconnected after %d requests", i+1)
}
t.Fatalf("write failed: %v", err)
}
}
// Check if peer disconnects at the end.
code, _, err := conn.Read()
if err == errDisc || code == discMsg {
t.Fatal("peer improperly disconnected")
}
}
func (s *Suite) TestSimultaneousRequests(t *utesting.T) {
t.Log(`This test requests blocks headers from the node, performing two requests
concurrently, with different request IDs.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Create two different requests.
req1 := &eth.GetBlockHeadersPacket{
@ -235,15 +265,11 @@ concurrently, with different request IDs.`)
func (s *Suite) TestSameRequestID(t *utesting.T) {
t.Log(`This test requests block headers, performing two concurrent requests with the
same request ID. The node should handle the request by responding to both requests.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Create two different requests with the same ID.
reqID := uint64(1234)
@ -306,15 +332,12 @@ same request ID. The node should handle the request by responding to both reques
func (s *Suite) TestZeroRequestID(t *utesting.T) {
t.Log(`This test sends a GetBlockHeaders message with a request-id of zero,
and expects a response.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
req := &eth.GetBlockHeadersPacket{
GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
Origin: eth.HashOrNumber{Number: 0},
@ -341,15 +364,12 @@ and expects a response.`)
func (s *Suite) TestGetBlockBodies(t *utesting.T) {
t.Log(`This test sends GetBlockBodies requests to the node for known blocks in the test chain.`)
conn, err := s.dial()
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Create block bodies request.
req := &eth.GetBlockBodiesPacket{
RequestId: 55,
@ -375,6 +395,47 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) {
}
}
func (s *Suite) TestGetReceipts(t *utesting.T) {
t.Log(`This test sends GetReceipts requests to the node for known blocks in the test chain.`)
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatalf("peering failed: %v", err)
}
defer conn.Close()
// Find some blocks containing receipts.
var hashes = make([]common.Hash, 0, 3)
for i := range s.chain.Len() {
block := s.chain.GetBlock(i)
if len(block.Transactions()) > 0 {
hashes = append(hashes, block.Hash())
}
if len(hashes) == cap(hashes) {
break
}
}
// Create block bodies request.
req := &eth.GetReceiptsPacket{
RequestId: 66,
GetReceiptsRequest: (eth.GetReceiptsRequest)(hashes),
}
if err := conn.Write(ethProto, eth.GetReceiptsMsg, req); err != nil {
t.Fatalf("could not write to connection: %v", err)
}
// Wait for response.
resp := new(eth.ReceiptsPacket[*eth.ReceiptList69])
if err := conn.ReadMsg(ethProto, eth.ReceiptsMsg, &resp); err != nil {
t.Fatalf("error reading block bodies msg: %v", err)
}
if got, want := resp.RequestId, req.RequestId; got != want {
t.Fatalf("unexpected request id in respond", got, want)
}
if len(resp.List) != len(req.GetReceiptsRequest) {
t.Fatalf("wrong bodies in response: expected %d bodies, got %d", len(req.GetReceiptsRequest), len(resp.List))
}
}
// randBuf makes a random buffer size kilobytes large.
func randBuf(size int) []byte {
buf := make([]byte, size*1024)
@ -457,6 +518,97 @@ func (s *Suite) TestMaliciousHandshake(t *utesting.T) {
}
}
func (s *Suite) TestBlockRangeUpdateInvalid(t *utesting.T) {
t.Log(`This test sends an invalid BlockRangeUpdate message to the node and expects to be disconnected.`)
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
conn.Write(ethProto, eth.BlockRangeUpdateMsg, &eth.BlockRangeUpdatePacket{
EarliestBlock: 10,
LatestBlock: 8,
LatestBlockHash: s.chain.GetBlock(8).Hash(),
})
if code, _, err := conn.Read(); err != nil {
t.Fatalf("expected disconnect, got err: %v", err)
} else if code != discMsg {
t.Fatalf("expected disconnect message, got msg code %d", code)
}
}
func (s *Suite) TestBlockRangeUpdateFuture(t *utesting.T) {
t.Log(`This test sends a BlockRangeUpdate that is beyond the chain head.
The node should accept the update and should not disonnect.`)
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
head := s.chain.Head().NumberU64()
var hash common.Hash
rand.Read(hash[:])
conn.Write(ethProto, eth.BlockRangeUpdateMsg, &eth.BlockRangeUpdatePacket{
EarliestBlock: head + 10,
LatestBlock: head + 50,
LatestBlockHash: hash,
})
// Ensure the node does not disconnect us.
// Just send a few ping messages.
for range 10 {
time.Sleep(100 * time.Millisecond)
if err := conn.Write(baseProto, pingMsg, []any{}); err != nil {
t.Fatal("write error:", err)
}
code, _, err := conn.Read()
switch {
case err != nil:
t.Fatal("read error:", err)
case code == discMsg:
t.Fatal("got disconnect")
case code == pongMsg:
}
}
}
func (s *Suite) TestBlockRangeUpdateHistoryExp(t *utesting.T) {
t.Log(`This test sends a BlockRangeUpdate announcing incomplete (expired) history.
The node should accept the update and should not disonnect.`)
conn, err := s.dialAndPeer(nil)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
head := s.chain.Head()
conn.Write(ethProto, eth.BlockRangeUpdateMsg, &eth.BlockRangeUpdatePacket{
EarliestBlock: head.NumberU64() - 10,
LatestBlock: head.NumberU64(),
LatestBlockHash: head.Hash(),
})
// Ensure the node does not disconnect us.
// Just send a few ping messages.
for range 10 {
time.Sleep(100 * time.Millisecond)
if err := conn.Write(baseProto, pingMsg, []any{}); err != nil {
t.Fatal("write error:", err)
}
code, _, err := conn.Read()
switch {
case err != nil:
t.Fatal("read error:", err)
case code == discMsg:
t.Fatal("got disconnect")
case code == pongMsg:
}
}
}
func (s *Suite) TestTransaction(t *utesting.T) {
t.Log(`This test sends a valid transaction to the node and checks if the
transaction gets propagated.`)

35
cmd/devp2p/internal/v5test/discv5tests.go
View file

@ -59,8 +59,9 @@ func (s *Suite) AllTests() []utesting.Test {
}
}
// TestPing sends PING and expects a PONG response.
func (s *Suite) TestPing(t *utesting.T) {
t.Log(`This test is just a sanity check. It sends PING and expects a PONG response.`)
conn, l1 := s.listen1(t)
defer conn.close()
@ -85,9 +86,10 @@ func checkPong(t *utesting.T, pong *v5wire.Pong, ping *v5wire.Ping, c net.Packet
}
}
// TestPingLargeRequestID sends PING with a 9-byte request ID, which isn't allowed by the spec.
// The remote node should not respond.
func (s *Suite) TestPingLargeRequestID(t *utesting.T) {
t.Log(`This test sends PING with a 9-byte request ID, which isn't allowed by the spec.
The remote node should not respond.`)
conn, l1 := s.listen1(t)
defer conn.close()
@ -104,10 +106,11 @@ func (s *Suite) TestPingLargeRequestID(t *utesting.T) {
}
}
// TestPingMultiIP establishes a session from one IP as usual. The session is then reused
// on another IP, which shouldn't work. The remote node should respond with WHOAREYOU for
// the attempt from a different IP.
func (s *Suite) TestPingMultiIP(t *utesting.T) {
t.Log(`This test establishes a session from one IP as usual. The session is then reused
on another IP, which shouldn't work. The remote node should respond with WHOAREYOU for
the attempt from a different IP.`)
conn, l1, l2 := s.listen2(t)
defer conn.close()
@ -120,6 +123,7 @@ func (s *Suite) TestPingMultiIP(t *utesting.T) {
checkPong(t, resp.(*v5wire.Pong), ping, l1)
// Send on l2. This reuses the session because there is only one codec.
t.Log("sending ping from alternate IP", l2.LocalAddr())
ping2 := &v5wire.Ping{ReqID: conn.nextReqID()}
conn.write(l2, ping2, nil)
switch resp := conn.read(l2).(type) {
@ -158,6 +162,10 @@ func (s *Suite) TestPingMultiIP(t *utesting.T) {
// packet instead of a handshake message packet. The remote node should respond with
// another WHOAREYOU challenge for the second packet.
func (s *Suite) TestPingHandshakeInterrupted(t *utesting.T) {
t.Log(`TestPingHandshakeInterrupted starts a handshake, but doesn't finish it and sends a second ordinary message
packet instead of a handshake message packet. The remote node should respond with
another WHOAREYOU challenge for the second packet.`)
conn, l1 := s.listen1(t)
defer conn.close()
@ -181,8 +189,10 @@ func (s *Suite) TestPingHandshakeInterrupted(t *utesting.T) {
}
}
// TestTalkRequest sends TALKREQ and expects an empty TALKRESP response.
func (s *Suite) TestTalkRequest(t *utesting.T) {
t.Log(`This test sends some examples of TALKREQ with a protocol-id of "test-protocol"
and expects an empty TALKRESP response.`)
conn, l1 := s.listen1(t)
defer conn.close()
@ -202,6 +212,7 @@ func (s *Suite) TestTalkRequest(t *utesting.T) {
}
// Empty request ID.
t.Log("sending TALKREQ with empty request-id")
resp = conn.reqresp(l1, &v5wire.TalkRequest{Protocol: "test-protocol"})
switch resp := resp.(type) {
case *v5wire.TalkResponse:
@ -216,8 +227,9 @@ func (s *Suite) TestTalkRequest(t *utesting.T) {
}
}
// TestFindnodeZeroDistance checks that the remote node returns itself for FINDNODE with distance zero.
func (s *Suite) TestFindnodeZeroDistance(t *utesting.T) {
t.Log(`This test checks that the remote node returns itself for FINDNODE with distance zero.`)
conn, l1 := s.listen1(t)
defer conn.close()
@ -233,9 +245,11 @@ func (s *Suite) TestFindnodeZeroDistance(t *utesting.T) {
}
}
// TestFindnodeResults pings the node under test from multiple nodes. After waiting for them to be
// accepted into the remote table, the test checks that they are returned by FINDNODE.
func (s *Suite) TestFindnodeResults(t *utesting.T) {
t.Log(`This test pings the node under test from multiple other endpoints and node identities
(the 'bystanders'). After waiting for them to be accepted into the remote table, the test checks
that they are returned by FINDNODE.`)
// Create bystanders.
nodes := make([]*bystander, 5)
added := make(chan enode.ID, len(nodes))
@ -273,6 +287,7 @@ func (s *Suite) TestFindnodeResults(t *utesting.T) {
}
// Send FINDNODE for all distances.
t.Log("requesting nodes")
conn, l1 := s.listen1(t)
defer conn.close()
foundNodes, err := conn.findnode(l1, dists)

2
cmd/evm/README.md
View file

@ -16,7 +16,7 @@ which can
1. Take a prestate, including
- Accounts,
- Block context information,
- Previous blockshashes (*optional)
- Previous block hashes (*optional)
2. Apply a set of transactions,
3. Apply a mining-reward (*optional),
4. And generate a post-state, including

2
cmd/evm/eest.go
View file

@ -22,7 +22,7 @@ import "regexp"
// within its filename by the execution spec test (EEST).
type testMetadata struct {
fork string
module string // which python module gnerated the test, e.g. eip7702
module string // which python module generated the test, e.g. eip7702
file string // exact file the test came from, e.g. test_gas.py
function string // func that created the test, e.g. test_valid_mcopy_operations
parameters string // the name of the parameters which were used to fill the test, e.g. zero_inputs

228
cmd/evm/eofparse.go
View file

@ -1,228 +0,0 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bufio"
"encoding/hex"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/log"
"github.com/urfave/cli/v2"
)
var jt vm.JumpTable
const initcode = "INITCODE"
func init() {
jt = vm.NewEOFInstructionSetForTesting()
}
var (
hexFlag = &cli.StringFlag{
Name: "hex",
Usage: "Single container data parse and validation",
}
refTestFlag = &cli.StringFlag{
Name: "test",
Usage: "Path to EOF validation reference test.",
}
eofParseCommand = &cli.Command{
Name: "eofparse",
Aliases: []string{"eof"},
Usage: "Parses hex eof container and returns validation errors (if any)",
Action: eofParseAction,
Flags: []cli.Flag{
hexFlag,
refTestFlag,
},
}
eofDumpCommand = &cli.Command{
Name: "eofdump",
Usage: "Parses hex eof container and prints out human-readable representation of the container.",
Action: eofDumpAction,
Flags: []cli.Flag{
hexFlag,
},
}
)
func eofParseAction(ctx *cli.Context) error {
// If `--test` is set, parse and validate the reference test at the provided path.
if ctx.IsSet(refTestFlag.Name) {
var (
file = ctx.String(refTestFlag.Name)
executedTests int
passedTests int
)
err := filepath.Walk(file, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
log.Debug("Executing test", "name", info.Name())
passed, tot, err := executeTest(path)
passedTests += passed
executedTests += tot
return err
})
if err != nil {
return err
}
log.Info("Executed tests", "passed", passedTests, "total executed", executedTests)
return nil
}
// If `--hex` is set, parse and validate the hex string argument.
if ctx.IsSet(hexFlag.Name) {
if _, err := parseAndValidate(ctx.String(hexFlag.Name), false); err != nil {
return fmt.Errorf("err: %w", err)
}
fmt.Println("OK")
return nil
}
// If neither are passed in, read input from stdin.
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
for scanner.Scan() {
l := strings.TrimSpace(scanner.Text())
if strings.HasPrefix(l, "#") || l == "" {
continue
}
if _, err := parseAndValidate(l, false); err != nil {
fmt.Printf("err: %v\n", err)
} else {
fmt.Println("OK")
}
}
if err := scanner.Err(); err != nil {
fmt.Println(err.Error())
}
return nil
}
type refTests struct {
Vectors map[string]eOFTest `json:"vectors"`
}
type eOFTest struct {
Code string `json:"code"`
Results map[string]etResult `json:"results"`
ContainerKind string `json:"containerKind"`
}
type etResult struct {
Result bool `json:"result"`
Exception string `json:"exception,omitempty"`
}
func executeTest(path string) (int, int, error) {
src, err := os.ReadFile(path)
if err != nil {
return 0, 0, err
}
var testsByName map[string]refTests
if err := json.Unmarshal(src, &testsByName); err != nil {
return 0, 0, err
}
passed, total := 0, 0
for testsName, tests := range testsByName {
for name, tt := range tests.Vectors {
for fork, r := range tt.Results {
total++
_, err := parseAndValidate(tt.Code, tt.ContainerKind == initcode)
if r.Result && err != nil {
log.Error("Test failure, expected validation success", "name", testsName, "idx", name, "fork", fork, "err", err)
continue
}
if !r.Result && err == nil {
log.Error("Test failure, expected validation error", "name", testsName, "idx", name, "fork", fork, "have err", r.Exception, "err", err)
continue
}
passed++
}
}
}
return passed, total, nil
}
func parseAndValidate(s string, isInitCode bool) (*vm.Container, error) {
if len(s) >= 2 && strings.HasPrefix(s, "0x") {
s = s[2:]
}
b, err := hex.DecodeString(s)
if err != nil {
return nil, fmt.Errorf("unable to decode data: %w", err)
}
return parse(b, isInitCode)
}
func parse(b []byte, isInitCode bool) (*vm.Container, error) {
var c vm.Container
if err := c.UnmarshalBinary(b, isInitCode); err != nil {
return nil, err
}
if err := c.ValidateCode(&jt, isInitCode); err != nil {
return nil, err
}
return &c, nil
}
func eofDumpAction(ctx *cli.Context) error {
// If `--hex` is set, parse and validate the hex string argument.
if ctx.IsSet(hexFlag.Name) {
return eofDump(ctx.String(hexFlag.Name))
}
// Otherwise read from stdin
scanner := bufio.NewScanner(os.Stdin)
scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
for scanner.Scan() {
l := strings.TrimSpace(scanner.Text())
if strings.HasPrefix(l, "#") || l == "" {
continue
}
if err := eofDump(l); err != nil {
return err
}
fmt.Println("")
}
return scanner.Err()
}
func eofDump(hexdata string) error {
if len(hexdata) >= 2 && strings.HasPrefix(hexdata, "0x") {
hexdata = hexdata[2:]
}
b, err := hex.DecodeString(hexdata)
if err != nil {
return fmt.Errorf("unable to decode data: %w", err)
}
var c vm.Container
if err := c.UnmarshalBinary(b, false); err != nil {
return err
}
fmt.Println(c.String())
return nil
}

182
cmd/evm/eofparse_test.go
View file

@ -1,182 +0,0 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"bufio"
"bytes"
"encoding/hex"
"fmt"
"os"
"strings"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/vm"
)
func FuzzEofParsing(f *testing.F) {
// Seed with corpus from execution-spec-tests
for i := 0; ; i++ {
fname := fmt.Sprintf("testdata/eof/eof_corpus_%d.txt", i)
corpus, err := os.Open(fname)
if err != nil {
break
}
f.Logf("Reading seed data from %v", fname)
scanner := bufio.NewScanner(corpus)
scanner.Buffer(make([]byte, 1024), 10*1024*1024)
for scanner.Scan() {
s := scanner.Text()
if len(s) >= 2 && strings.HasPrefix(s, "0x") {
s = s[2:]
}
b, err := hex.DecodeString(s)
if err != nil {
panic(err) // rotten corpus
}
f.Add(b)
}
corpus.Close()
if err := scanner.Err(); err != nil {
panic(err) // rotten corpus
}
}
// And do the fuzzing
f.Fuzz(func(t *testing.T, data []byte) {
var (
jt = vm.NewEOFInstructionSetForTesting()
c vm.Container
)
cpy := common.CopyBytes(data)
if err := c.UnmarshalBinary(data, true); err == nil {
c.ValidateCode(&jt, true)
if have := c.MarshalBinary(); !bytes.Equal(have, data) {
t.Fatal("Unmarshal-> Marshal failure!")
}
}
if err := c.UnmarshalBinary(data, false); err == nil {
c.ValidateCode(&jt, false)
if have := c.MarshalBinary(); !bytes.Equal(have, data) {
t.Fatal("Unmarshal-> Marshal failure!")
}
}
if !bytes.Equal(cpy, data) {
panic("data modified during unmarshalling")
}
})
}
func TestEofParseInitcode(t *testing.T) {
testEofParse(t, true, "testdata/eof/results.initcode.txt")
}
func TestEofParseRegular(t *testing.T) {
testEofParse(t, false, "testdata/eof/results.regular.txt")
}
func testEofParse(t *testing.T, isInitCode bool, wantFile string) {
var wantFn func() string
var wantLoc = 0
{ // Configure the want-reader
wants, err := os.Open(wantFile)
if err != nil {
t.Fatal(err)
}
scanner := bufio.NewScanner(wants)
scanner.Buffer(make([]byte, 1024), 10*1024*1024)
wantFn = func() string {
if scanner.Scan() {
wantLoc++
return scanner.Text()
}
return "end of file reached"
}
}
for i := 0; ; i++ {
fname := fmt.Sprintf("testdata/eof/eof_corpus_%d.txt", i)
corpus, err := os.Open(fname)
if err != nil {
break
}
t.Logf("# Reading seed data from %v", fname)
scanner := bufio.NewScanner(corpus)
scanner.Buffer(make([]byte, 1024), 10*1024*1024)
line := 1
for scanner.Scan() {
s := scanner.Text()
if len(s) >= 2 && strings.HasPrefix(s, "0x") {
s = s[2:]
}
b, err := hex.DecodeString(s)
if err != nil {
panic(err) // rotten corpus
}
have := "OK"
if _, err := parse(b, isInitCode); err != nil {
have = fmt.Sprintf("ERR: %v", err)
}
if false { // Change this to generate the want-output
fmt.Printf("%v\n", have)
} else {
want := wantFn()
if have != want {
if len(want) > 100 {
want = want[:100]
}
if len(b) > 100 {
b = b[:100]
}
t.Errorf("%v:%d\n%v\ninput %x\nisInit: %v\nhave: %q\nwant: %q\n",
fname, line, fmt.Sprintf("%v:%d", wantFile, wantLoc), b, isInitCode, have, want)
}
}
line++
}
corpus.Close()
}
}
func BenchmarkEofParse(b *testing.B) {
corpus, err := os.Open("testdata/eof/eof_benches.txt")
if err != nil {
b.Fatal(err)
}
defer corpus.Close()
scanner := bufio.NewScanner(corpus)
scanner.Buffer(make([]byte, 1024), 10*1024*1024)
line := 1
for scanner.Scan() {
s := scanner.Text()
if len(s) >= 2 && strings.HasPrefix(s, "0x") {
s = s[2:]
}
data, err := hex.DecodeString(s)
if err != nil {
b.Fatal(err) // rotten corpus
}
b.Run(fmt.Sprintf("test-%d", line), func(b *testing.B) {
b.ReportAllocs()
b.SetBytes(int64(len(data)))
for i := 0; i < b.N; i++ {
_, _ = parse(data, false)
}
})
line++
}
}

10
cmd/evm/internal/t8ntool/execution.go
View file

@ -303,7 +303,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
}
// Set the receipt logs and create the bloom filter.
receipt.Logs = statedb.GetLogs(tx.Hash(), vmContext.BlockNumber.Uint64(), blockHash)
receipt.Logs = statedb.GetLogs(tx.Hash(), vmContext.BlockNumber.Uint64(), blockHash, vmContext.Time)
receipt.Bloom = types.CreateBloom(receipt)
// These three are non-consensus fields:
@ -363,9 +363,13 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not parse requests logs: %v", err))
}
// EIP-7002
core.ProcessWithdrawalQueue(&requests, evm)
if err := core.ProcessWithdrawalQueue(&requests, evm); err != nil {
return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not process withdrawal requests: %v", err))
}
// EIP-7251
core.ProcessConsolidationQueue(&requests, evm)
if err := core.ProcessConsolidationQueue(&requests, evm); err != nil {
return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not process consolidation requests: %v", err))
}
}
// Commit block

2
cmd/evm/internal/t8ntool/tx_iterator.go
View file

@ -102,7 +102,7 @@ func signUnsignedTransactions(txs []*txWithKey, signer types.Signer) (types.Tran
if tx.protected {
signed, err = types.SignTx(tx.tx, signer, tx.key)
} else {
signed, err = types.SignTx(tx.tx, types.FrontierSigner{}, tx.key)
signed, err = types.SignTx(tx.tx, types.HomesteadSigner{}, tx.key)
}
if err != nil {
return nil, NewError(ErrorJson, fmt.Errorf("tx %d: failed to sign tx: %v", i, err))

2
cmd/evm/main.go
View file

@ -210,8 +210,6 @@ func init() {
stateTransitionCommand,
transactionCommand,
blockBuilderCommand,
eofParseCommand,
eofDumpCommand,
}
app.Before = func(ctx *cli.Context) error {
flags.MigrateGlobalFlags(ctx)

241
cmd/geth/chaincmd.go
View file

@ -21,9 +21,12 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"runtime"
"slices"
"strconv"
"strings"
"sync/atomic"
"time"
@ -31,13 +34,18 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/history"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/internal/debug"
"github.com/ethereum/go-ethereum/internal/era"
"github.com/ethereum/go-ethereum/internal/era/eradl"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/params"
"github.com/urfave/cli/v2"
)
@ -65,7 +73,7 @@ It expects the genesis file as argument.`,
Name: "dumpgenesis",
Usage: "Dumps genesis block JSON configuration to stdout",
ArgsUsage: "",
Flags: append([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags...),
Flags: slices.Concat([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags),
Description: `
The dumpgenesis command prints the genesis configuration of the network preset
if one is set. Otherwise it prints the genesis from the datadir.`,
@ -76,12 +84,16 @@ if one is set. Otherwise it prints the genesis from the datadir.`,
Usage: "Import a blockchain file",
ArgsUsage: "<filename> (<filename 2> ... <filename N>) ",
Flags: slices.Concat([]cli.Flag{
utils.CacheFlag,
utils.SyncModeFlag,
utils.GCModeFlag,
utils.SnapshotFlag,
utils.CacheFlag,
utils.CacheDatabaseFlag,
utils.CacheTrieFlag,
utils.CacheGCFlag,
utils.CacheSnapshotFlag,
utils.CacheNoPrefetchFlag,
utils.CachePreimagesFlag,
utils.NoCompactionFlag,
utils.MetricsEnabledFlag,
utils.MetricsEnabledExpensiveFlag,
utils.MetricsHTTPFlag,
@ -104,7 +116,11 @@ if one is set. Otherwise it prints the genesis from the datadir.`,
utils.LogNoHistoryFlag,
utils.LogExportCheckpointsFlag,
utils.StateHistoryFlag,
}, utils.DatabaseFlags),
}, utils.DatabaseFlags, debug.Flags),
Before: func(ctx *cli.Context) error {
flags.MigrateGlobalFlags(ctx)
return debug.Setup(ctx)
},
Description: `
The import command allows the import of blocks from an RLP-encoded format. This format can be a single file
containing multiple RLP-encoded blocks, or multiple files can be given.
@ -118,10 +134,7 @@ to import successfully.`,
Name: "export",
Usage: "Export blockchain into file",
ArgsUsage: "<filename> [<blockNumFirst> <blockNumLast>]",
Flags: slices.Concat([]cli.Flag{
utils.CacheFlag,
utils.SyncModeFlag,
}, utils.DatabaseFlags),
Flags: slices.Concat([]cli.Flag{utils.CacheFlag}, utils.DatabaseFlags),
Description: `
Requires a first argument of the file to write to.
Optional second and third arguments control the first and
@ -134,12 +147,7 @@ be gzipped.`,
Name: "import-history",
Usage: "Import an Era archive",
ArgsUsage: "<dir>",
Flags: slices.Concat([]cli.Flag{
utils.TxLookupLimitFlag,
},
utils.DatabaseFlags,
utils.NetworkFlags,
),
Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag}, utils.DatabaseFlags, utils.NetworkFlags),
Description: `
The import-history command will import blocks and their corresponding receipts
from Era archives.
@ -150,7 +158,7 @@ from Era archives.
Name: "export-history",
Usage: "Export blockchain history to Era archives",
ArgsUsage: "<dir> <first> <last>",
Flags: slices.Concat(utils.DatabaseFlags),
Flags: utils.DatabaseFlags,
Description: `
The export-history command will export blocks and their corresponding receipts
into Era archives. Eras are typically packaged in steps of 8192 blocks.
@ -161,10 +169,7 @@ into Era archives. Eras are typically packaged in steps of 8192 blocks.
Name: "import-preimages",
Usage: "Import the preimage database from an RLP stream",
ArgsUsage: "<datafile>",
Flags: slices.Concat([]cli.Flag{
utils.CacheFlag,
utils.SyncModeFlag,
}, utils.DatabaseFlags),
Flags: slices.Concat([]cli.Flag{utils.CacheFlag}, utils.DatabaseFlags),
Description: `
The import-preimages command imports hash preimages from an RLP encoded stream.
It's deprecated, please use "geth db import" instead.
@ -189,6 +194,54 @@ It's deprecated, please use "geth db import" instead.
This command dumps out the state for a given block (or latest, if none provided).
`,
}
pruneHistoryCommand = &cli.Command{
Action: pruneHistory,
Name: "prune-history",
Usage: "Prune blockchain history (block bodies and receipts) up to the merge block",
ArgsUsage: "",
Flags: utils.DatabaseFlags,
Description: `
The prune-history command removes historical block bodies and receipts from the
blockchain database up to the merge block, while preserving block headers. This
helps reduce storage requirements for nodes that don't need full historical data.`,
}
downloadEraCommand = &cli.Command{
Action: downloadEra,
Name: "download-era",
Usage: "Fetches era1 files (pre-merge history) from an HTTP endpoint",
ArgsUsage: "",
Flags: slices.Concat(
utils.DatabaseFlags,
utils.NetworkFlags,
[]cli.Flag{
eraBlockFlag,
eraEpochFlag,
eraAllFlag,
eraServerFlag,
},
),
}
)
var (
eraBlockFlag = &cli.StringFlag{
Name: "block",
Usage: "Block number to fetch. (can also be a range <start>-<end>)",
}
eraEpochFlag = &cli.StringFlag{
Name: "epoch",
Usage: "Epoch number to fetch (can also be a range <start>-<end>)",
}
eraAllFlag = &cli.BoolFlag{
Name: "all",
Usage: "Download all available era1 files",
}
eraServerFlag = &cli.StringFlag{
Name: "server",
Usage: "era1 server URL",
}
)
// initGenesis will initialise the given JSON format genesis file and writes it as
@ -225,19 +278,19 @@ func initGenesis(ctx *cli.Context) error {
overrides.OverrideVerkle = &v
}
chaindb, err := stack.OpenDatabaseWithFreezer("chaindata", 0, 0, ctx.String(utils.AncientFlag.Name), "", false)
if err != nil {
utils.Fatalf("Failed to open database: %v", err)
}
chaindb := utils.MakeChainDatabase(ctx, stack, false)
defer chaindb.Close()
triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle())
defer triedb.Close()
_, hash, _, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides)
_, hash, compatErr, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides)
if err != nil {
utils.Fatalf("Failed to write genesis block: %v", err)
}
if compatErr != nil {
utils.Fatalf("Failed to write chain config: %v", compatErr)
}
log.Info("Successfully wrote genesis state", "database", "chaindata", "hash", hash)
return nil
@ -262,7 +315,7 @@ func dumpGenesis(ctx *cli.Context) error {
// dump whatever already exists in the datadir
stack, _ := makeConfigNode(ctx)
db, err := stack.OpenDatabase("chaindata", 0, 0, "", true)
db, err := stack.OpenDatabaseWithOptions("chaindata", node.DatabaseOptions{ReadOnly: true})
if err != nil {
return err
}
@ -422,6 +475,10 @@ func importHistory(ctx *cli.Context) error {
network = "mainnet"
case ctx.Bool(utils.SepoliaFlag.Name):
network = "sepolia"
case ctx.Bool(utils.HoleskyFlag.Name):
network = "holesky"
case ctx.Bool(utils.HoodiFlag.Name):
network = "hoodi"
}
} else {
// No network flag set, try to determine network based on files
@ -445,7 +502,7 @@ func importHistory(ctx *cli.Context) error {
network = networks[0]
}
if err := utils.ImportHistory(chain, db, dir, network); err != nil {
if err := utils.ImportHistory(chain, dir, network); err != nil {
return err
}
fmt.Printf("Import done in %v\n", time.Since(start))
@ -598,3 +655,135 @@ func hashish(x string) bool {
_, err := strconv.Atoi(x)
return err != nil
}
func pruneHistory(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
// Open the chain database
chain, chaindb := utils.MakeChain(ctx, stack, false)
defer chaindb.Close()
defer chain.Stop()
// Determine the prune point. This will be the first PoS block.
prunePoint, ok := history.PrunePoints[chain.Genesis().Hash()]
if !ok || prunePoint == nil {
return errors.New("prune point not found")
}
var (
mergeBlock = prunePoint.BlockNumber
mergeBlockHash = prunePoint.BlockHash.Hex()
)
// Check we're far enough past merge to ensure all data is in freezer
currentHeader := chain.CurrentHeader()
if currentHeader == nil {
return errors.New("current header not found")
}
if currentHeader.Number.Uint64() < mergeBlock+params.FullImmutabilityThreshold {
return fmt.Errorf("chain not far enough past merge block, need %d more blocks",
mergeBlock+params.FullImmutabilityThreshold-currentHeader.Number.Uint64())
}
// Double-check the prune block in db has the expected hash.
hash := rawdb.ReadCanonicalHash(chaindb, mergeBlock)
if hash != common.HexToHash(mergeBlockHash) {
return fmt.Errorf("merge block hash mismatch: got %s, want %s", hash.Hex(), mergeBlockHash)
}
log.Info("Starting history pruning", "head", currentHeader.Number, "tail", mergeBlock, "tailHash", mergeBlockHash)
start := time.Now()
rawdb.PruneTransactionIndex(chaindb, mergeBlock)
if _, err := chaindb.TruncateTail(mergeBlock); err != nil {
return fmt.Errorf("failed to truncate ancient data: %v", err)
}
log.Info("History pruning completed", "tail", mergeBlock, "elapsed", common.PrettyDuration(time.Since(start)))
// TODO(s1na): what if there is a crash between the two prune operations?
return nil
}
// downladEra is the era1 file downloader tool.
func downloadEra(ctx *cli.Context) error {
flags.CheckExclusive(ctx, eraBlockFlag, eraEpochFlag, eraAllFlag)
// Resolve the network.
var network = "mainnet"
if utils.IsNetworkPreset(ctx) {
switch {
case ctx.IsSet(utils.MainnetFlag.Name):
case ctx.IsSet(utils.SepoliaFlag.Name):
network = "sepolia"
default:
return fmt.Errorf("unsupported network, no known era1 checksums")
}
}
// Resolve the destination directory.
stack, _ := makeConfigNode(ctx)
defer stack.Close()
ancients := stack.ResolveAncient("chaindata", "")
dir := filepath.Join(ancients, rawdb.ChainFreezerName, "era")
if ctx.IsSet(utils.EraFlag.Name) {
dir = filepath.Join(ancients, ctx.String(utils.EraFlag.Name))
}
baseURL := ctx.String(eraServerFlag.Name)
if baseURL == "" {
return fmt.Errorf("need --%s flag to download", eraServerFlag.Name)
}
l, err := eradl.New(baseURL, network)
if err != nil {
return err
}
switch {
case ctx.IsSet(eraAllFlag.Name):
return l.DownloadAll(dir)
case ctx.IsSet(eraBlockFlag.Name):
s := ctx.String(eraBlockFlag.Name)
start, end, ok := parseRange(s)
if !ok {
return fmt.Errorf("invalid block range: %q", s)
}
return l.DownloadBlockRange(start, end, dir)
case ctx.IsSet(eraEpochFlag.Name):
s := ctx.String(eraEpochFlag.Name)
start, end, ok := parseRange(s)
if !ok {
return fmt.Errorf("invalid epoch range: %q", s)
}
return l.DownloadEpochRange(start, end, dir)
default:
return fmt.Errorf("specify one of --%s, --%s, or --%s to download", eraAllFlag.Name, eraBlockFlag.Name, eraEpochFlag.Name)
}
}
func parseRange(s string) (start uint64, end uint64, ok bool) {
if m, _ := regexp.MatchString("[0-9]+", s); m {
start, err := strconv.ParseUint(s, 10, 64)
if err != nil {
return 0, 0, false
}
end = start
return start, end, true
}
if m, _ := regexp.MatchString("[0-9]+-[0-9]+", s); m {
s1, s2, _ := strings.Cut(s, "-")
start, err := strconv.ParseUint(s1, 10, 64)
if err != nil {
return 0, 0, false
}
end, err = strconv.ParseUint(s2, 10, 64)
if err != nil {
return 0, 0, false
}
return start, end, true
}
return 0, 0, false
}

45
cmd/geth/config.go
View file

@ -36,6 +36,7 @@ import (
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/eth/catalyst"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/internal/flags"
@ -180,6 +181,45 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
return stack, cfg
}
// constructs the disclaimer text block which will be printed in the logs upon
// startup when Geth is running in dev mode.
func constructDevModeBanner(ctx *cli.Context, cfg gethConfig) string {
devModeBanner := `You are running Geth in --dev mode. Please note the following:
1. This mode is only intended for fast, iterative development without assumptions on
security or persistence.
2. The database is created in memory unless specified otherwise. Therefore, shutting down
your computer or losing power will wipe your entire block data and chain state for
your dev environment.
3. A random, pre-allocated developer account will be available and unlocked as
eth.coinbase, which can be used for testing. The random dev account is temporary,
stored on a ramdisk, and will be lost if your machine is restarted.
4. Mining is enabled by default. However, the client will only seal blocks if transactions
are pending in the mempool. The miner's minimum accepted gas price is 1.
5. Networking is disabled; there is no listen-address, the maximum number of peers is set
to 0, and discovery is disabled.
`
if !ctx.IsSet(utils.DataDirFlag.Name) {
devModeBanner += fmt.Sprintf(`
Running in ephemeral mode. The following account has been prefunded in the genesis:
Account
------------------
0x%x (10^49 ETH)
`, cfg.Eth.Miner.PendingFeeRecipient)
if cfg.Eth.Miner.PendingFeeRecipient == utils.DeveloperAddr {
devModeBanner += fmt.Sprintf(`
Private Key
------------------
0x%x
`, crypto.FromECDSA(utils.DeveloperKey))
}
}
return devModeBanner
}
// makeFullNode loads geth configuration and creates the Ethereum backend.
func makeFullNode(ctx *cli.Context) *node.Node {
stack, cfg := makeConfigNode(ctx)
@ -239,6 +279,11 @@ func makeFullNode(ctx *cli.Context) *node.Node {
}
catalyst.RegisterSimulatedBeaconAPIs(stack, simBeacon)
stack.RegisterLifecycle(simBeacon)
banner := constructDevModeBanner(ctx, cfg)
for _, line := range strings.Split(banner, "\n") {
log.Warn(line)
}
} else if ctx.IsSet(utils.BeaconApiFlag.Name) {
// Start blsync mode.
srv := rpc.NewServer()

9
cmd/geth/consolecmd_test.go
View file

@ -39,9 +39,8 @@ const (
// child g gets a temporary data directory.
func runMinimalGeth(t *testing.T, args ...string) *testgeth {
// --holesky to make the 'writing genesis to disk' faster (no accounts)
// --networkid=1337 to avoid cache bump
// --syncmode=full to avoid allocating fast sync bloom
allArgs := []string{"--holesky", "--networkid", "1337", "--authrpc.port", "0", "--syncmode=full", "--port", "0",
allArgs := []string{"--holesky", "--authrpc.port", "0", "--syncmode=full", "--port", "0",
"--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64",
"--datadir.minfreedisk", "0"}
return runGeth(t, append(allArgs, args...)...)
@ -103,17 +102,17 @@ func TestAttachWelcome(t *testing.T) {
"--http", "--http.port", httpPort,
"--ws", "--ws.port", wsPort)
t.Run("ipc", func(t *testing.T) {
waitForEndpoint(t, ipc, 4*time.Second)
waitForEndpoint(t, ipc, 2*time.Minute)
testAttachWelcome(t, geth, "ipc:"+ipc, ipcAPIs)
})
t.Run("http", func(t *testing.T) {
endpoint := "http://127.0.0.1:" + httpPort
waitForEndpoint(t, endpoint, 4*time.Second)
waitForEndpoint(t, endpoint, 2*time.Minute)
testAttachWelcome(t, geth, endpoint, httpAPIs)
})
t.Run("ws", func(t *testing.T) {
endpoint := "ws://127.0.0.1:" + wsPort
waitForEndpoint(t, endpoint, 4*time.Second)
waitForEndpoint(t, endpoint, 2*time.Minute)
testAttachWelcome(t, geth, endpoint, httpAPIs)
})
geth.Kill()

94
cmd/geth/dbcmd.go
View file

@ -86,12 +86,10 @@ Remove blockchain and state databases`,
},
}
dbInspectCmd = &cli.Command{
Action: inspect,
Name: "inspect",
ArgsUsage: "<prefix> <start>",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
Action: inspect,
Name: "inspect",
ArgsUsage: "<prefix> <start>",
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Usage: "Inspect the storage size for each type of data in the database",
Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
}
@ -109,16 +107,13 @@ a data corruption.`,
Action: dbStats,
Name: "stats",
Usage: "Print leveldb statistics",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
}
dbCompactCmd = &cli.Command{
Action: dbCompact,
Name: "compact",
Usage: "Compact leveldb database. WARNING: May take a very long time",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
utils.CacheFlag,
utils.CacheDatabaseFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
@ -127,13 +122,11 @@ WARNING: This operation may take a very long time to finish, and may cause datab
corruption if it is aborted during execution'!`,
}
dbGetCmd = &cli.Command{
Action: dbGet,
Name: "get",
Usage: "Show the value of a database key",
ArgsUsage: "<hex-encoded key>",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
Action: dbGet,
Name: "get",
Usage: "Show the value of a database key",
ArgsUsage: "<hex-encoded key>",
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Description: "This command looks up the specified database key from the database.",
}
dbDeleteCmd = &cli.Command{
@ -141,9 +134,7 @@ corruption if it is aborted during execution'!`,
Name: "delete",
Usage: "Delete a database key (WARNING: may corrupt your database)",
ArgsUsage: "<hex-encoded key>",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Description: `This command deletes the specified database key from the database.
WARNING: This is a low-level operation which may cause database corruption!`,
}
@ -152,59 +143,47 @@ WARNING: This is a low-level operation which may cause database corruption!`,
Name: "put",
Usage: "Set the value of a database key (WARNING: may corrupt your database)",
ArgsUsage: "<hex-encoded key> <hex-encoded value>",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Description: `This command sets a given database key to the given value.
WARNING: This is a low-level operation which may cause database corruption!`,
}
dbGetSlotsCmd = &cli.Command{
Action: dbDumpTrie,
Name: "dumptrie",
Usage: "Show the storage key/values of a given storage trie",
ArgsUsage: "<hex-encoded state root> <hex-encoded account hash> <hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
Action: dbDumpTrie,
Name: "dumptrie",
Usage: "Show the storage key/values of a given storage trie",
ArgsUsage: "<hex-encoded state root> <hex-encoded account hash> <hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Description: "This command looks up the specified database key from the database.",
}
dbDumpFreezerIndex = &cli.Command{
Action: freezerInspect,
Name: "freezer-index",
Usage: "Dump out the index of a specific freezer table",
ArgsUsage: "<freezer-type> <table-type> <start (int)> <end (int)>",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
Action: freezerInspect,
Name: "freezer-index",
Usage: "Dump out the index of a specific freezer table",
ArgsUsage: "<freezer-type> <table-type> <start (int)> <end (int)>",
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Description: "This command displays information about the freezer index.",
}
dbImportCmd = &cli.Command{
Action: importLDBdata,
Name: "import",
Usage: "Imports leveldb-data from an exported RLP dump.",
ArgsUsage: "<dumpfile> <start (optional)",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
Action: importLDBdata,
Name: "import",
Usage: "Imports leveldb-data from an exported RLP dump.",
ArgsUsage: "<dumpfile> <start (optional)",
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Description: "The import command imports the specific chain data from an RLP encoded stream.",
}
dbExportCmd = &cli.Command{
Action: exportChaindata,
Name: "export",
Usage: "Exports the chain data into an RLP dump. If the <dumpfile> has .gz suffix, gzip compression will be used.",
ArgsUsage: "<type> <dumpfile>",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
Action: exportChaindata,
Name: "export",
Usage: "Exports the chain data into an RLP dump. If the <dumpfile> has .gz suffix, gzip compression will be used.",
ArgsUsage: "<type> <dumpfile>",
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
}
dbMetadataCmd = &cli.Command{
Action: showMetaData,
Name: "metadata",
Usage: "Shows metadata about the chain status.",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabaseFlags),
Action: showMetaData,
Name: "metadata",
Usage: "Shows metadata about the chain status.",
Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
Description: "Shows metadata about the chain status.",
}
dbInspectHistoryCmd = &cli.Command{
@ -213,7 +192,6 @@ WARNING: This is a low-level operation which may cause database corruption!`,
Usage: "Inspect the state history within block range",
ArgsUsage: "<address> [OPTIONAL <storage-slot>]",
Flags: slices.Concat([]cli.Flag{
utils.SyncModeFlag,
&cli.Uint64Flag{
Name: "start",
Usage: "block number of the range start, zero means earliest history",

29
cmd/geth/main.go
View file

@ -91,16 +91,9 @@ var (
utils.LogNoHistoryFlag,
utils.LogExportCheckpointsFlag,
utils.StateHistoryFlag,
utils.LightServeFlag, // deprecated
utils.LightIngressFlag, // deprecated
utils.LightEgressFlag, // deprecated
utils.LightMaxPeersFlag, // deprecated
utils.LightNoPruneFlag, // deprecated
utils.LightKDFFlag,
utils.LightNoSyncServeFlag, // deprecated
utils.EthRequiredBlocksFlag,
utils.LegacyWhitelistFlag, // deprecated
utils.BloomFilterSizeFlag,
utils.CacheFlag,
utils.CacheDatabaseFlag,
utils.CacheTrieFlag,
@ -142,7 +135,6 @@ var (
utils.VMTraceJsonConfigFlag,
utils.NetworkIdFlag,
utils.EthStatsURLFlag,
utils.NoCompactionFlag,
utils.GpoBlocksFlag,
utils.GpoPercentileFlag,
utils.GpoMaxGasPriceFlag,
@ -158,6 +150,7 @@ var (
utils.BeaconGenesisRootFlag,
utils.BeaconGenesisTimeFlag,
utils.BeaconCheckpointFlag,
utils.BeaconCheckpointFileFlag,
}, utils.NetworkFlags, utils.DatabaseFlags)
rpcFlags = []cli.Flag{
@ -226,6 +219,8 @@ func init() {
removedbCommand,
dumpCommand,
dumpGenesisCommand,
pruneHistoryCommand,
downloadEraCommand,
// See accountcmd.go:
accountCommand,
walletCommand,
@ -299,24 +294,6 @@ func prepare(ctx *cli.Context) {
case ctx.IsSet(utils.HoodiFlag.Name):
log.Info("Starting Geth on Hoodi testnet...")
case ctx.IsSet(utils.DeveloperFlag.Name):
log.Info("Starting Geth in ephemeral dev mode...")
log.Warn(`You are running Geth in --dev mode. Please note the following:
1. This mode is only intended for fast, iterative development without assumptions on
security or persistence.
2. The database is created in memory unless specified otherwise. Therefore, shutting down
your computer or losing power will wipe your entire block data and chain state for
your dev environment.
3. A random, pre-allocated developer account will be available and unlocked as
eth.coinbase, which can be used for testing. The random dev account is temporary,
stored on a ramdisk, and will be lost if your machine is restarted.
4. Mining is enabled by default. However, the client will only seal blocks if transactions
are pending in the mempool. The miner's minimum accepted gas price is 1.
5. Networking is disabled; there is no listen-address, the maximum number of peers is set
to 0, and discovery is disabled.
`)
case !ctx.IsSet(utils.NetworkIdFlag.Name):
log.Info("Starting Geth on Ethereum mainnet...")
}

13
cmd/utils/cmd.go
View file

@ -246,8 +246,9 @@ func readList(filename string) ([]string, error) {
}
// ImportHistory imports Era1 files containing historical block information,
// starting from genesis.
func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error {
// starting from genesis. The assumption is held that the provided chain
// segment in Era1 file should all be canonical and verified.
func ImportHistory(chain *core.BlockChain, dir string, network string) error {
if chain.CurrentSnapBlock().Number.BitLen() != 0 {
return errors.New("history import only supported when starting from genesis")
}
@ -308,12 +309,8 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ
if err != nil {
return fmt.Errorf("error reading receipts %d: %w", it.Number(), err)
}
if status, err := chain.HeaderChain().InsertHeaderChain([]*types.Header{block.Header()}, start); err != nil {
return fmt.Errorf("error inserting header %d: %w", it.Number(), err)
} else if status != core.CanonStatTy {
return fmt.Errorf("error inserting header %d, not canon: %v", it.Number(), status)
}
if _, err := chain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{receipts}, 2^64-1); err != nil {
encReceipts := types.EncodeBlockReceiptLists([]types.Receipts{receipts})
if _, err := chain.InsertReceiptChain([]*types.Block{block}, encReceipts, 2^64-1); err != nil {
return fmt.Errorf("error inserting body %d: %w", it.Number(), err)
}
imported += 1

14
cmd/utils/export_test.go
View file

@ -54,8 +54,8 @@ func (iter *testIterator) Next() (byte, []byte, []byte, bool) {
if iter.index == 42 {
iter.index += 1
}
return OpBatchAdd, []byte(fmt.Sprintf("key-%04d", iter.index)),
[]byte(fmt.Sprintf("value %d", iter.index)), true
return OpBatchAdd, fmt.Appendf(nil, "key-%04d", iter.index),
fmt.Appendf(nil, "value %d", iter.index), true
}
func (iter *testIterator) Release() {}
@ -72,7 +72,7 @@ func testExport(t *testing.T, f string) {
}
// verify
for i := 0; i < 1000; i++ {
v, err := db.Get([]byte(fmt.Sprintf("key-%04d", i)))
v, err := db.Get(fmt.Appendf(nil, "key-%04d", i))
if (i < 5 || i == 42) && err == nil {
t.Fatalf("expected no element at idx %d, got '%v'", i, string(v))
}
@ -85,7 +85,7 @@ func testExport(t *testing.T, f string) {
}
}
}
v, err := db.Get([]byte(fmt.Sprintf("key-%04d", 1000)))
v, err := db.Get(fmt.Appendf(nil, "key-%04d", 1000))
if err == nil {
t.Fatalf("expected no element at idx %d, got '%v'", 1000, string(v))
}
@ -120,7 +120,7 @@ func (iter *deletionIterator) Next() (byte, []byte, []byte, bool) {
if iter.index == 42 {
iter.index += 1
}
return OpBatchDel, []byte(fmt.Sprintf("key-%04d", iter.index)), nil, true
return OpBatchDel, fmt.Appendf(nil, "key-%04d", iter.index), nil, true
}
func (iter *deletionIterator) Release() {}
@ -132,14 +132,14 @@ func testDeletion(t *testing.T, f string) {
}
db := rawdb.NewMemoryDatabase()
for i := 0; i < 1000; i++ {
db.Put([]byte(fmt.Sprintf("key-%04d", i)), []byte(fmt.Sprintf("value %d", i)))
db.Put(fmt.Appendf(nil, "key-%04d", i), fmt.Appendf(nil, "value %d", i))
}
err = ImportLDBData(db, f, 5, make(chan struct{}))
if err != nil {
t.Fatal(err)
}
for i := 0; i < 1000; i++ {
v, err := db.Get([]byte(fmt.Sprintf("key-%04d", i)))
v, err := db.Get(fmt.Appendf(nil, "key-%04d", i))
if i < 5 || i == 42 {
if err != nil {
t.Fatalf("expected element at idx %d, got '%v'", i, err)

153
cmd/utils/flags.go
View file

@ -111,6 +111,11 @@ var (
Usage: "Root directory for ancient data (default = inside chaindata)",
Category: flags.EthCategory,
}
EraFlag = &flags.DirectoryFlag{
Name: "datadir.era",
Usage: "Root directory for era1 history (default = inside ancient/chain)",
Category: flags.EthCategory,
}
MinFreeDiskSpaceFlag = &flags.DirectoryFlag{
Name: "datadir.minfreedisk",
Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)",
@ -145,7 +150,7 @@ var (
}
SepoliaFlag = &cli.BoolFlag{
Name: "sepolia",
Usage: "Sepolia network: pre-configured proof-of-work test network",
Usage: "Sepolia network: pre-configured proof-of-stake test network",
Category: flags.EthCategory,
}
HoleskyFlag = &cli.BoolFlag{
@ -342,6 +347,11 @@ var (
Usage: "Beacon chain weak subjectivity checkpoint block hash",
Category: flags.BeaconCategory,
}
BeaconCheckpointFileFlag = &cli.StringFlag{
Name: "beacon.checkpoint.file",
Usage: "Beacon chain weak subjectivity checkpoint import/export file",
Category: flags.BeaconCategory,
}
BlsyncApiFlag = &cli.StringFlag{
Name: "blsync.engine.api",
Usage: "Target EL engine API URL",
@ -972,6 +982,7 @@ var (
DatabaseFlags = []cli.Flag{
DataDirFlag,
AncientFlag,
EraFlag,
RemoteDBFlag,
DBEngineFlag,
StateSchemeFlag,
@ -979,6 +990,12 @@ var (
}
)
// default account to prefund when running Geth in dev mode
var (
DeveloperKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
DeveloperAddr = crypto.PubkeyToAddress(DeveloperKey.PublicKey)
)
// MakeDataDir retrieves the currently requested data directory, terminating
// if none (or the empty string) is specified. If the node is starting a testnet,
// then a subdirectory of the specified datadir will be used.
@ -1240,28 +1257,6 @@ func setIPC(ctx *cli.Context, cfg *node.Config) {
}
}
// setLes shows the deprecation warnings for LES flags.
func setLes(ctx *cli.Context, cfg *ethconfig.Config) {
if ctx.IsSet(LightServeFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightServeFlag.Name)
}
if ctx.IsSet(LightIngressFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightIngressFlag.Name)
}
if ctx.IsSet(LightEgressFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightEgressFlag.Name)
}
if ctx.IsSet(LightMaxPeersFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightMaxPeersFlag.Name)
}
if ctx.IsSet(LightNoPruneFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightNoPruneFlag.Name)
}
if ctx.IsSet(LightNoSyncServeFlag.Name) {
log.Warn("The light server has been deprecated, please remove this flag", "flag", LightNoSyncServeFlag.Name)
}
}
// MakeDatabaseHandles raises out the number of allowed file handles per process
// for Geth and returns half of the allowance to assign to the database.
func MakeDatabaseHandles(max int) int {
@ -1566,8 +1561,8 @@ func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) {
// SetEthConfig applies eth-related command line flags to the config.
func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
// Avoid conflicting network flags
flags.CheckExclusive(ctx, MainnetFlag, DeveloperFlag, SepoliaFlag, HoleskyFlag, HoodiFlag)
// Avoid conflicting network flags, don't allow network id override on preset networks
flags.CheckExclusive(ctx, MainnetFlag, DeveloperFlag, SepoliaFlag, HoleskyFlag, HoodiFlag, NetworkIdFlag)
flags.CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
// Set configurations from CLI flags
@ -1577,7 +1572,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
setBlobPool(ctx, &cfg.BlobPool)
setMiner(ctx, &cfg.Miner)
setRequiredBlocks(ctx, cfg)
setLes(ctx, cfg)
// Cap the cache allowance and tune the garbage collector
mem, err := gopsutil.VirtualMemory()
@ -1625,6 +1619,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
if ctx.IsSet(AncientFlag.Name) {
cfg.DatabaseFreezer = ctx.String(AncientFlag.Name)
}
if ctx.IsSet(EraFlag.Name) {
cfg.DatabaseEra = ctx.String(EraFlag.Name)
}
if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
@ -1659,12 +1656,16 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
log.Warn("The flag --txlookuplimit is deprecated and will be removed, please use --history.transactions")
cfg.TransactionHistory = ctx.Uint64(TxLookupLimitFlag.Name)
}
if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 {
cfg.TransactionHistory = 0
log.Warn("Disabled transaction unindexing for archive node")
if ctx.String(GCModeFlag.Name) == "archive" {
if cfg.TransactionHistory != 0 {
cfg.TransactionHistory = 0
log.Warn("Disabled transaction unindexing for archive node")
}
cfg.StateScheme = rawdb.HashScheme
log.Warn("Forcing hash state-scheme for archive mode")
if cfg.StateScheme != rawdb.HashScheme {
cfg.StateScheme = rawdb.HashScheme
log.Warn("Forcing hash state-scheme for archive mode")
}
}
if ctx.IsSet(LogHistoryFlag.Name) {
cfg.LogHistory = ctx.Uint64(LogHistoryFlag.Name)
@ -1703,7 +1704,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
}
}
if ctx.IsSet(VMEnableDebugFlag.Name) {
// TODO(fjl): force-enable this in --dev mode
cfg.EnablePreimageRecording = ctx.Bool(VMEnableDebugFlag.Name)
}
@ -1734,34 +1734,25 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
// Override any default configs for hard coded networks.
switch {
case ctx.Bool(MainnetFlag.Name):
if !ctx.IsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 1
}
cfg.NetworkId = 1
cfg.Genesis = core.DefaultGenesisBlock()
SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash)
case ctx.Bool(HoleskyFlag.Name):
if !ctx.IsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 17000
}
cfg.NetworkId = 17000
cfg.Genesis = core.DefaultHoleskyGenesisBlock()
SetDNSDiscoveryDefaults(cfg, params.HoleskyGenesisHash)
case ctx.Bool(SepoliaFlag.Name):
if !ctx.IsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 11155111
}
cfg.NetworkId = 11155111
cfg.Genesis = core.DefaultSepoliaGenesisBlock()
SetDNSDiscoveryDefaults(cfg, params.SepoliaGenesisHash)
case ctx.Bool(HoodiFlag.Name):
if !ctx.IsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 560048
}
cfg.NetworkId = 560048
cfg.Genesis = core.DefaultHoodiGenesisBlock()
SetDNSDiscoveryDefaults(cfg, params.HoodiGenesisHash)
case ctx.Bool(DeveloperFlag.Name):
if !ctx.IsSet(NetworkIdFlag.Name) {
cfg.NetworkId = 1337
}
cfg.NetworkId = 1337
cfg.SyncMode = ethconfig.FullSync
cfg.EnablePreimageRecording = true
// Create new developer account or reuse existing one
var (
developer accounts.Account
@ -1793,9 +1784,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
} else if accs := ks.Accounts(); len(accs) > 0 {
developer = ks.Accounts()[0]
} else {
developer, err = ks.NewAccount(passphrase)
developer, err = ks.ImportECDSA(DeveloperKey, passphrase)
if err != nil {
Fatalf("Failed to create developer account: %v", err)
Fatalf("Failed to import developer account: %v", err)
}
}
// Make sure the address is configured as fee recipient, otherwise
@ -1810,14 +1801,18 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
}
log.Info("Using developer account", "address", developer.Address)
// Create a new developer genesis block or reuse existing one
// configure default developer genesis which will be used unless a
// datadir is specified and a chain is preexisting at that location.
cfg.Genesis = core.DeveloperGenesisBlock(ctx.Uint64(DeveloperGasLimitFlag.Name), &developer.Address)
// If a datadir is specified, ensure that any preexisting chain in that location
// has a configuration that is compatible with dev mode: it must be merged at genesis.
if ctx.IsSet(DataDirFlag.Name) {
chaindb := tryMakeReadOnlyDatabase(ctx, stack)
if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) {
cfg.Genesis = nil // fallback to db content
// signal fallback to preexisting chain on disk
cfg.Genesis = nil
//validate genesis has PoS enabled in block 0
genesis, err := core.ReadGenesis(chaindb)
if err != nil {
Fatalf("Could not read genesis from database: %v", err)
@ -1845,10 +1840,16 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
if ctx.String(CryptoKZGFlag.Name) != "gokzg" && ctx.String(CryptoKZGFlag.Name) != "ckzg" {
Fatalf("--%s flag must be 'gokzg' or 'ckzg'", CryptoKZGFlag.Name)
}
log.Info("Initializing the KZG library", "backend", ctx.String(CryptoKZGFlag.Name))
if err := kzg4844.UseCKZG(ctx.String(CryptoKZGFlag.Name) == "ckzg"); err != nil {
Fatalf("Failed to set KZG library implementation to %s: %v", ctx.String(CryptoKZGFlag.Name), err)
}
// The initialization of the KZG library can take up to 2 seconds
// We start this here in parallel, so it should be available
// once we start executing blocks. It's threadsafe.
go func() {
log.Info("Initializing the KZG library", "backend", ctx.String(CryptoKZGFlag.Name))
if err := kzg4844.UseCKZG(ctx.String(CryptoKZGFlag.Name) == "ckzg"); err != nil {
Fatalf("Failed to set KZG library implementation to %s: %v", ctx.String(CryptoKZGFlag.Name), err)
}
}()
// VM tracing config.
if ctx.IsSet(VMTraceFlag.Name) {
if name := ctx.String(VMTraceFlag.Name); name != "" {
@ -1886,7 +1887,7 @@ func MakeBeaconLightConfig(ctx *cli.Context) bparams.ClientConfig {
if !ctx.IsSet(BeaconGenesisTimeFlag.Name) {
Fatalf("Custom beacon chain config is specified but genesis time is missing")
}
if !ctx.IsSet(BeaconCheckpointFlag.Name) {
if !ctx.IsSet(BeaconCheckpointFlag.Name) && !ctx.IsSet(BeaconCheckpointFileFlag.Name) {
Fatalf("Custom beacon chain config is specified but checkpoint is missing")
}
config.ChainConfig = bparams.ChainConfig{
@ -1911,13 +1912,28 @@ func MakeBeaconLightConfig(ctx *cli.Context) bparams.ClientConfig {
}
}
// Checkpoint is required with custom chain config and is optional with pre-defined config
if ctx.IsSet(BeaconCheckpointFlag.Name) {
if c, err := hexutil.Decode(ctx.String(BeaconCheckpointFlag.Name)); err == nil && len(c) <= 32 {
copy(config.Checkpoint[:len(c)], c)
} else {
Fatalf("Invalid hex string", "beacon.checkpoint", ctx.String(BeaconCheckpointFlag.Name), "error", err)
// If both checkpoint block hash and checkpoint file are specified then the
// client is initialized with the specified block hash and new checkpoints
// are saved to the specified file.
if ctx.IsSet(BeaconCheckpointFileFlag.Name) {
if _, err := config.SetCheckpointFile(ctx.String(BeaconCheckpointFileFlag.Name)); err != nil {
Fatalf("Could not load beacon checkpoint file", "beacon.checkpoint.file", ctx.String(BeaconCheckpointFileFlag.Name), "error", err)
}
}
if ctx.IsSet(BeaconCheckpointFlag.Name) {
hex := ctx.String(BeaconCheckpointFlag.Name)
c, err := hexutil.Decode(hex)
if err != nil {
Fatalf("Invalid hex string", "beacon.checkpoint", hex, "error", err)
}
if len(c) != 32 {
Fatalf("Invalid hex string length", "beacon.checkpoint", hex, "length", len(c))
}
copy(config.Checkpoint[:len(c)], c)
}
if config.Checkpoint == (common.Hash{}) {
Fatalf("Beacon checkpoint not specified")
}
config.Apis = ctx.StringSlice(BeaconApiFlag.Name)
if config.Apis == nil {
Fatalf("Beacon node light client API URL not specified")
@ -2023,7 +2039,6 @@ func SetupMetrics(cfg *metrics.Config) {
log.Info("Enabling metrics export to InfluxDB")
go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap)
} else if enableExportV2 {
tagsMap := SplitTagsFlag(cfg.InfluxDBTags)
log.Info("Enabling metrics export to InfluxDB (v2)")
go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, token, bucket, organization, "geth.", tagsMap)
}
@ -2076,7 +2091,15 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
}
chainDb = remotedb.New(client)
default:
chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "eth/db/chaindata/", readonly)
options := node.DatabaseOptions{
ReadOnly: readonly,
Cache: cache,
Handles: handles,
AncientsDirectory: ctx.String(AncientFlag.Name),
MetricsNamespace: "eth/db/chaindata/",
EraDirectory: ctx.String(EraFlag.Name),
}
chainDb, err = stack.OpenDatabaseWithOptions("chaindata", options)
}
if err != nil {
Fatalf("Could not open database: %v", err)
@ -2185,6 +2208,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
}
if !ctx.Bool(SnapshotFlag.Name) {
cache.SnapshotLimit = 0 // Disabled
} else if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheSnapshotFlag.Name) {
cache.SnapshotLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheSnapshotFlag.Name) / 100
}
// If we're in readonly, do not bother generating snapshot data.
if readonly {

37
cmd/utils/flags_legacy.go
View file

@ -39,12 +39,6 @@ var DeprecatedFlags = []cli.Flag{
CacheTrieRejournalFlag,
LegacyDiscoveryV5Flag,
TxLookupLimitFlag,
LightServeFlag,
LightIngressFlag,
LightEgressFlag,
LightMaxPeersFlag,
LightNoPruneFlag,
LightNoSyncServeFlag,
LogBacktraceAtFlag,
LogDebugFlag,
MinerNewPayloadTimeoutFlag,
@ -88,37 +82,6 @@ var (
Value: ethconfig.Defaults.TransactionHistory,
Category: flags.DeprecatedCategory,
}
// Light server and client settings, Deprecated November 2023
LightServeFlag = &cli.IntFlag{
Name: "light.serve",
Usage: "Maximum percentage of time allowed for serving LES requests (deprecated)",
Category: flags.DeprecatedCategory,
}
LightIngressFlag = &cli.IntFlag{
Name: "light.ingress",
Usage: "Incoming bandwidth limit for serving light clients (deprecated)",
Category: flags.DeprecatedCategory,
}
LightEgressFlag = &cli.IntFlag{
Name: "light.egress",
Usage: "Outgoing bandwidth limit for serving light clients (deprecated)",
Category: flags.DeprecatedCategory,
}
LightMaxPeersFlag = &cli.IntFlag{
Name: "light.maxpeers",
Usage: "Maximum number of light clients to serve, or light servers to attach to (deprecated)",
Category: flags.DeprecatedCategory,
}
LightNoPruneFlag = &cli.BoolFlag{
Name: "light.nopruning",
Usage: "Disable ancient light chain data pruning (deprecated)",
Category: flags.DeprecatedCategory,
}
LightNoSyncServeFlag = &cli.BoolFlag{
Name: "light.nosyncserve",
Usage: "Enables serving light clients before syncing (deprecated)",
Category: flags.DeprecatedCategory,
}
// Deprecated November 2023
LogBacktraceAtFlag = &cli.StringFlag{
Name: "log.backtrace",

4
cmd/utils/history_test.go
View file

@ -158,7 +158,7 @@ func TestHistoryImportAndExport(t *testing.T) {
}
// Now import Era.
db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db2, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
if err != nil {
panic(err)
}
@ -171,7 +171,7 @@ func TestHistoryImportAndExport(t *testing.T) {
if err != nil {
t.Fatalf("unable to initialize chain: %v", err)
}
if err := ImportHistory(imported, db2, dir, "mainnet"); err != nil {
if err := ImportHistory(imported, dir, "mainnet"); err != nil {
t.Fatalf("failed to import chain: %v", err)
}
if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() {

1
cmd/workload/README.md
View file

@ -26,4 +26,5 @@ the following commands (in this directory) against a synced mainnet node:
```shell
> go run . filtergen --queries queries/filter_queries_mainnet.json http://host:8545
> go run . historygen --history-tests queries/history_mainnet.json http://host:8545
> go run . tracegen --trace-tests queries/trace_mainnet.json http://host:8545
```

104
cmd/workload/client.go Normal file
View file

@ -0,0 +1,104 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/ethclient/gethclient"
"github.com/ethereum/go-ethereum/rpc"
"github.com/urfave/cli/v2"
)
type client struct {
Eth *ethclient.Client
Geth *gethclient.Client
RPC *rpc.Client
}
func makeClient(ctx *cli.Context) *client {
if ctx.NArg() < 1 {
exit("missing RPC endpoint URL as command-line argument")
}
url := ctx.Args().First()
cl, err := rpc.Dial(url)
if err != nil {
exit(fmt.Errorf("could not create RPC client at %s: %v", url, err))
}
return &client{
RPC: cl,
Eth: ethclient.NewClient(cl),
Geth: gethclient.New(cl),
}
}
type simpleBlock struct {
Number hexutil.Uint64 `json:"number"`
Hash common.Hash `json:"hash"`
}
type simpleTransaction struct {
Hash common.Hash `json:"hash"`
TransactionIndex hexutil.Uint64 `json:"transactionIndex"`
}
func (c *client) getBlockByHash(ctx context.Context, arg common.Hash, fullTx bool) (*simpleBlock, error) {
var r *simpleBlock
err := c.RPC.CallContext(ctx, &r, "eth_getBlockByHash", arg, fullTx)
return r, err
}
func (c *client) getBlockByNumber(ctx context.Context, arg uint64, fullTx bool) (*simpleBlock, error) {
var r *simpleBlock
err := c.RPC.CallContext(ctx, &r, "eth_getBlockByNumber", hexutil.Uint64(arg), fullTx)
return r, err
}
func (c *client) getTransactionByBlockHashAndIndex(ctx context.Context, block common.Hash, index uint64) (*simpleTransaction, error) {
var r *simpleTransaction
err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockHashAndIndex", block, hexutil.Uint64(index))
return r, err
}
func (c *client) getTransactionByBlockNumberAndIndex(ctx context.Context, block uint64, index uint64) (*simpleTransaction, error) {
var r *simpleTransaction
err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockNumberAndIndex", hexutil.Uint64(block), hexutil.Uint64(index))
return r, err
}
func (c *client) getBlockTransactionCountByHash(ctx context.Context, block common.Hash) (uint64, error) {
var r hexutil.Uint64
err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByHash", block)
return uint64(r), err
}
func (c *client) getBlockTransactionCountByNumber(ctx context.Context, block uint64) (uint64, error) {
var r hexutil.Uint64
err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByNumber", hexutil.Uint64(block))
return uint64(r), err
}
func (c *client) getBlockReceipts(ctx context.Context, arg any) ([]*types.Receipt, error) {
var result []*types.Receipt
err := c.RPC.CallContext(ctx, &result, "eth_getBlockReceipts", arg)
return result, err
}

31
cmd/workload/filtertest.go
View file

@ -45,11 +45,11 @@ func newFilterTestSuite(cfg testConfig) *filterTestSuite {
return s
}
func (s *filterTestSuite) allTests() []utesting.Test {
return []utesting.Test{
{Name: "Filter/ShortRange", Fn: s.filterShortRange},
{Name: "Filter/LongRange", Fn: s.filterLongRange, Slow: true},
{Name: "Filter/FullRange", Fn: s.filterFullRange, Slow: true},
func (s *filterTestSuite) allTests() []workloadTest {
return []workloadTest{
newWorkLoadTest("Filter/ShortRange", s.filterShortRange),
newSlowWorkloadTest("Filter/LongRange", s.filterLongRange),
newSlowWorkloadTest("Filter/FullRange", s.filterFullRange),
}
}
@ -109,6 +109,9 @@ func (s *filterTestSuite) filterFullRange(t *utesting.T) {
func (s *filterTestSuite) queryAndCheck(t *utesting.T, query *filterQuery) {
query.run(s.cfg.client, s.cfg.historyPruneBlock)
if query.Err == errPrunedHistory {
return
}
if query.Err != nil {
t.Errorf("Filter query failed (fromBlock: %d toBlock: %d addresses: %v topics: %v error: %v)", query.FromBlock, query.ToBlock, query.Address, query.Topics, query.Err)
return
@ -126,6 +129,9 @@ func (s *filterTestSuite) fullRangeQueryAndCheck(t *utesting.T, query *filterQue
Topics: query.Topics,
}
frQuery.run(s.cfg.client, s.cfg.historyPruneBlock)
if frQuery.Err == errPrunedHistory {
return
}
if frQuery.Err != nil {
t.Errorf("Full range filter query failed (addresses: %v topics: %v error: %v)", frQuery.Address, frQuery.Topics, frQuery.Err)
return
@ -206,14 +212,11 @@ func (fq *filterQuery) run(client *client, historyPruneBlock *uint64) {
Addresses: fq.Address,
Topics: fq.Topics,
})
if err != nil {
if err = validateHistoryPruneErr(fq.Err, uint64(fq.FromBlock), historyPruneBlock); err == errPrunedHistory {
return
} else if err != nil {
fmt.Printf("Filter query failed: fromBlock: %d toBlock: %d addresses: %v topics: %v error: %v\n",
fq.FromBlock, fq.ToBlock, fq.Address, fq.Topics, err)
}
fq.Err = err
}
fq.results = logs
fq.Err = validateHistoryPruneErr(err, uint64(fq.FromBlock), historyPruneBlock)
}
func (fq *filterQuery) printError() {
fmt.Printf("Filter query failed: fromBlock: %d toBlock: %d addresses: %v topics: %v error: %v\n",
fq.FromBlock, fq.ToBlock, fq.Address, fq.Topics, fq.Err)
}

24
cmd/workload/filtertestgen.go
View file

@ -40,7 +40,6 @@ var (
Action: filterGenCmd,
Flags: []cli.Flag{
filterQueryFileFlag,
filterErrorFileFlag,
},
}
filterQueryFileFlag = &cli.StringFlag{
@ -72,8 +71,8 @@ func filterGenCmd(ctx *cli.Context) error {
query := f.newQuery()
query.run(f.client, nil)
if query.Err != nil {
f.errors = append(f.errors, query)
continue
query.printError()
exit("filter query failed")
}
if len(query.results) > 0 && len(query.results) <= maxFilterResultSize {
for {
@ -90,8 +89,8 @@ func filterGenCmd(ctx *cli.Context) error {
)
}
if extQuery.Err != nil {
f.errors = append(f.errors, extQuery)
break
extQuery.printError()
exit("filter query failed")
}
if len(extQuery.results) > maxFilterResultSize {
break
@ -101,7 +100,6 @@ func filterGenCmd(ctx *cli.Context) error {
f.storeQuery(query)
if time.Since(lastWrite) > time.Second*10 {
f.writeQueries()
f.writeErrors()
lastWrite = time.Now()
}
}
@ -112,18 +110,15 @@ func filterGenCmd(ctx *cli.Context) error {
type filterTestGen struct {
client *client
queryFile string
errorFile string
finalizedBlock int64
queries [filterBuckets][]*filterQuery
errors []*filterQuery
}
func newFilterTestGen(ctx *cli.Context) *filterTestGen {
return &filterTestGen{
client: makeClient(ctx),
queryFile: ctx.String(filterQueryFileFlag.Name),
errorFile: ctx.String(filterErrorFileFlag.Name),
}
}
@ -360,17 +355,6 @@ func (s *filterTestGen) writeQueries() {
file.Close()
}
// writeQueries serializes the generated errors to the error file.
func (s *filterTestGen) writeErrors() {
file, err := os.Create(s.errorFile)
if err != nil {
exit(fmt.Errorf("Error creating filter error file %s: %v", s.errorFile, err))
return
}
defer file.Close()
json.NewEncoder(file).Encode(s.errors)
}
func mustGetFinalizedBlock(client *client) int64 {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()

39
cmd/workload/filtertestperf.go
View file

@ -17,8 +17,10 @@
package main
import (
"encoding/json"
"fmt"
"math/rand"
"os"
"slices"
"sort"
"time"
@ -41,7 +43,7 @@ var (
}
)
const passCount = 1
const passCount = 3
func filterPerfCmd(ctx *cli.Context) error {
cfg := testConfigFromCLI(ctx)
@ -61,7 +63,10 @@ func filterPerfCmd(ctx *cli.Context) error {
}
// Run test queries.
var failed, mismatch int
var (
failed, pruned, mismatch int
errors []*filterQuery
)
for i := 1; i <= passCount; i++ {
fmt.Println("Performance test pass", i, "/", passCount)
for len(queries) > 0 {
@ -71,27 +76,35 @@ func filterPerfCmd(ctx *cli.Context) error {
queries = queries[:len(queries)-1]
start := time.Now()
qt.query.run(cfg.client, cfg.historyPruneBlock)
if qt.query.Err == errPrunedHistory {
pruned++
continue
}
qt.runtime = append(qt.runtime, time.Since(start))
slices.Sort(qt.runtime)
qt.medianTime = qt.runtime[len(qt.runtime)/2]
if qt.query.Err != nil {
qt.query.printError()
errors = append(errors, qt.query)
failed++
continue
}
if rhash := qt.query.calculateHash(); *qt.query.ResultHash != rhash {
fmt.Printf("Filter query result mismatch: fromBlock: %d toBlock: %d addresses: %v topics: %v expected hash: %064x calculated hash: %064x\n", qt.query.FromBlock, qt.query.ToBlock, qt.query.Address, qt.query.Topics, *qt.query.ResultHash, rhash)
errors = append(errors, qt.query)
mismatch++
continue
}
processed = append(processed, qt)
if len(processed)%50 == 0 {
fmt.Println(" processed:", len(processed), "remaining", len(queries), "failed:", failed, "result mismatch:", mismatch)
fmt.Println(" processed:", len(processed), "remaining", len(queries), "failed:", failed, "pruned:", pruned, "result mismatch:", mismatch)
}
}
queries, processed = processed, nil
}
// Show results and stats.
fmt.Println("Performance test finished; processed:", len(queries), "failed:", failed, "result mismatch:", mismatch)
fmt.Println("Performance test finished; processed:", len(queries), "failed:", failed, "pruned:", pruned, "result mismatch:", mismatch)
stats := make([]bucketStats, len(f.queries))
var wildcardStats bucketStats
for _, qt := range queries {
@ -114,11 +127,14 @@ func filterPerfCmd(ctx *cli.Context) error {
sort.Slice(queries, func(i, j int) bool {
return queries[i].medianTime > queries[j].medianTime
})
for i := 0; i < 10; i++ {
q := queries[i]
for i, q := range queries {
if i >= 10 {
break
}
fmt.Printf("Most expensive query #%-2d median runtime: %13v max runtime: %13v result count: %4d fromBlock: %9d toBlock: %9d addresses: %v topics: %v\n",
i+1, q.medianTime, q.runtime[len(q.runtime)-1], len(q.query.results), q.query.FromBlock, q.query.ToBlock, q.query.Address, q.query.Topics)
}
writeErrors(ctx.String(filterErrorFileFlag.Name), errors)
return nil
}
@ -135,3 +151,14 @@ func (st *bucketStats) print(name string) {
fmt.Printf("%-20s queries: %4d average block length: %12.2f average log count: %7.2f average runtime: %13v\n",
name, st.count, float64(st.blocks)/float64(st.count), float64(st.logs)/float64(st.count), st.runtime/time.Duration(st.count))
}
// writeQueries serializes the generated errors to the error file.
func writeErrors(errorFile string, errors []*filterQuery) {
file, err := os.Create(errorFile)
if err != nil {
exit(fmt.Errorf("Error creating filter error file %s: %v", errorFile, err))
return
}
defer file.Close()
json.NewEncoder(file).Encode(errors)
}

97
cmd/workload/historytest.go
View file

@ -23,7 +23,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/internal/utesting"
)
@ -65,40 +64,16 @@ func (s *historyTestSuite) loadTests() error {
return nil
}
func (s *historyTestSuite) allTests() []utesting.Test {
return []utesting.Test{
{
Name: "History/getBlockByHash",
Fn: s.testGetBlockByHash,
},
{
Name: "History/getBlockByNumber",
Fn: s.testGetBlockByNumber,
},
{
Name: "History/getBlockReceiptsByHash",
Fn: s.testGetBlockReceiptsByHash,
},
{
Name: "History/getBlockReceiptsByNumber",
Fn: s.testGetBlockReceiptsByNumber,
},
{
Name: "History/getBlockTransactionCountByHash",
Fn: s.testGetBlockTransactionCountByHash,
},
{
Name: "History/getBlockTransactionCountByNumber",
Fn: s.testGetBlockTransactionCountByNumber,
},
{
Name: "History/getTransactionByBlockHashAndIndex",
Fn: s.testGetTransactionByBlockHashAndIndex,
},
{
Name: "History/getTransactionByBlockNumberAndIndex",
Fn: s.testGetTransactionByBlockNumberAndIndex,
},
func (s *historyTestSuite) allTests() []workloadTest {
return []workloadTest{
newWorkLoadTest("History/getBlockByHash", s.testGetBlockByHash),
newWorkLoadTest("History/getBlockByNumber", s.testGetBlockByNumber),
newWorkLoadTest("History/getBlockReceiptsByHash", s.testGetBlockReceiptsByHash),
newWorkLoadTest("History/getBlockReceiptsByNumber", s.testGetBlockReceiptsByNumber),
newWorkLoadTest("History/getBlockTransactionCountByHash", s.testGetBlockTransactionCountByHash),
newWorkLoadTest("History/getBlockTransactionCountByNumber", s.testGetBlockTransactionCountByNumber),
newWorkLoadTest("History/getTransactionByBlockHashAndIndex", s.testGetTransactionByBlockHashAndIndex),
newWorkLoadTest("History/getTransactionByBlockNumberAndIndex", s.testGetTransactionByBlockNumberAndIndex),
}
}
@ -279,55 +254,3 @@ func (s *historyTestSuite) testGetTransactionByBlockNumberAndIndex(t *utesting.T
}
}
}
type simpleBlock struct {
Number hexutil.Uint64 `json:"number"`
Hash common.Hash `json:"hash"`
}
type simpleTransaction struct {
Hash common.Hash `json:"hash"`
TransactionIndex hexutil.Uint64 `json:"transactionIndex"`
}
func (c *client) getBlockByHash(ctx context.Context, arg common.Hash, fullTx bool) (*simpleBlock, error) {
var r *simpleBlock
err := c.RPC.CallContext(ctx, &r, "eth_getBlockByHash", arg, fullTx)
return r, err
}
func (c *client) getBlockByNumber(ctx context.Context, arg uint64, fullTx bool) (*simpleBlock, error) {
var r *simpleBlock
err := c.RPC.CallContext(ctx, &r, "eth_getBlockByNumber", hexutil.Uint64(arg), fullTx)
return r, err
}
func (c *client) getTransactionByBlockHashAndIndex(ctx context.Context, block common.Hash, index uint64) (*simpleTransaction, error) {
var r *simpleTransaction
err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockHashAndIndex", block, hexutil.Uint64(index))
return r, err
}
func (c *client) getTransactionByBlockNumberAndIndex(ctx context.Context, block uint64, index uint64) (*simpleTransaction, error) {
var r *simpleTransaction
err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockNumberAndIndex", hexutil.Uint64(block), hexutil.Uint64(index))
return r, err
}
func (c *client) getBlockTransactionCountByHash(ctx context.Context, block common.Hash) (uint64, error) {
var r hexutil.Uint64
err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByHash", block)
return uint64(r), err
}
func (c *client) getBlockTransactionCountByNumber(ctx context.Context, block uint64) (uint64, error) {
var r hexutil.Uint64
err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByNumber", hexutil.Uint64(block))
return uint64(r), err
}
func (c *client) getBlockReceipts(ctx context.Context, arg any) ([]*types.Receipt, error) {
var result []*types.Receipt
err := c.RPC.CallContext(ctx, &result, "eth_getBlockReceipts", arg)
return result, err
}

4
cmd/workload/historytestgen.go
View file

@ -51,7 +51,7 @@ var (
}
historyTestEarliestFlag = &cli.IntFlag{
Name: "earliest",
Usage: "JSON file containing filter test queries",
Usage: "The earliest block to test queries",
Value: 0,
Category: flags.TestingCategory,
}
@ -139,7 +139,7 @@ func calcReceiptsHash(rcpt []*types.Receipt) common.Hash {
func writeJSON(fileName string, value any) {
file, err := os.Create(fileName)
if err != nil {
exit(fmt.Errorf("Error creating %s: %v", fileName, err))
exit(fmt.Errorf("error creating %s: %v", fileName, err))
return
}
defer file.Close()

23
cmd/workload/main.go
View file

@ -20,10 +20,8 @@ import (
"fmt"
"os"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/internal/debug"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/rpc"
"github.com/urfave/cli/v2"
)
@ -49,6 +47,7 @@ func init() {
runTestCommand,
historyGenerateCommand,
filterGenerateCommand,
traceGenerateCommand,
filterPerfCommand,
}
}
@ -57,26 +56,6 @@ func main() {
exit(app.Run(os.Args))
}
type client struct {
Eth *ethclient.Client
RPC *rpc.Client
}
func makeClient(ctx *cli.Context) *client {
if ctx.NArg() < 1 {
exit("missing RPC endpoint URL as command-line argument")
}
url := ctx.Args().First()
cl, err := rpc.Dial(url)
if err != nil {
exit(fmt.Errorf("Could not create RPC client at %s: %v", url, err))
}
return &client{
RPC: cl,
Eth: ethclient.NewClient(cl),
}
}
func exit(err any) {
if err == nil {
os.Exit(0)

1
cmd/workload/queries/trace_mainnet.json Normal file
View file

File diff suppressed because one or more lines are too long

1
cmd/workload/queries/trace_sepolia.json Normal file
View file

File diff suppressed because one or more lines are too long

84
cmd/workload/testsuite.go
View file

@ -21,9 +21,8 @@ import (
"fmt"
"io/fs"
"os"
"slices"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/core/history"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/internal/utesting"
"github.com/ethereum/go-ethereum/log"
@ -45,10 +44,13 @@ var (
testPatternFlag,
testTAPFlag,
testSlowFlag,
testArchiveFlag,
testSepoliaFlag,
testMainnetFlag,
filterQueryFileFlag,
historyTestFileFlag,
traceTestFileFlag,
traceTestInvalidOutputFlag,
},
}
testPatternFlag = &cli.StringFlag{
@ -67,6 +69,12 @@ var (
Value: false,
Category: flags.TestingCategory,
}
testArchiveFlag = &cli.BoolFlag{
Name: "archive",
Usage: "Enable archive tests",
Value: false,
Category: flags.TestingCategory,
}
testSepoliaFlag = &cli.BoolFlag{
Name: "sepolia",
Usage: "Use test cases for sepolia network",
@ -86,6 +94,7 @@ type testConfig struct {
filterQueryFile string
historyTestFile string
historyPruneBlock *uint64
traceTestFile string
}
var errPrunedHistory = fmt.Errorf("attempt to access pruned history")
@ -124,37 +133,86 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) {
cfg.filterQueryFile = "queries/filter_queries_mainnet.json"
cfg.historyTestFile = "queries/history_mainnet.json"
cfg.historyPruneBlock = new(uint64)
*cfg.historyPruneBlock = ethconfig.HistoryPrunePoints[params.MainnetGenesisHash].BlockNumber
*cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber
cfg.traceTestFile = "queries/trace_mainnet.json"
case ctx.Bool(testSepoliaFlag.Name):
cfg.fsys = builtinTestFiles
cfg.filterQueryFile = "queries/filter_queries_sepolia.json"
cfg.historyTestFile = "queries/history_sepolia.json"
cfg.historyPruneBlock = new(uint64)
*cfg.historyPruneBlock = ethconfig.HistoryPrunePoints[params.SepoliaGenesisHash].BlockNumber
*cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber
cfg.traceTestFile = "queries/trace_sepolia.json"
default:
cfg.fsys = os.DirFS(".")
cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name)
cfg.historyTestFile = ctx.String(historyTestFileFlag.Name)
cfg.traceTestFile = ctx.String(traceTestFileFlag.Name)
}
return cfg
}
// workloadTest represents a single test in the workload. It's a wrapper
// of utesting.Test by adding a few additional attributes.
type workloadTest struct {
utesting.Test
archive bool // Flag whether the archive node (full state history) is required for this test
}
func newWorkLoadTest(name string, fn func(t *utesting.T)) workloadTest {
return workloadTest{
Test: utesting.Test{
Name: name,
Fn: fn,
},
}
}
func newSlowWorkloadTest(name string, fn func(t *utesting.T)) workloadTest {
t := newWorkLoadTest(name, fn)
t.Slow = true
return t
}
func newArchiveWorkloadTest(name string, fn func(t *utesting.T)) workloadTest {
t := newWorkLoadTest(name, fn)
t.archive = true
return t
}
func filterTests(tests []workloadTest, pattern string, filterFn func(t workloadTest) bool) []utesting.Test {
var utests []utesting.Test
for _, t := range tests {
if filterFn(t) {
utests = append(utests, t.Test)
}
}
if pattern == "" {
return utests
}
return utesting.MatchTests(utests, pattern)
}
func runTestCmd(ctx *cli.Context) error {
cfg := testConfigFromCLI(ctx)
filterSuite := newFilterTestSuite(cfg)
historySuite := newHistoryTestSuite(cfg)
traceSuite := newTraceTestSuite(cfg, ctx)
// Filter test cases.
tests := filterSuite.allTests()
tests = append(tests, historySuite.allTests()...)
if ctx.IsSet(testPatternFlag.Name) {
tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
}
if !ctx.Bool(testSlowFlag.Name) {
tests = slices.DeleteFunc(tests, func(test utesting.Test) bool {
return test.Slow
})
}
tests = append(tests, traceSuite.allTests()...)
utests := filterTests(tests, ctx.String(testPatternFlag.Name), func(t workloadTest) bool {
if t.Slow && !ctx.Bool(testSlowFlag.Name) {
return false
}
if t.archive && !ctx.Bool(testArchiveFlag.Name) {
return false
}
return true
})
// Disable logging unless explicitly enabled.
if !ctx.IsSet("verbosity") && !ctx.IsSet("vmodule") {
@ -166,7 +224,7 @@ func runTestCmd(ctx *cli.Context) error {
if ctx.Bool(testTAPFlag.Name) {
run = utesting.RunTAP
}
results := run(tests, os.Stdout)
results := run(utests, os.Stdout)
if utesting.CountFailures(results) > 0 {
os.Exit(1)
}

126
cmd/workload/tracetest.go Normal file
View file

@ -0,0 +1,126 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/eth/tracers"
"github.com/ethereum/go-ethereum/internal/utesting"
"github.com/ethereum/go-ethereum/log"
"github.com/urfave/cli/v2"
)
// traceTest is the content of a history test.
type traceTest struct {
TxHashes []common.Hash `json:"txHashes"`
TraceConfigs []tracers.TraceConfig `json:"traceConfigs"`
ResultHashes []common.Hash `json:"resultHashes"`
}
type traceTestSuite struct {
cfg testConfig
tests traceTest
invalidDir string
}
func newTraceTestSuite(cfg testConfig, ctx *cli.Context) *traceTestSuite {
s := &traceTestSuite{
cfg: cfg,
invalidDir: ctx.String(traceTestInvalidOutputFlag.Name),
}
if err := s.loadTests(); err != nil {
exit(err)
}
return s
}
func (s *traceTestSuite) loadTests() error {
file, err := s.cfg.fsys.Open(s.cfg.traceTestFile)
if err != nil {
return fmt.Errorf("can't open traceTestFile: %v", err)
}
defer file.Close()
if err := json.NewDecoder(file).Decode(&s.tests); err != nil {
return fmt.Errorf("invalid JSON in %s: %v", s.cfg.traceTestFile, err)
}
if len(s.tests.TxHashes) == 0 {
return fmt.Errorf("traceTestFile %s has no test data", s.cfg.traceTestFile)
}
return nil
}
func (s *traceTestSuite) allTests() []workloadTest {
return []workloadTest{
newArchiveWorkloadTest("Trace/Transaction", s.traceTransaction),
}
}
// traceTransaction runs all transaction tracing tests
func (s *traceTestSuite) traceTransaction(t *utesting.T) {
ctx := context.Background()
for i, hash := range s.tests.TxHashes {
config := s.tests.TraceConfigs[i]
result, err := s.cfg.client.Geth.TraceTransaction(ctx, hash, &config)
if err != nil {
t.Fatalf("Transaction %d (hash %v): error %v", i, hash, err)
}
blob, err := json.Marshal(result)
if err != nil {
t.Fatalf("Transaction %d (hash %v): error %v", i, hash, err)
continue
}
if crypto.Keccak256Hash(blob) != s.tests.ResultHashes[i] {
t.Errorf("Transaction %d (hash %v): invalid result", i, hash)
writeInvalidTraceResult(s.invalidDir, hash, result)
}
}
}
func writeInvalidTraceResult(dir string, hash common.Hash, result any) {
if dir == "" {
return
}
err := os.MkdirAll(dir, os.ModePerm)
if err != nil {
log.Info("Failed to make output directory", "err", err)
return
}
name := filepath.Join(dir, "invalid"+"_"+hash.String())
file, err := os.Create(name)
if err != nil {
exit(fmt.Errorf("error creating %s: %v", name, err))
return
}
defer file.Close()
data, _ := json.MarshalIndent(result, "", " ")
_, err = file.Write(data)
if err != nil {
exit(fmt.Errorf("error writing %s: %v", name, err))
return
}
}

195
cmd/workload/tracetestgen.go Normal file
View file

@ -0,0 +1,195 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>
package main
import (
"context"
"encoding/json"
"fmt"
"math/big"
"math/rand"
"os"
"path/filepath"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/eth/tracers"
"github.com/ethereum/go-ethereum/eth/tracers/logger"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/log"
"github.com/urfave/cli/v2"
)
var (
defaultBlocksToTrace = 64 // the number of states assumed to be available
traceGenerateCommand = &cli.Command{
Name: "tracegen",
Usage: "Generates tests for state tracing",
ArgsUsage: "<RPC endpoint URL>",
Action: generateTraceTests,
Flags: []cli.Flag{
traceTestFileFlag,
traceTestResultOutputFlag,
traceTestBlockFlag,
},
}
traceTestFileFlag = &cli.StringFlag{
Name: "trace-tests",
Usage: "JSON file containing trace test queries",
Value: "trace_tests.json",
Category: flags.TestingCategory,
}
traceTestResultOutputFlag = &cli.StringFlag{
Name: "trace-output",
Usage: "Folder containing the trace output files",
Value: "",
Category: flags.TestingCategory,
}
traceTestBlockFlag = &cli.IntFlag{
Name: "trace-blocks",
Usage: "The number of blocks for tracing",
Value: defaultBlocksToTrace,
Category: flags.TestingCategory,
}
traceTestInvalidOutputFlag = &cli.StringFlag{
Name: "trace-invalid",
Usage: "Folder containing the mismatched trace output files",
Value: "",
Category: flags.TestingCategory,
}
)
func generateTraceTests(clictx *cli.Context) error {
var (
client = makeClient(clictx)
outputFile = clictx.String(traceTestFileFlag.Name)
outputDir = clictx.String(traceTestResultOutputFlag.Name)
blocks = clictx.Int(traceTestBlockFlag.Name)
ctx = context.Background()
test = new(traceTest)
)
if outputDir != "" {
err := os.MkdirAll(outputDir, os.ModePerm)
if err != nil {
return err
}
}
latest, err := client.Eth.BlockNumber(ctx)
if err != nil {
exit(err)
}
if latest < uint64(blocks) {
exit(fmt.Errorf("node seems not synced, latest block is %d", latest))
}
// Get blocks and assign block info into the test
var (
start = time.Now()
logged = time.Now()
failed int
)
log.Info("Trace transactions around the chain tip", "head", latest, "blocks", blocks)
for i := 0; i < blocks; i++ {
number := latest - uint64(i)
block, err := client.Eth.BlockByNumber(ctx, big.NewInt(int64(number)))
if err != nil {
exit(err)
}
for _, tx := range block.Transactions() {
config, configName := randomTraceOption()
result, err := client.Geth.TraceTransaction(ctx, tx.Hash(), config)
if err != nil {
failed += 1
break
}
blob, err := json.Marshal(result)
if err != nil {
failed += 1
break
}
test.TxHashes = append(test.TxHashes, tx.Hash())
test.TraceConfigs = append(test.TraceConfigs, *config)
test.ResultHashes = append(test.ResultHashes, crypto.Keccak256Hash(blob))
writeTraceResult(outputDir, tx.Hash(), result, configName)
}
if time.Since(logged) > time.Second*8 {
logged = time.Now()
log.Info("Tracing transactions", "executed", len(test.TxHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start)))
}
}
log.Info("Traced transactions", "executed", len(test.TxHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start)))
// Write output file.
writeJSON(outputFile, test)
return nil
}
func randomTraceOption() (*tracers.TraceConfig, string) {
x := rand.Intn(11)
if x == 0 {
// struct-logger, with all fields enabled, very heavy
return &tracers.TraceConfig{
Config: &logger.Config{
EnableMemory: true,
EnableReturnData: true,
},
}, "structAll"
}
if x == 1 {
// default options for struct-logger, with stack and storage capture
// enabled
return &tracers.TraceConfig{
Config: &logger.Config{},
}, "structDefault"
}
if x == 2 || x == 3 || x == 4 {
// struct-logger with storage capture enabled
return &tracers.TraceConfig{
Config: &logger.Config{
DisableStack: true,
},
}, "structStorage"
}
// Native tracer
loggers := []string{"callTracer", "4byteTracer", "flatCallTracer", "muxTracer", "noopTracer", "prestateTracer"}
return &tracers.TraceConfig{
Tracer: &loggers[x-5],
}, loggers[x-5]
}
func writeTraceResult(dir string, hash common.Hash, result any, configName string) {
if dir == "" {
return
}
name := filepath.Join(dir, configName+"_"+hash.String())
file, err := os.Create(name)
if err != nil {
exit(fmt.Errorf("error creating %s: %v", name, err))
return
}
defer file.Close()
data, _ := json.MarshalIndent(result, "", " ")
_, err = file.Write(data)
if err != nil {
exit(fmt.Errorf("error writing %s: %v", name, err))
return
}
}

17
common/lru/basiclru.go
View file

@ -47,30 +47,37 @@ func NewBasicLRU[K comparable, V any](capacity int) BasicLRU[K, V] {
// Add adds a value to the cache. Returns true if an item was evicted to store the new item.
func (c *BasicLRU[K, V]) Add(key K, value V) (evicted bool) {
_, _, evicted = c.Add3(key, value)
return evicted
}
// Add3 adds a value to the cache. If an item was evicted to store the new one, it returns the evicted item.
func (c *BasicLRU[K, V]) Add3(key K, value V) (ek K, ev V, evicted bool) {
item, ok := c.items[key]
if ok {
// Already exists in cache.
item.value = value
c.items[key] = item
c.list.moveToFront(item.elem)
return false
return ek, ev, false
}
var elem *listElem[K]
if c.Len() >= c.cap {
elem = c.list.removeLast()
delete(c.items, elem.v)
evicted = true
ek = elem.v
ev = c.items[ek].value
delete(c.items, ek)
} else {
elem = new(listElem[K])
}
// Store the new item.
// Note that, if another item was evicted, we re-use its list element here.
// Note that if another item was evicted, we re-use its list element here.
elem.v = key
c.items[key] = cacheItem[K, V]{elem, value}
c.list.pushElem(elem)
return evicted
return ek, ev, evicted
}
// Contains reports whether the given key exists in the cache.

2
common/math/big.go
View file

@ -72,7 +72,7 @@ func (i *HexOrDecimal256) MarshalText() ([]byte, error) {
if i == nil {
return []byte("0x0"), nil
}
return []byte(fmt.Sprintf("%#x", (*big.Int)(i))), nil
return fmt.Appendf(nil, "%#x", (*big.Int)(i)), nil
}
// Decimal256 unmarshals big.Int as a decimal string. When unmarshalling,

2
common/math/integer.go
View file

@ -48,7 +48,7 @@ func (i *HexOrDecimal64) UnmarshalText(input []byte) error {
// MarshalText implements encoding.TextMarshaler.
func (i HexOrDecimal64) MarshalText() ([]byte, error) {
return []byte(fmt.Sprintf("%#x", uint64(i))), nil
return fmt.Appendf(nil, "%#x", uint64(i)), nil
}
// ParseUint64 parses s as an integer in decimal or hexadecimal syntax.

6
consensus/beacon/consensus.go
View file

@ -30,7 +30,6 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie"
"github.com/holiman/uint256"
)
@ -445,11 +444,6 @@ func (beacon *Beacon) CalcDifficulty(chain consensus.ChainHeaderReader, time uin
return beaconDifficulty
}
// APIs implements consensus.Engine, returning the user facing RPC APIs.
func (beacon *Beacon) APIs(chain consensus.ChainHeaderReader) []rpc.API {
return beacon.ethone.APIs(chain)
}
// Close shutdowns the consensus engine
func (beacon *Beacon) Close() error {
return beacon.ethone.Close()

235
consensus/clique/api.go
View file

@ -1,235 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package clique
import (
"encoding/json"
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
)
// API is a user facing RPC API to allow controlling the signer and voting
// mechanisms of the proof-of-authority scheme.
type API struct {
chain consensus.ChainHeaderReader
clique *Clique
}
// GetSnapshot retrieves the state snapshot at a given block.
func (api *API) GetSnapshot(number *rpc.BlockNumber) (*Snapshot, error) {
// Retrieve the requested block number (or current if none requested)
var header *types.Header
if number == nil || *number == rpc.LatestBlockNumber {
header = api.chain.CurrentHeader()
} else {
header = api.chain.GetHeaderByNumber(uint64(number.Int64()))
}
// Ensure we have an actually valid block and return its snapshot
if header == nil {
return nil, errUnknownBlock
}
return api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
}
// GetSnapshotAtHash retrieves the state snapshot at a given block.
func (api *API) GetSnapshotAtHash(hash common.Hash) (*Snapshot, error) {
header := api.chain.GetHeaderByHash(hash)
if header == nil {
return nil, errUnknownBlock
}
return api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
}
// GetSigners retrieves the list of authorized signers at the specified block.
func (api *API) GetSigners(number *rpc.BlockNumber) ([]common.Address, error) {
// Retrieve the requested block number (or current if none requested)
var header *types.Header
if number == nil || *number == rpc.LatestBlockNumber {
header = api.chain.CurrentHeader()
} else {
header = api.chain.GetHeaderByNumber(uint64(number.Int64()))
}
// Ensure we have an actually valid block and return the signers from its snapshot
if header == nil {
return nil, errUnknownBlock
}
snap, err := api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
if err != nil {
return nil, err
}
return snap.signers(), nil
}
// GetSignersAtHash retrieves the list of authorized signers at the specified block.
func (api *API) GetSignersAtHash(hash common.Hash) ([]common.Address, error) {
header := api.chain.GetHeaderByHash(hash)
if header == nil {
return nil, errUnknownBlock
}
snap, err := api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
if err != nil {
return nil, err
}
return snap.signers(), nil
}
// Proposals returns the current proposals the node tries to uphold and vote on.
func (api *API) Proposals() map[common.Address]bool {
api.clique.lock.RLock()
defer api.clique.lock.RUnlock()
proposals := make(map[common.Address]bool)
for address, auth := range api.clique.proposals {
proposals[address] = auth
}
return proposals
}
// Propose injects a new authorization proposal that the signer will attempt to
// push through.
func (api *API) Propose(address common.Address, auth bool) {
api.clique.lock.Lock()
defer api.clique.lock.Unlock()
api.clique.proposals[address] = auth
}
// Discard drops a currently running proposal, stopping the signer from casting
// further votes (either for or against).
func (api *API) Discard(address common.Address) {
api.clique.lock.Lock()
defer api.clique.lock.Unlock()
delete(api.clique.proposals, address)
}
type status struct {
InturnPercent float64 `json:"inturnPercent"`
SigningStatus map[common.Address]int `json:"sealerActivity"`
NumBlocks uint64 `json:"numBlocks"`
}
// Status returns the status of the last N blocks,
// - the number of active signers,
// - the number of signers,
// - the percentage of in-turn blocks
func (api *API) Status() (*status, error) {
var (
numBlocks = uint64(64)
header = api.chain.CurrentHeader()
diff = uint64(0)
optimals = 0
)
snap, err := api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
if err != nil {
return nil, err
}
var (
signers = snap.signers()
end = header.Number.Uint64()
start = end - numBlocks
)
if numBlocks > end {
start = 1
numBlocks = end - start
}
signStatus := make(map[common.Address]int)
for _, s := range signers {
signStatus[s] = 0
}
for n := start; n < end; n++ {
h := api.chain.GetHeaderByNumber(n)
if h == nil {
return nil, fmt.Errorf("missing block %d", n)
}
if h.Difficulty.Cmp(diffInTurn) == 0 {
optimals++
}
diff += h.Difficulty.Uint64()
sealer, err := api.clique.Author(h)
if err != nil {
return nil, err
}
signStatus[sealer]++
}
return &status{
InturnPercent: float64(100*optimals) / float64(numBlocks),
SigningStatus: signStatus,
NumBlocks: numBlocks,
}, nil
}
type blockNumberOrHashOrRLP struct {
*rpc.BlockNumberOrHash
RLP hexutil.Bytes `json:"rlp,omitempty"`
}
func (sb *blockNumberOrHashOrRLP) UnmarshalJSON(data []byte) error {
bnOrHash := new(rpc.BlockNumberOrHash)
// Try to unmarshal bNrOrHash
if err := bnOrHash.UnmarshalJSON(data); err == nil {
sb.BlockNumberOrHash = bnOrHash
return nil
}
// Try to unmarshal RLP
var input string
if err := json.Unmarshal(data, &input); err != nil {
return err
}
blob, err := hexutil.Decode(input)
if err != nil {
return err
}
sb.RLP = blob
return nil
}
// GetSigner returns the signer for a specific clique block.
// Can be called with a block number, a block hash or a rlp encoded blob.
// The RLP encoded blob can either be a block or a header.
func (api *API) GetSigner(rlpOrBlockNr *blockNumberOrHashOrRLP) (common.Address, error) {
if len(rlpOrBlockNr.RLP) == 0 {
blockNrOrHash := rlpOrBlockNr.BlockNumberOrHash
var header *types.Header
if blockNrOrHash == nil {
header = api.chain.CurrentHeader()
} else if hash, ok := blockNrOrHash.Hash(); ok {
header = api.chain.GetHeaderByHash(hash)
} else if number, ok := blockNrOrHash.Number(); ok {
header = api.chain.GetHeaderByNumber(uint64(number.Int64()))
}
if header == nil {
return common.Address{}, fmt.Errorf("missing block %v", blockNrOrHash.String())
}
return api.clique.Author(header)
}
block := new(types.Block)
if err := rlp.DecodeBytes(rlpOrBlockNr.RLP, block); err == nil {
return api.clique.Author(block.Header())
}
header := new(types.Header)
if err := rlp.DecodeBytes(rlpOrBlockNr.RLP, header); err != nil {
return common.Address{}, err
}
return api.clique.Author(header)
}

10
consensus/clique/clique.go
View file

@ -41,7 +41,6 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/trie"
"golang.org/x/crypto/sha3"
)
@ -641,15 +640,6 @@ func (c *Clique) Close() error {
return nil
}
// APIs implements consensus.Engine, returning the user facing RPC API to allow
// controlling the signer voting.
func (c *Clique) APIs(chain consensus.ChainHeaderReader) []rpc.API {
return []rpc.API{{
Namespace: "clique",
Service: &API{chain: chain, clique: c},
}}
}
// SealHash returns the hash of a block prior to it being sealed.
func SealHash(header *types.Header) (hash common.Hash) {
hasher := sha3.NewLegacyKeccak256()

4
consensus/consensus.go
View file

@ -25,7 +25,6 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
)
// ChainHeaderReader defines a small collection of methods needed to access the local
@ -109,9 +108,6 @@ type Engine interface {
// that a new block should have.
CalcDifficulty(chain ChainHeaderReader, time uint64, parent *types.Header) *big.Int
// APIs returns the RPC APIs this consensus engine provides.
APIs(chain ChainHeaderReader) []rpc.API
// Close terminates any background threads maintained by the consensus engine.
Close() error
}

7
consensus/ethash/ethash.go
View file

@ -22,7 +22,6 @@ import (
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rpc"
)
// Ethash is a consensus engine based on proof-of-work implementing the ethash
@ -71,12 +70,6 @@ func (ethash *Ethash) Close() error {
return nil
}
// APIs implements consensus.Engine, returning no APIs as ethash is an empty
// shell in the post-merge world.
func (ethash *Ethash) APIs(chain consensus.ChainHeaderReader) []rpc.API {
return []rpc.API{}
}
// Seal generates a new sealing request for the given input block and pushes
// the result into the given channel. For the ethash engine, this method will
// just panic as sealing is not supported anymore.

10
core/block_validator_test.go
View file

@ -50,8 +50,11 @@ func testHeaderVerification(t *testing.T, scheme string) {
headers[i] = block.Header()
}
// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer chain.Stop()
if err != nil {
t.Fatal(err)
}
for i := 0; i < len(blocks); i++ {
for j, valid := range []bool{true, false} {
@ -163,8 +166,11 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) {
postHeaders[i] = block.Header()
}
// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil)
chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil)
defer chain.Stop()
if err != nil {
t.Fatal(err)
}
// Verify the blocks before the merging
for i := 0; i < len(preBlocks); i++ {

648
core/blockchain.go
View file

@ -21,9 +21,11 @@ import (
"errors"
"fmt"
"io"
"math"
"math/big"
"runtime"
"slices"
"sort"
"strings"
"sync"
"sync/atomic"
@ -35,6 +37,7 @@ import (
"github.com/ethereum/go-ethereum/common/prque"
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
"github.com/ethereum/go-ethereum/core/history"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/state/snapshot"
@ -62,7 +65,8 @@ var (
headFinalizedBlockGauge = metrics.NewRegisteredGauge("chain/head/finalized", nil)
headSafeBlockGauge = metrics.NewRegisteredGauge("chain/head/safe", nil)
chainInfoGauge = metrics.NewRegisteredGaugeInfo("chain/info", nil)
chainInfoGauge = metrics.NewRegisteredGaugeInfo("chain/info", nil)
chainMgaspsMeter = metrics.NewRegisteredResettingTimer("chain/mgasps", nil)
accountReadTimer = metrics.NewRegisteredResettingTimer("chain/account/reads", nil)
accountHashTimer = metrics.NewRegisteredResettingTimer("chain/account/hashes", nil)
@ -89,8 +93,10 @@ var (
blockReorgAddMeter = metrics.NewRegisteredMeter("chain/reorg/add", nil)
blockReorgDropMeter = metrics.NewRegisteredMeter("chain/reorg/drop", nil)
blockPrefetchExecuteTimer = metrics.NewRegisteredTimer("chain/prefetch/executes", nil)
blockPrefetchInterruptMeter = metrics.NewRegisteredMeter("chain/prefetch/interrupts", nil)
blockPrefetchExecuteTimer = metrics.NewRegisteredResettingTimer("chain/prefetch/executes", nil)
blockPrefetchInterruptMeter = metrics.NewRegisteredMeter("chain/prefetch/interrupts", nil)
blockPrefetchTxsInvalidMeter = metrics.NewRegisteredMeter("chain/prefetch/txs/invalid", nil)
blockPrefetchTxsValidMeter = metrics.NewRegisteredMeter("chain/prefetch/txs/valid", nil)
errInsertionInterrupted = errors.New("insertion is interrupted")
errChainStopped = errors.New("blockchain is stopped")
@ -98,6 +104,10 @@ var (
errInvalidNewChain = errors.New("invalid new chain")
)
var (
forkReadyInterval = 3 * time.Minute
)
const (
bodyCacheLimit = 256
blockCacheLimit = 256
@ -157,7 +167,7 @@ type CacheConfig struct {
// This defines the cutoff block for history expiry.
// Blocks before this number may be unavailable in the chain database.
HistoryPruningCutoff uint64
ChainHistoryMode history.HistoryMode
}
// triedbConfig derives the configures for trie database.
@ -173,8 +183,13 @@ func (c *CacheConfig) triedbConfig(isVerkle bool) *triedb.Config {
}
if c.StateScheme == rawdb.PathScheme {
config.PathDB = &pathdb.Config{
StateHistory: c.StateHistory,
CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
StateHistory: c.StateHistory,
TrieCleanSize: c.TrieCleanLimit * 1024 * 1024,
StateCleanSize: c.SnapshotLimit * 1024 * 1024,
// TODO(rjl493456442): The write buffer represents the memory limit used
// for flushing both trie data and state data to disk. The config name
// should be updated to eliminate the confusion.
WriteBufferSize: c.TrieDirtyLimit * 1024 * 1024,
}
}
@ -253,6 +268,7 @@ type BlockChain struct {
currentSnapBlock atomic.Pointer[types.Header] // Current head of snap-sync
currentFinalBlock atomic.Pointer[types.Header] // Latest (consensus) finalized block
currentSafeBlock atomic.Pointer[types.Header] // Latest (consensus) safe block
historyPrunePoint atomic.Pointer[history.PrunePoint]
bodyCache *lru.Cache[common.Hash, *types.Body]
bodyRLPCache *lru.Cache[common.Hash, rlp.RawValue]
@ -262,7 +278,6 @@ type BlockChain struct {
txLookupLock sync.RWMutex
txLookupCache *lru.Cache[common.Hash, txLookup]
wg sync.WaitGroup
quit chan struct{} // shutdown signal, closed in Stop.
stopping atomic.Bool // false if chain is running, true when stopped
procInterrupt atomic.Bool // interrupt signaler for block processing
@ -273,6 +288,8 @@ type BlockChain struct {
processor Processor // Block transaction processor interface
vmConfig vm.Config
logger *tracing.Hooks
lastForkReadyAlert time.Time // Last time there was a fork readiness print out
}
// NewBlockChain returns a fully initialised block chain using information
@ -332,10 +349,11 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
bc.prefetcher = newStatePrefetcher(chainConfig, bc.hc)
bc.processor = NewStateProcessor(chainConfig, bc.hc)
bc.genesisBlock = bc.GetBlockByNumber(0)
if bc.genesisBlock == nil {
genesisHeader := bc.GetHeaderByNumber(0)
if genesisHeader == nil {
return nil, ErrNoGenesis
}
bc.genesisBlock = types.NewBlockWithHeader(genesisHeader)
bc.currentBlock.Store(nil)
bc.currentSnapBlock.Store(nil)
@ -367,11 +385,14 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
// Do nothing here until the state syncer picks it up.
log.Info("Genesis state is missing, wait state sync")
} else {
// Head state is missing, before the state recovery, find out the
// disk layer point of snapshot(if it's enabled). Make sure the
// rewound point is lower than disk layer.
// Head state is missing, before the state recovery, find out the disk
// layer point of snapshot(if it's enabled). Make sure the rewound point
// is lower than disk layer.
//
// Note it's unnecessary in path mode which always keep trie data and
// state data consistent.
var diskRoot common.Hash
if bc.cacheConfig.SnapshotLimit > 0 {
if bc.cacheConfig.SnapshotLimit > 0 && bc.cacheConfig.StateScheme == rawdb.HashScheme {
diskRoot = rawdb.ReadSnapshotRoot(bc.db)
}
if diskRoot != (common.Hash{}) {
@ -444,7 +465,32 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
bc.logger.OnGenesisBlock(bc.genesisBlock, alloc)
}
}
bc.setupSnapshot()
// Rewind the chain in case of an incompatible config upgrade.
if compatErr != nil {
log.Warn("Rewinding chain to upgrade configuration", "err", compatErr)
if compatErr.RewindToTime > 0 {
bc.SetHeadWithTimestamp(compatErr.RewindToTime)
} else {
bc.SetHead(compatErr.RewindToBlock)
}
rawdb.WriteChainConfig(db, genesisHash, chainConfig)
}
// Start tx indexer if it's enabled.
if txLookupLimit != nil {
bc.txIndexer = newTxIndexer(*txLookupLimit, bc)
}
return bc, nil
}
func (bc *BlockChain) setupSnapshot() {
// Short circuit if the chain is established with path scheme, as the
// state snapshot has been integrated into path database natively.
if bc.cacheConfig.StateScheme == rawdb.PathScheme {
return
}
// Load any existing snapshot, regenerating it if loading failed
if bc.cacheConfig.SnapshotLimit > 0 {
// If the chain was rewound past the snapshot persistent layer (causing
@ -452,7 +498,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
// in recovery mode and in that case, don't invalidate the snapshot on a
// head mismatch.
var recover bool
head := bc.CurrentBlock()
if layer := rawdb.ReadSnapshotRecoveryNumber(bc.db); layer != nil && *layer >= head.Number.Uint64() {
log.Warn("Enabling snapshot recovery", "chainhead", head.Number, "diskbase", *layer)
@ -469,22 +514,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
// Re-initialize the state database with snapshot
bc.statedb = state.NewDatabase(bc.triedb, bc.snaps)
}
// Rewind the chain in case of an incompatible config upgrade.
if compatErr != nil {
log.Warn("Rewinding chain to upgrade configuration", "err", compatErr)
if compatErr.RewindToTime > 0 {
bc.SetHeadWithTimestamp(compatErr.RewindToTime)
} else {
bc.SetHead(compatErr.RewindToBlock)
}
rawdb.WriteChainConfig(db, genesisHash, chainConfig)
}
// Start tx indexer if it's enabled.
if txLookupLimit != nil {
bc.txIndexer = newTxIndexer(*txLookupLimit, bc)
}
return bc, nil
}
// empty returns an indicator whether the blockchain is empty.
@ -511,19 +540,33 @@ func (bc *BlockChain) loadLastState() error {
log.Warn("Empty database, resetting chain")
return bc.Reset()
}
// Make sure the entire head block is available
headBlock := bc.GetBlockByHash(head)
headHeader := bc.GetHeaderByHash(head)
if headHeader == nil {
// Corrupt or empty database, init from scratch
log.Warn("Head header missing, resetting chain", "hash", head)
return bc.Reset()
}
var headBlock *types.Block
if cmp := headHeader.Number.Cmp(new(big.Int)); cmp == 1 {
// Make sure the entire head block is available.
headBlock = bc.GetBlockByHash(head)
} else if cmp == 0 {
// On a pruned node the block body might not be available. But a pruned
// block should never be the head block. The only exception is when, as
// a last resort, chain is reset to genesis.
headBlock = bc.genesisBlock
}
if headBlock == nil {
// Corrupt or empty database, init from scratch
log.Warn("Head block missing, resetting chain", "hash", head)
return bc.Reset()
}
// Everything seems to be fine, set as the head block
bc.currentBlock.Store(headBlock.Header())
bc.currentBlock.Store(headHeader)
headBlockGauge.Update(int64(headBlock.NumberU64()))
// Restore the last known head header
headHeader := headBlock.Header()
if head := rawdb.ReadHeadHeaderHash(bc.db); head != (common.Hash{}) {
if header := bc.GetHeaderByHash(head); header != nil {
headHeader = header
@ -531,6 +574,12 @@ func (bc *BlockChain) loadLastState() error {
}
bc.hc.SetCurrentHeader(headHeader)
// Initialize history pruning.
latest := max(headBlock.NumberU64(), headHeader.Number.Uint64())
if err := bc.initializeHistoryPruning(latest); err != nil {
return err
}
// Restore the last known head snap block
bc.currentSnapBlock.Store(headBlock.Header())
headFastBlockGauge.Update(int64(headBlock.NumberU64()))
@ -553,6 +602,7 @@ func (bc *BlockChain) loadLastState() error {
headSafeBlockGauge.Update(int64(block.NumberU64()))
}
}
// Issue a status log for the user
var (
currentSnapBlock = bc.CurrentSnapBlock()
@ -571,9 +621,57 @@ func (bc *BlockChain) loadLastState() error {
if pivot := rawdb.ReadLastPivotNumber(bc.db); pivot != nil {
log.Info("Loaded last snap-sync pivot marker", "number", *pivot)
}
if pruning := bc.historyPrunePoint.Load(); pruning != nil {
log.Info("Chain history is pruned", "earliest", pruning.BlockNumber, "hash", pruning.BlockHash)
}
return nil
}
// initializeHistoryPruning sets bc.historyPrunePoint.
func (bc *BlockChain) initializeHistoryPruning(latest uint64) error {
freezerTail, _ := bc.db.Tail()
switch bc.cacheConfig.ChainHistoryMode {
case history.KeepAll:
if freezerTail == 0 {
return nil
}
// The database was pruned somehow, so we need to figure out if it's a known
// configuration or an error.
predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()]
if predefinedPoint == nil || freezerTail != predefinedPoint.BlockNumber {
log.Error("Chain history database is pruned with unknown configuration", "tail", freezerTail)
return fmt.Errorf("unexpected database tail")
}
bc.historyPrunePoint.Store(predefinedPoint)
return nil
case history.KeepPostMerge:
if freezerTail == 0 && latest != 0 {
// This is the case where a user is trying to run with --history.chain
// postmerge directly on an existing DB. We could just trigger the pruning
// here, but it'd be a bit dangerous since they may not have intended this
// action to happen. So just tell them how to do it.
log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cacheConfig.ChainHistoryMode.String()))
log.Error(fmt.Sprintf("Run 'geth prune-history' to prune pre-merge history."))
return fmt.Errorf("history pruning requested via configuration")
}
predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()]
if predefinedPoint == nil {
log.Error("Chain history pruning is not supported for this network", "genesis", bc.genesisBlock.Hash())
return fmt.Errorf("history pruning requested for unknown network")
} else if freezerTail > 0 && freezerTail != predefinedPoint.BlockNumber {
log.Error("Chain history database is pruned to unknown block", "tail", freezerTail)
return fmt.Errorf("unexpected database tail")
}
bc.historyPrunePoint.Store(predefinedPoint)
return nil
default:
return fmt.Errorf("invalid history mode: %d", bc.cacheConfig.ChainHistoryMode)
}
}
// SetHead rewinds the local chain to a new head. Depending on whether the node
// was snap synced or full synced and in which state, the method will try to
// delete minimal data from disk whilst retaining chain consistency.
@ -584,11 +682,15 @@ func (bc *BlockChain) SetHead(head uint64) error {
// Send chain head event to update the transaction pool
header := bc.CurrentBlock()
if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil {
// This should never happen. In practice, previously currentBlock
// contained the entire block whereas now only a "marker", so there
// is an ever so slight chance for a race we should handle.
log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash())
return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4])
// In a pruned node the genesis block will not exist in the freezer.
// It should not happen that we set head to any other pruned block.
if header.Number.Uint64() > 0 {
// This should never happen. In practice, previously currentBlock
// contained the entire block whereas now only a "marker", so there
// is an ever so slight chance for a race we should handle.
log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash())
return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4])
}
}
bc.chainHeadFeed.Send(ChainHeadEvent{Header: header})
return nil
@ -605,11 +707,15 @@ func (bc *BlockChain) SetHeadWithTimestamp(timestamp uint64) error {
// Send chain head event to update the transaction pool
header := bc.CurrentBlock()
if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil {
// This should never happen. In practice, previously currentBlock
// contained the entire block whereas now only a "marker", so there
// is an ever so slight chance for a race we should handle.
log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash())
return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4])
// In a pruned node the genesis block will not exist in the freezer.
// It should not happen that we set head to any other pruned block.
if header.Number.Uint64() > 0 {
// This should never happen. In practice, previously currentBlock
// contained the entire block whereas now only a "marker", so there
// is an ever so slight chance for a race we should handle.
log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash())
return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4])
}
}
bc.chainHeadFeed.Send(ChainHeadEvent{Header: header})
return nil
@ -892,21 +998,20 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
// Ignore the error here since light client won't hit this path
frozen, _ := bc.db.Ancients()
if num+1 <= frozen {
// Truncate all relative data(header, total difficulty, body, receipt
// and canonical hash) from ancient store.
if _, err := bc.db.TruncateHead(num); err != nil {
log.Crit("Failed to truncate ancient data", "number", num, "err", err)
}
// Remove the hash <-> number mapping from the active store.
rawdb.DeleteHeaderNumber(db, hash)
// The chain segment, such as the block header, canonical hash,
// body, and receipt, will be removed from the ancient store
// in one go.
//
// The hash-to-number mapping in the key-value store will be
// removed by the hc.SetHead function.
} else {
// Remove relative body and receipts from the active store.
// The header, total difficulty and canonical hash will be
// removed in the hc.SetHead function.
// Remove the associated body and receipts from the key-value store.
// The header, hash-to-number mapping, and canonical hash will be
// removed by the hc.SetHead function.
rawdb.DeleteBody(db, hash, num)
rawdb.DeleteReceipts(db, hash, num)
}
// Todo(rjl493456442) txlookup, bloombits, etc
// Todo(rjl493456442) txlookup, log index, etc
}
// If SetHead was only called as a chain reparation method, try to skip
// touching the header chain altogether, unless the freezer is broken
@ -1012,7 +1117,9 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
bc.hc.SetCurrentHeader(bc.genesisBlock.Header())
bc.currentSnapBlock.Store(bc.genesisBlock.Header())
headFastBlockGauge.Update(int64(bc.genesisBlock.NumberU64()))
return nil
// Reset history pruning status.
return bc.initializeHistoryPruning(0)
}
// Export writes the active chain to the given writer.
@ -1109,7 +1216,6 @@ func (bc *BlockChain) stopWithoutSaving() {
// the mutex should become available quickly. It cannot be taken again after Close has
// returned.
bc.chainmu.Close()
bc.wg.Wait()
}
// Stop stops the blockchain service. If any imports are currently in progress
@ -1196,81 +1302,65 @@ const (
SideStatTy
)
// InsertReceiptChain attempts to complete an already existing header chain with
// transaction and receipt data.
func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts, ancientLimit uint64) (int, error) {
// We don't require the chainMu here since we want to maximize the
// concurrency of header insertion and receipt insertion.
bc.wg.Add(1)
defer bc.wg.Done()
var (
ancientBlocks, liveBlocks types.Blocks
ancientReceipts, liveReceipts []types.Receipts
)
// Do a sanity check that the provided chain is actually ordered and linked
for i, block := range blockChain {
if i != 0 {
prev := blockChain[i-1]
if block.NumberU64() != prev.NumberU64()+1 || block.ParentHash() != prev.Hash() {
log.Error("Non contiguous receipt insert",
"number", block.Number(), "hash", block.Hash(), "parent", block.ParentHash(),
"prevnumber", prev.Number(), "prevhash", prev.Hash())
return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])",
i-1, prev.NumberU64(), prev.Hash().Bytes()[:4],
i, block.NumberU64(), block.Hash().Bytes()[:4], block.ParentHash().Bytes()[:4])
}
}
if block.NumberU64() <= ancientLimit {
ancientBlocks, ancientReceipts = append(ancientBlocks, block), append(ancientReceipts, receiptChain[i])
} else {
liveBlocks, liveReceipts = append(liveBlocks, block), append(liveReceipts, receiptChain[i])
}
// Here we also validate that blob transactions in the block do not contain a sidecar.
// While the sidecar does not affect the block hash / tx hash, sending blobs within a block is not allowed.
// InsertReceiptChain inserts a batch of blocks along with their receipts into
// the database. Unlike InsertChain, this function does not verify the state root
// in the blocks. It is used exclusively for snap sync. All the inserted blocks
// will be regarded as canonical, chain reorg is not supported.
//
// The optional ancientLimit can also be specified and chain segment before that
// will be directly stored in the ancient, getting rid of the chain migration.
func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []rlp.RawValue, ancientLimit uint64) (int, error) {
// Verify the supplied headers before insertion without lock
var headers []*types.Header
for _, block := range blockChain {
headers = append(headers, block.Header())
// Here we also validate that blob transactions in the block do not
// contain a sidecar. While the sidecar does not affect the block hash
// or tx hash, sending blobs within a block is not allowed.
for txIndex, tx := range block.Transactions() {
if tx.Type() == types.BlobTxType && tx.BlobTxSidecar() != nil {
return 0, fmt.Errorf("block #%d contains unexpected blob sidecar in tx at index %d", block.NumberU64(), txIndex)
}
}
}
if n, err := bc.hc.ValidateHeaderChain(headers); err != nil {
return n, err
}
// Hold the mutation lock
if !bc.chainmu.TryLock() {
return 0, errChainStopped
}
defer bc.chainmu.Unlock()
var (
stats = struct{ processed, ignored int32 }{}
start = time.Now()
size = int64(0)
)
// updateHead updates the head snap sync block if the inserted blocks are better
// and returns an indicator whether the inserted blocks are canonical.
updateHead := func(head *types.Block) bool {
if !bc.chainmu.TryLock() {
return false
// updateHead updates the head header and head snap block flags.
updateHead := func(header *types.Header) error {
batch := bc.db.NewBatch()
hash := header.Hash()
rawdb.WriteHeadHeaderHash(batch, hash)
rawdb.WriteHeadFastBlockHash(batch, hash)
if err := batch.Write(); err != nil {
return err
}
defer bc.chainmu.Unlock()
// Rewind may have occurred, skip in that case.
if bc.CurrentHeader().Number.Cmp(head.Number()) >= 0 {
rawdb.WriteHeadFastBlockHash(bc.db, head.Hash())
bc.currentSnapBlock.Store(head.Header())
headFastBlockGauge.Update(int64(head.NumberU64()))
return true
}
return false
bc.hc.currentHeader.Store(header)
bc.currentSnapBlock.Store(header)
headHeaderGauge.Update(header.Number.Int64())
headFastBlockGauge.Update(header.Number.Int64())
return nil
}
// writeAncient writes blockchain and corresponding receipt chain into ancient store.
//
// this function only accepts canonical chain data. All side chain will be reverted
// eventually.
writeAncient := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) {
first := blockChain[0]
last := blockChain[len(blockChain)-1]
// Ensure genesis is in ancients.
if first.NumberU64() == 1 {
writeAncient := func(blockChain types.Blocks, receiptChain []rlp.RawValue) (int, error) {
// Ensure genesis is in the ancient store
if blockChain[0].NumberU64() == 1 {
if frozen, _ := bc.db.Ancients(); frozen == 0 {
writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil})
writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []rlp.RawValue{rlp.EmptyList})
if err != nil {
log.Error("Error writing genesis to ancients", "err", err)
return 0, err
@ -1279,12 +1369,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
log.Info("Wrote genesis to ancients")
}
}
// Before writing the blocks to the ancients, we need to ensure that
// they correspond to the what the headerchain 'expects'.
// We only check the last block/header, since it's a contiguous chain.
if !bc.HasHeader(last.Hash(), last.NumberU64()) {
return 0, fmt.Errorf("containing header #%d [%x..] unknown", last.Number(), last.Hash().Bytes()[:4])
}
// Write all chain data to ancients.
writeSize, err := rawdb.WriteAncientBlocks(bc.db, blockChain, receiptChain)
if err != nil {
@ -1294,48 +1378,32 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
size += writeSize
// Sync the ancient store explicitly to ensure all data has been flushed to disk.
if err := bc.db.Sync(); err != nil {
if err := bc.db.SyncAncient(); err != nil {
return 0, err
}
// Write hash to number mappings
batch := bc.db.NewBatch()
for _, block := range blockChain {
rawdb.WriteHeaderNumber(batch, block.Hash(), block.NumberU64())
}
if err := batch.Write(); err != nil {
return 0, err
}
// Update the current snap block because all block data is now present in DB.
previousSnapBlock := bc.CurrentSnapBlock().Number.Uint64()
if !updateHead(blockChain[len(blockChain)-1]) {
// We end up here if the header chain has reorg'ed, and the blocks/receipts
// don't match the canonical chain.
if _, err := bc.db.TruncateHead(previousSnapBlock + 1); err != nil {
log.Error("Can't truncate ancient store after failed insert", "err", err)
}
return 0, errSideChainReceipts
}
// Delete block data from the main database.
var (
batch = bc.db.NewBatch()
canonHashes = make(map[common.Hash]struct{}, len(blockChain))
)
for _, block := range blockChain {
canonHashes[block.Hash()] = struct{}{}
if block.NumberU64() == 0 {
continue
}
rawdb.DeleteCanonicalHash(batch, block.NumberU64())
rawdb.DeleteBlockWithoutNumber(batch, block.Hash(), block.NumberU64())
}
// Delete side chain hash-to-number mappings.
for _, nh := range rawdb.ReadAllHashesInRange(bc.db, first.NumberU64(), last.NumberU64()) {
if _, canon := canonHashes[nh.Hash]; !canon {
rawdb.DeleteHeader(batch, nh.Hash, nh.Number)
}
}
if err := batch.Write(); err != nil {
if err := updateHead(blockChain[len(blockChain)-1].Header()); err != nil {
return 0, err
}
stats.processed += int32(len(blockChain))
return 0, nil
}
// writeLive writes blockchain and corresponding receipt chain into active store.
writeLive := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) {
// writeLive writes the blockchain and corresponding receipt chain to the active store.
//
// Notably, in different snap sync cycles, the supplied chain may partially reorganize
// existing local chain segments (reorg around the chain tip). The reorganized part
// will be included in the provided chain segment, and stale canonical markers will be
// silently rewritten. Therefore, no explicit reorg logic is needed.
writeLive := func(blockChain types.Blocks, receiptChain []rlp.RawValue) (int, error) {
var (
skipPresenceCheck = false
batch = bc.db.NewBatch()
@ -1345,10 +1413,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
if bc.insertStopped() {
return 0, errInsertionInterrupted
}
// Short circuit if the owner header is unknown
if !bc.HasHeader(block.Hash(), block.NumberU64()) {
return i, fmt.Errorf("containing header #%d [%x..] unknown", block.Number(), block.Hash().Bytes()[:4])
}
if !skipPresenceCheck {
// Ignore if the entire data is already known
if bc.HasBlock(block.Hash(), block.NumberU64()) {
@ -1362,8 +1426,9 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
}
}
// Write all the data out into the database
rawdb.WriteBody(batch, block.Hash(), block.NumberU64(), block.Body())
rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i])
rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64())
rawdb.WriteBlock(batch, block)
rawdb.WriteRawReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i])
// Write everything belongs to the blocks into the database. So that
// we can ensure all components of body is completed(body, receipts)
@ -1386,21 +1451,27 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
return 0, err
}
}
updateHead(blockChain[len(blockChain)-1])
if err := updateHead(blockChain[len(blockChain)-1].Header()); err != nil {
return 0, err
}
return 0, nil
}
// Write downloaded chain data and corresponding receipt chain data
if len(ancientBlocks) > 0 {
if n, err := writeAncient(ancientBlocks, ancientReceipts); err != nil {
// Split the supplied blocks into two groups, according to the
// given ancient limit.
index := sort.Search(len(blockChain), func(i int) bool {
return blockChain[i].NumberU64() >= ancientLimit
})
if index > 0 {
if n, err := writeAncient(blockChain[:index], receiptChain[:index]); err != nil {
if err == errInsertionInterrupted {
return 0, nil
}
return n, err
}
}
if len(liveBlocks) > 0 {
if n, err := writeLive(liveBlocks, liveReceipts); err != nil {
if index != len(blockChain) {
if n, err := writeLive(blockChain[index:], receiptChain[index:]); err != nil {
if err == errInsertionInterrupted {
return 0, nil
}
@ -1419,7 +1490,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
context = append(context, []interface{}{"ignored", stats.ignored}...)
}
log.Debug("Imported new block receipts", context...)
return 0, nil
}
@ -1706,18 +1776,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness
bc.reportBlock(block, nil, err)
return nil, it.index, err
}
// No validation errors for the first block (or chain prefix skipped)
var activeState *state.StateDB
defer func() {
// The chain importer is starting and stopping trie prefetchers. If a bad
// block or other error is hit however, an early return may not properly
// terminate the background threads. This defer ensures that we clean up
// and dangling prefetcher, without deferring each and holding on live refs.
if activeState != nil {
activeState.StopPrefetcher()
}
}()
// Track the singleton witness from this chain insertion (if any)
var witness *stateless.Witness
@ -1773,63 +1831,20 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness
continue
}
// Retrieve the parent block and it's state to execute on top
start := time.Now()
parent := it.previous()
if parent == nil {
parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
}
statedb, err := state.New(parent.Root, bc.statedb)
if err != nil {
return nil, it.index, err
}
// If we are past Byzantium, enable prefetching to pull in trie node paths
// while processing transactions. Before Byzantium the prefetcher is mostly
// useless due to the intermediate root hashing after each transaction.
if bc.chainConfig.IsByzantium(block.Number()) {
// Generate witnesses either if we're self-testing, or if it's the
// only block being inserted. A bit crude, but witnesses are huge,
// so we refuse to make an entire chain of them.
if bc.vmConfig.StatelessSelfValidation || (makeWitness && len(chain) == 1) {
witness, err = stateless.NewWitness(block.Header(), bc)
if err != nil {
return nil, it.index, err
}
}
statedb.StartPrefetcher("chain", witness)
}
activeState = statedb
// If we have a followup block, run that against the current state to pre-cache
// transactions and probabilistically some of the account/storage trie nodes.
var followupInterrupt atomic.Bool
if !bc.cacheConfig.TrieCleanNoPrefetch {
if followup, err := it.peek(); followup != nil && err == nil {
throwaway, _ := state.New(parent.Root, bc.statedb)
go func(start time.Time, followup *types.Block, throwaway *state.StateDB) {
// Disable tracing for prefetcher executions.
vmCfg := bc.vmConfig
vmCfg.Tracer = nil
bc.prefetcher.Prefetch(followup, throwaway, vmCfg, &followupInterrupt)
blockPrefetchExecuteTimer.Update(time.Since(start))
if followupInterrupt.Load() {
blockPrefetchInterruptMeter.Mark(1)
}
}(time.Now(), followup, throwaway)
}
}
// The traced section of block import.
res, err := bc.processBlock(block, statedb, start, setHead)
followupInterrupt.Store(true)
start := time.Now()
res, err := bc.processBlock(parent.Root, block, setHead, makeWitness && len(chain) == 1)
if err != nil {
return nil, it.index, err
}
// Report the import stats before returning the various results
stats.processed++
stats.usedGas += res.usedGas
witness = res.witness
var snapDiffItems, snapBufItems common.StorageSize
if bc.snaps != nil {
@ -1838,6 +1853,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness
trieDiffNodes, trieBufNodes, _ := bc.triedb.Size()
stats.report(chain, it.index, snapDiffItems, snapBufItems, trieDiffNodes, trieBufNodes, setHead)
// Print confirmation that a future fork is scheduled, but not yet active.
bc.logForkReadiness(block)
if !setHead {
// After merge we expect few side chains. Simply count
// all blocks the CL gives us for GC processing time
@ -1871,6 +1889,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness
"root", block.Root())
}
}
stats.ignored += it.remaining()
return witness, it.index, err
}
@ -1881,11 +1900,74 @@ type blockProcessingResult struct {
usedGas uint64
procTime time.Duration
status WriteStatus
witness *stateless.Witness
}
// processBlock executes and validates the given block. If there was no error
// it writes the block and associated state to database.
func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, start time.Time, setHead bool) (_ *blockProcessingResult, blockEndErr error) {
func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, setHead bool, makeWitness bool) (_ *blockProcessingResult, blockEndErr error) {
var (
err error
startTime = time.Now()
statedb *state.StateDB
interrupt atomic.Bool
)
defer interrupt.Store(true) // terminate the prefetch at the end
if bc.cacheConfig.TrieCleanNoPrefetch {
statedb, err = state.New(parentRoot, bc.statedb)
if err != nil {
return nil, err
}
} else {
// If prefetching is enabled, run that against the current state to pre-cache
// transactions and probabilistically some of the account/storage trie nodes.
//
// Note: the main processor and prefetcher share the same reader with a local
// cache for mitigating the overhead of state access.
reader, err := bc.statedb.ReaderWithCache(parentRoot)
if err != nil {
return nil, err
}
throwaway, err := state.NewWithReader(parentRoot, bc.statedb, reader)
if err != nil {
return nil, err
}
statedb, err = state.NewWithReader(parentRoot, bc.statedb, reader)
if err != nil {
return nil, err
}
go func(start time.Time, throwaway *state.StateDB, block *types.Block) {
// Disable tracing for prefetcher executions.
vmCfg := bc.vmConfig
vmCfg.Tracer = nil
bc.prefetcher.Prefetch(block, throwaway, vmCfg, &interrupt)
blockPrefetchExecuteTimer.Update(time.Since(start))
if interrupt.Load() {
blockPrefetchInterruptMeter.Mark(1)
}
}(time.Now(), throwaway, block)
}
// If we are past Byzantium, enable prefetching to pull in trie node paths
// while processing transactions. Before Byzantium the prefetcher is mostly
// useless due to the intermediate root hashing after each transaction.
var witness *stateless.Witness
if bc.chainConfig.IsByzantium(block.Number()) {
// Generate witnesses either if we're self-testing, or if it's the
// only block being inserted. A bit crude, but witnesses are huge,
// so we refuse to make an entire chain of them.
if bc.vmConfig.StatelessSelfValidation || makeWitness {
witness, err = stateless.NewWitness(block.Header(), bc)
if err != nil {
return nil, err
}
}
statedb.StartPrefetcher("chain", witness)
defer statedb.StopPrefetcher()
}
if bc.logger != nil && bc.logger.OnBlockStart != nil {
bc.logger.OnBlockStart(tracing.BlockEvent{
Block: block,
@ -1944,7 +2026,7 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s
}
}
xvtime := time.Since(xvstart)
proctime := time.Since(start) // processing + validation + cross validation
proctime := time.Since(startTime) // processing + validation + cross validation
// Update the metrics touched during block processing and validation
accountReadTimer.Update(statedb.AccountReads) // Account reads are complete(in processing)
@ -1985,9 +2067,19 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s
triedbCommitTimer.Update(statedb.TrieDBCommits) // Trie database commits are complete, we can mark them
blockWriteTimer.Update(time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.SnapshotCommits - statedb.TrieDBCommits)
blockInsertTimer.UpdateSince(start)
elapsed := time.Since(startTime) + 1 // prevent zero division
blockInsertTimer.Update(elapsed)
return &blockProcessingResult{usedGas: res.GasUsed, procTime: proctime, status: status}, nil
// TODO(rjl493456442) generalize the ResettingTimer
mgasps := float64(res.GasUsed) * 1000 / float64(elapsed)
chainMgaspsMeter.Update(time.Duration(mgasps))
return &blockProcessingResult{
usedGas: res.GasUsed,
procTime: proctime,
status: status,
witness: witness,
}, nil
}
// insertSideChain is called when an import batch hits upon a pruned ancestor
@ -2468,6 +2560,31 @@ func (bc *BlockChain) reportBlock(block *types.Block, res *ProcessResult, err er
log.Error(summarizeBadBlock(block, receipts, bc.Config(), err))
}
// logForkReadiness will write a log when a future fork is scheduled, but not
// active. This is useful so operators know their client is ready for the fork.
func (bc *BlockChain) logForkReadiness(block *types.Block) {
config := bc.Config()
current, last := config.LatestFork(block.Time()), config.LatestFork(math.MaxUint64)
// Short circuit if the timestamp of the last fork is undefined,
// or if the network has already passed the last configured fork.
t := config.Timestamp(last)
if t == nil || current >= last {
return
}
at := time.Unix(int64(*t), 0)
// Only log if:
// - Current time is before the fork activation time
// - Enough time has passed since last alert
now := time.Now()
if now.Before(at) && now.After(bc.lastForkReadyAlert.Add(forkReadyInterval)) {
log.Info("Ready for fork activation", "fork", last, "date", at.Format(time.RFC822),
"remaining", time.Until(at).Round(time.Second), "timestamp", at.Unix())
bc.lastForkReadyAlert = time.Now()
}
}
// summarizeBadBlock returns a string summarizing the bad block and other
// relevant information.
func summarizeBadBlock(block *types.Block, receipts []*types.Receipt, config *params.ChainConfig, err error) string {
@ -2504,15 +2621,84 @@ func (bc *BlockChain) InsertHeaderChain(chain []*types.Header) (int, error) {
if i, err := bc.hc.ValidateHeaderChain(chain); err != nil {
return i, err
}
if !bc.chainmu.TryLock() {
return 0, errChainStopped
}
defer bc.chainmu.Unlock()
_, err := bc.hc.InsertHeaderChain(chain, start)
return 0, err
}
// InsertHeadersBeforeCutoff inserts the given headers into the ancient store
// as they are claimed older than the configured chain cutoff point. All the
// inserted headers are regarded as canonical and chain reorg is not supported.
func (bc *BlockChain) InsertHeadersBeforeCutoff(headers []*types.Header) (int, error) {
if len(headers) == 0 {
return 0, nil
}
// TODO(rjl493456442): Headers before the configured cutoff have already
// been verified by the hash of cutoff header. Theoretically, header validation
// could be skipped here.
if n, err := bc.hc.ValidateHeaderChain(headers); err != nil {
return n, err
}
if !bc.chainmu.TryLock() {
return 0, errChainStopped
}
defer bc.chainmu.Unlock()
// Initialize the ancient store with genesis block if it's empty.
var (
frozen, _ = bc.db.Ancients()
first = headers[0].Number.Uint64()
)
if first == 1 && frozen == 0 {
_, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []rlp.RawValue{rlp.EmptyList})
if err != nil {
log.Error("Error writing genesis to ancients", "err", err)
return 0, err
}
log.Info("Wrote genesis to ancient store")
} else if frozen != first {
return 0, fmt.Errorf("headers are gapped with the ancient store, first: %d, ancient: %d", first, frozen)
}
// Write headers to the ancient store, with block bodies and receipts set to nil
// to ensure consistency across tables in the freezer.
_, err := rawdb.WriteAncientHeaderChain(bc.db, headers)
if err != nil {
return 0, err
}
// Sync the ancient store explicitly to ensure all data has been flushed to disk.
if err := bc.db.SyncAncient(); err != nil {
return 0, err
}
// Write hash to number mappings
batch := bc.db.NewBatch()
for _, header := range headers {
rawdb.WriteHeaderNumber(batch, header.Hash(), header.Number.Uint64())
}
// Write head header and head snap block flags
last := headers[len(headers)-1]
rawdb.WriteHeadHeaderHash(batch, last.Hash())
rawdb.WriteHeadFastBlockHash(batch, last.Hash())
if err := batch.Write(); err != nil {
return 0, err
}
// Truncate the useless chain segment (zero bodies and receipts) in the
// ancient store.
if _, err := bc.db.TruncateTail(last.Number.Uint64() + 1); err != nil {
return 0, err
}
// Last step update all in-memory markers
bc.hc.currentHeader.Store(last)
bc.currentSnapBlock.Store(last)
headHeaderGauge.Update(last.Number.Int64())
headFastBlockGauge.Update(last.Number.Int64())
return 0, nil
}
// SetBlockValidatorAndProcessorForTesting sets the current validator and processor.
// This method can be used to force an invalid blockchain to be verified for tests.
// This method is unsafe and should only be used before block import starts.
@ -2532,9 +2718,3 @@ func (bc *BlockChain) SetTrieFlushInterval(interval time.Duration) {
func (bc *BlockChain) GetTrieFlushInterval() time.Duration {
return time.Duration(bc.flushInterval.Load())
}
// HistoryPruningCutoff returns the configured history pruning point.
// Blocks before this might not be available in the database.
func (bc *BlockChain) HistoryPruningCutoff() uint64 {
return bc.cacheConfig.HistoryPruningCutoff
}

17
core/blockchain_insert.go
View file

@ -27,10 +27,10 @@ import (
// insertStats tracks and reports on block insertion.
type insertStats struct {
queued, processed, ignored int
usedGas uint64
lastIndex int
startTime mclock.AbsTime
processed, ignored int
usedGas uint64
lastIndex int
startTime mclock.AbsTime
}
// statsReportLimit is the time limit during import and export after which we
@ -43,7 +43,8 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn
// Fetch the timings for the batch
var (
now = mclock.Now()
elapsed = now.Sub(st.startTime)
elapsed = now.Sub(st.startTime) + 1 // prevent zero division
mgasps = float64(st.usedGas) * 1000 / float64(elapsed)
)
// If we're at the last block of the batch or report period reached, log
if index == len(chain)-1 || elapsed >= statsReportLimit {
@ -58,7 +59,7 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn
context := []interface{}{
"number", end.Number(), "hash", end.Hash(),
"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
"elapsed", common.PrettyDuration(elapsed), "mgasps", mgasps,
}
if timestamp := time.Unix(int64(end.Time()), 0); time.Since(timestamp) > time.Minute {
context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
@ -74,9 +75,6 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn
}
context = append(context, []interface{}{"triedirty", triebufNodes}...)
if st.queued > 0 {
context = append(context, []interface{}{"queued", st.queued}...)
}
if st.ignored > 0 {
context = append(context, []interface{}{"ignored", st.ignored}...)
}
@ -138,6 +136,7 @@ func (it *insertIterator) next() (*types.Block, error) {
//
// Both header and body validation errors (nil too) is cached into the iterator
// to avoid duplicating work on the following next() call.
// nolint:unused
func (it *insertIterator) peek() (*types.Block, error) {
// If we reached the end of the chain, abort
if it.index+1 >= len(it.chain) {

87
core/blockchain_reader.go
View file

@ -86,6 +86,11 @@ func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header {
return bc.hc.GetHeaderByNumber(number)
}
// GetBlockNumber retrieves the block number associated with a block hash.
func (bc *BlockChain) GetBlockNumber(hash common.Hash) *uint64 {
return bc.hc.GetBlockNumber(hash)
}
// GetHeadersFrom returns a contiguous segment of headers, in rlp-form, going
// backwards from the given number.
func (bc *BlockChain) GetHeadersFrom(number, count uint64) []rlp.RawValue {
@ -229,6 +234,24 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
return receipts
}
// GetRawReceipts retrieves the receipts for all transactions in a given block
// without deriving the internal fields and the Bloom.
func (bc *BlockChain) GetRawReceipts(hash common.Hash, number uint64) types.Receipts {
if receipts, ok := bc.receiptsCache.Get(hash); ok {
return receipts
}
return rawdb.ReadRawReceipts(bc.db, hash, number)
}
// GetReceiptsRLP retrieves the receipts of a block.
func (bc *BlockChain) GetReceiptsRLP(hash common.Hash) rlp.RawValue {
number := rawdb.ReadHeaderNumber(bc.db, hash)
if number == nil {
return nil
}
return rawdb.ReadReceiptsRLP(bc.db, hash, *number)
}
// GetUnclesInChain retrieves all the uncles from a given block backwards until
// a specific distance is reached.
func (bc *BlockChain) GetUnclesInChain(block *types.Block, length int) []*types.Header {
@ -257,42 +280,20 @@ func (bc *BlockChain) GetAncestor(hash common.Hash, number, ancestor uint64, max
// GetTransactionLookup retrieves the lookup along with the transaction
// itself associate with the given transaction hash.
//
// An error will be returned if the transaction is not found, and background
// indexing for transactions is still in progress. The transaction might be
// reachable shortly once it's indexed.
//
// A null will be returned in the transaction is not found and background
// transaction indexing is already finished. The transaction is not existent
// from the node's perspective.
func (bc *BlockChain) GetTransactionLookup(hash common.Hash) (*rawdb.LegacyTxLookupEntry, *types.Transaction, error) {
// A null will be returned if the transaction is not found. This can be due to
// the transaction indexer not being finished. The caller must explicitly check
// the indexer progress.
func (bc *BlockChain) GetTransactionLookup(hash common.Hash) (*rawdb.LegacyTxLookupEntry, *types.Transaction) {
bc.txLookupLock.RLock()
defer bc.txLookupLock.RUnlock()
// Short circuit if the txlookup already in the cache, retrieve otherwise
if item, exist := bc.txLookupCache.Get(hash); exist {
return item.lookup, item.transaction, nil
return item.lookup, item.transaction
}
tx, blockHash, blockNumber, txIndex := rawdb.ReadTransaction(bc.db, hash)
if tx == nil {
progress, err := bc.TxIndexProgress()
if err != nil {
// No error is returned if the transaction indexing progress is unreachable
// due to unexpected internal errors. In such cases, it is impossible to
// determine whether the transaction does not exist or has simply not been
// indexed yet without a progress marker.
//
// In such scenarios, the transaction is treated as unreachable, though
// this is clearly an unintended and unexpected situation.
return nil, nil, nil
}
// The transaction indexing is not finished yet, returning an
// error to explicitly indicate it.
if !progress.Done() {
return nil, nil, errors.New("transaction indexing still in progress")
}
// The transaction is already indexed, the transaction is either
// not existent or not in the range of index, returning null.
return nil, nil, nil
return nil, nil
}
lookup := &rawdb.LegacyTxLookupEntry{
BlockHash: blockHash,
@ -303,7 +304,23 @@ func (bc *BlockChain) GetTransactionLookup(hash common.Hash) (*rawdb.LegacyTxLoo
lookup: lookup,
transaction: tx,
})
return lookup, tx, nil
return lookup, tx
}
// TxIndexDone returns true if the transaction indexer has finished indexing.
func (bc *BlockChain) TxIndexDone() bool {
progress, err := bc.TxIndexProgress()
if err != nil {
// No error is returned if the transaction indexing progress is unreachable
// due to unexpected internal errors. In such cases, it is impossible to
// determine whether the transaction does not exist or has simply not been
// indexed yet without a progress marker.
//
// In such scenarios, the transaction is treated as unreachable, though
// this is clearly an unintended and unexpected situation.
return true
}
return progress.Done()
}
// HasState checks if state trie is fully present in the database or not.
@ -399,7 +416,17 @@ func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) {
if bc.txIndexer == nil {
return TxIndexProgress{}, errors.New("tx indexer is not enabled")
}
return bc.txIndexer.txIndexProgress()
return bc.txIndexer.txIndexProgress(), nil
}
// HistoryPruningCutoff returns the configured history pruning point.
// Blocks before this might not be available in the database.
func (bc *BlockChain) HistoryPruningCutoff() (uint64, common.Hash) {
pt := bc.historyPrunePoint.Load()
if pt == nil {
return 0, bc.genesisBlock.Hash()
}
return pt.BlockNumber, pt.BlockHash
}
// TrieDB retrieves the low level trie database used for data storage.

42
core/blockchain_repair_test.go
View file

@ -1769,7 +1769,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
@ -1791,7 +1791,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
}
)
defer engine.Close()
if snapshots {
if snapshots && scheme == rawdb.HashScheme {
config.SnapshotLimit = 256
config.SnapshotWait = true
}
@ -1820,7 +1820,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
if err := chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false); err != nil {
t.Fatalf("Failed to flush trie state: %v", err)
}
if snapshots {
if snapshots && scheme == rawdb.HashScheme {
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err)
}
@ -1854,7 +1854,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
if err != nil {
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
}
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err = rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
if err != nil {
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
}
@ -1919,7 +1919,7 @@ func testIssue23496(t *testing.T, scheme string) {
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
@ -1952,8 +1952,10 @@ func testIssue23496(t *testing.T, scheme string) {
if _, err := chain.InsertChain(blocks[1:2]); err != nil {
t.Fatalf("Failed to import canonical chain start: %v", err)
}
if err := chain.snaps.Cap(blocks[1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err)
if scheme == rawdb.HashScheme {
if err := chain.snaps.Cap(blocks[1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err)
}
}
// Insert block B3 and commit the state into disk
@ -1977,7 +1979,7 @@ func testIssue23496(t *testing.T, scheme string) {
if err != nil {
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
}
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err = rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
if err != nil {
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
}
@ -1997,15 +1999,23 @@ func testIssue23496(t *testing.T, scheme string) {
}
expHead := uint64(1)
if scheme == rawdb.PathScheme {
expHead = uint64(2)
// The pathdb database makes sure that snapshot and trie are consistent,
// so only the last block is reverted in case of a crash.
expHead = uint64(3)
}
if head := chain.CurrentBlock(); head.Number.Uint64() != expHead {
t.Errorf("Head block mismatch: have %d, want %d", head.Number, expHead)
}
// Reinsert B2-B4
if _, err := chain.InsertChain(blocks[1:]); err != nil {
t.Fatalf("Failed to import canonical chain tail: %v", err)
if scheme == rawdb.PathScheme {
// Reinsert B4
if _, err := chain.InsertChain(blocks[3:]); err != nil {
t.Fatalf("Failed to import canonical chain tail: %v", err)
}
} else {
// Reinsert B2-B4
if _, err := chain.InsertChain(blocks[1:]); err != nil {
t.Fatalf("Failed to import canonical chain tail: %v", err)
}
}
if head := chain.CurrentHeader(); head.Number.Uint64() != uint64(4) {
t.Errorf("Head header mismatch: have %d, want %d", head.Number, 4)
@ -2016,7 +2026,9 @@ func testIssue23496(t *testing.T, scheme string) {
if head := chain.CurrentBlock(); head.Number.Uint64() != uint64(4) {
t.Errorf("Head block mismatch: have %d, want %d", head.Number, uint64(4))
}
if layer := chain.Snapshots().Snapshot(blocks[2].Root()); layer == nil {
t.Error("Failed to regenerate the snapshot of known state")
if scheme == rawdb.HashScheme {
if layer := chain.Snapshots().Snapshot(blocks[2].Root()); layer == nil {
t.Error("Failed to regenerate the snapshot of known state")
}
}
}

4
core/blockchain_sethead_test.go
View file

@ -1973,7 +1973,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
@ -2023,7 +2023,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
}
if tt.commitBlock > 0 {
chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false)
if snapshots {
if snapshots && scheme == rawdb.HashScheme {
if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
t.Fatalf("Failed to flatten snapshots: %v", err)
}

26
core/blockchain_snapshot_test.go
View file

@ -70,7 +70,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
@ -105,7 +105,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
if basic.commitBlock > 0 && basic.commitBlock == point {
chain.TrieDB().Commit(blocks[point-1].Root(), false)
}
if basic.snapshotBlock > 0 && basic.snapshotBlock == point {
if basic.snapshotBlock > 0 && basic.snapshotBlock == point && basic.scheme == rawdb.HashScheme {
// Flushing the entire snap tree into the disk, the
// relevant (a) snapshot root and (b) snapshot generator
// will be persisted atomically.
@ -149,13 +149,17 @@ func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks [
block := chain.GetBlockByNumber(basic.expSnapshotBottom)
if block == nil {
t.Errorf("The corresponding block[%d] of snapshot disk layer is missing", basic.expSnapshotBottom)
} else if !bytes.Equal(chain.snaps.DiskRoot().Bytes(), block.Root().Bytes()) {
t.Errorf("The snapshot disk layer root is incorrect, want %x, get %x", block.Root(), chain.snaps.DiskRoot())
} else if basic.scheme == rawdb.HashScheme {
if !bytes.Equal(chain.snaps.DiskRoot().Bytes(), block.Root().Bytes()) {
t.Errorf("The snapshot disk layer root is incorrect, want %x, get %x", block.Root(), chain.snaps.DiskRoot())
}
}
// Check the snapshot, ensure it's integrated
if err := chain.snaps.Verify(block.Root()); err != nil {
t.Errorf("The disk layer is not integrated %v", err)
if basic.scheme == rawdb.HashScheme {
if err := chain.snaps.Verify(block.Root()); err != nil {
t.Errorf("The disk layer is not integrated %v", err)
}
}
}
@ -261,7 +265,7 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
newdb, err := rawdb.NewDatabaseWithFreezer(pdb, snaptest.ancient, "", false)
newdb, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: snaptest.ancient})
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
@ -565,12 +569,14 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
//
// Expected head header : C8
// Expected head fast block: C8
// Expected head block : G
// Expected snapshot disk : C4
// Expected head block : G (Hash mode), C6 (Hash mode)
// Expected snapshot disk : C4 (Hash mode)
for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
expHead := uint64(0)
if scheme == rawdb.PathScheme {
expHead = uint64(4)
// The pathdb database makes sure that snapshot and trie are consistent,
// so only the last two blocks are reverted in case of a crash.
expHead = uint64(6)
}
test := &crashSnapshotTest{
snapshotTestBasic{

397
core/blockchain_test.go
View file

@ -33,6 +33,7 @@ import (
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/consensus/beacon"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core/history"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
@ -733,18 +734,11 @@ func testFastVsFullChains(t *testing.T, scheme string) {
fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer fast.Stop()
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
if n, err := fast.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := fast.InsertReceiptChain(blocks, receipts, 0); err != nil {
if n, err := fast.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
// Freezer style fast import the chain.
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
ancientDb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
@ -753,10 +747,7 @@ func testFastVsFullChains(t *testing.T, scheme string) {
ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer ancient.Stop()
if n, err := ancient.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(len(blocks)/2)); err != nil {
if n, err := ancient.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), uint64(len(blocks)/2)); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
@ -833,7 +824,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
// makeDb creates a db instance for testing.
makeDb := func() ethdb.Database {
db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
@ -880,14 +871,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer fast.Stop()
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
if n, err := fast.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := fast.InsertReceiptChain(blocks, receipts, 0); err != nil {
if n, err := fast.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
assert(t, "fast", fast, height, height, 0)
@ -900,10 +884,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer ancient.Stop()
if n, err := ancient.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err != nil {
if n, err := ancient.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), uint64(3*len(blocks)/4)); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
assert(t, "ancient", ancient, height, height, 0)
@ -916,6 +897,11 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
// Import the chain as a light node and ensure all pointers are updated
lightDb := makeDb()
defer lightDb.Close()
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
light, _ := NewBlockChain(lightDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
if n, err := light.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
@ -1637,7 +1623,7 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) {
competitor, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*state.TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) })
// Import the shared chain and the original canonical one
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
defer db.Close()
chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil)
@ -1703,21 +1689,14 @@ func testBlockchainRecovery(t *testing.T, scheme string) {
_, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(height), nil)
// Import the chain as a ancient-first node and ensure all pointers are updated
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false)
ancientDb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{Ancient: t.TempDir()})
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
defer ancientDb.Close()
ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
headers[i] = block.Header()
}
if n, err := ancient.InsertHeaderChain(headers); err != nil {
t.Fatalf("failed to insert header %d: %v", n, err)
}
if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err != nil {
if n, err := ancient.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), uint64(3*len(blocks)/4)); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
rawdb.WriteLastPivotNumber(ancientDb, blocks[len(blocks)-1].NumberU64()) // Force fast sync behavior
@ -1741,82 +1720,6 @@ func testBlockchainRecovery(t *testing.T, scheme string) {
}
}
// This test checks that InsertReceiptChain will roll back correctly when attempting to insert a side chain.
func TestInsertReceiptChainRollback(t *testing.T) {
testInsertReceiptChainRollback(t, rawdb.HashScheme)
testInsertReceiptChainRollback(t, rawdb.PathScheme)
}
func testInsertReceiptChainRollback(t *testing.T, scheme string) {
// Generate forked chain. The returned BlockChain object is used to process the side chain blocks.
tmpChain, sideblocks, canonblocks, gspec, err := getLongAndShortChains(scheme)
if err != nil {
t.Fatal(err)
}
defer tmpChain.Stop()
// Get the side chain receipts.
if _, err := tmpChain.InsertChain(sideblocks); err != nil {
t.Fatal("processing side chain failed:", err)
}
t.Log("sidechain head:", tmpChain.CurrentBlock().Number, tmpChain.CurrentBlock().Hash())
sidechainReceipts := make([]types.Receipts, len(sideblocks))
for i, block := range sideblocks {
sidechainReceipts[i] = tmpChain.GetReceiptsByHash(block.Hash())
}
// Get the canon chain receipts.
if _, err := tmpChain.InsertChain(canonblocks); err != nil {
t.Fatal("processing canon chain failed:", err)
}
t.Log("canon head:", tmpChain.CurrentBlock().Number, tmpChain.CurrentBlock().Hash())
canonReceipts := make([]types.Receipts, len(canonblocks))
for i, block := range canonblocks {
canonReceipts[i] = tmpChain.GetReceiptsByHash(block.Hash())
}
// Set up a BlockChain that uses the ancient store.
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
defer ancientDb.Close()
ancientChain, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
defer ancientChain.Stop()
// Import the canonical header chain.
canonHeaders := make([]*types.Header, len(canonblocks))
for i, block := range canonblocks {
canonHeaders[i] = block.Header()
}
if _, err = ancientChain.InsertHeaderChain(canonHeaders); err != nil {
t.Fatal("can't import canon headers:", err)
}
// Try to insert blocks/receipts of the side chain.
_, err = ancientChain.InsertReceiptChain(sideblocks, sidechainReceipts, uint64(len(sideblocks)))
if err == nil {
t.Fatal("expected error from InsertReceiptChain.")
}
if ancientChain.CurrentSnapBlock().Number.Uint64() != 0 {
t.Fatalf("failed to rollback ancient data, want %d, have %d", 0, ancientChain.CurrentSnapBlock().Number)
}
if frozen, err := ancientChain.db.Ancients(); err != nil || frozen != 1 {
t.Fatalf("failed to truncate ancient data, frozen index is %d", frozen)
}
// Insert blocks/receipts of the canonical chain.
_, err = ancientChain.InsertReceiptChain(canonblocks, canonReceipts, uint64(len(canonblocks)))
if err != nil {
t.Fatalf("can't import canon chain receipts: %v", err)
}
if ancientChain.CurrentSnapBlock().Number.Uint64() != canonblocks[len(canonblocks)-1].NumberU64() {
t.Fatalf("failed to insert ancient recept chain after rollback")
}
if frozen, _ := ancientChain.db.Ancients(); frozen != uint64(len(canonblocks))+1 {
t.Fatalf("wrong ancients count %d", frozen)
}
}
// Tests that importing a very large side fork, which is larger than the canon chain,
// but where the difficulty per block is kept low: this means that it will not
// overtake the 'canon' chain until after it's passed canon by about 200 blocks.
@ -1844,7 +1747,7 @@ func testLowDiffLongChain(t *testing.T, scheme string) {
})
// Import the canonical chain
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
diskdb, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
defer diskdb.Close()
chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil)
@ -2056,7 +1959,7 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) {
b.OffsetTime(-9) // A higher difficulty
})
// Import the shared chain and the original canonical one
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
chaindb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
@ -2088,15 +1991,7 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) {
}
} else if typ == "receipts" {
inserter = func(blocks []*types.Block, receipts []types.Receipts) error {
headers := make([]*types.Header, 0, len(blocks))
for _, block := range blocks {
headers = append(headers, block.Header())
}
_, err := chain.InsertHeaderChain(headers)
if err != nil {
return err
}
_, err = chain.InsertReceiptChain(blocks, receipts, 0)
_, err = chain.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0)
return err
}
asserter = func(t *testing.T, block *types.Block) {
@ -2227,7 +2122,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i
}
})
// Import the shared chain and the original canonical one
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
chaindb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
@ -2262,15 +2157,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i
}
} else if typ == "receipts" {
inserter = func(blocks []*types.Block, receipts []types.Receipts) error {
headers := make([]*types.Header, 0, len(blocks))
for _, block := range blocks {
headers = append(headers, block.Header())
}
i, err := chain.InsertHeaderChain(headers)
if err != nil {
return fmt.Errorf("index %d: %w", i, err)
}
_, err = chain.InsertReceiptChain(blocks, receipts, 0)
_, err = chain.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0)
return err
}
asserter = func(t *testing.T, block *types.Block) {
@ -2609,7 +2496,7 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) {
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
@ -3516,7 +3403,7 @@ func testSetCanonical(t *testing.T, scheme string) {
}
gen.AddTx(tx)
})
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
diskdb, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
defer diskdb.Close()
chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil)
@ -4212,7 +4099,7 @@ func TestEIP7702(t *testing.T) {
// The way the auths are combined, it becomes
// 1. tx -> addr1 which is delegated to 0xaaaa
// 2. addr1:0xaaaa calls into addr2:0xbbbb
// 3. addr2:0xbbbb writes to storage
// 3. addr2:0xbbbb writes to storage
auth1, _ := types.SignSetCode(key1, types.SetCodeAuthorization{
ChainID: *uint256.MustFromBig(gspec.Config.ChainID),
Address: aa,
@ -4265,3 +4152,241 @@ func TestEIP7702(t *testing.T) {
t.Fatalf("addr2 storage wrong: expected %d, got %d", fortyTwo, actual)
}
}
// Tests the scenario that the synchronization target in snap sync has been changed
// with a chain reorg at the tip. In this case the reorg'd segment should be unmarked
// with canonical flags.
func TestChainReorgSnapSync(t *testing.T) {
testChainReorgSnapSync(t, 0)
testChainReorgSnapSync(t, 32)
testChainReorgSnapSync(t, gomath.MaxUint64)
}
func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) {
// log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
// Configure and generate a sample block chain
var (
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000000000)
gspec = &Genesis{
Config: params.TestChainConfig,
Alloc: types.GenesisAlloc{address: {Balance: funds}},
BaseFee: big.NewInt(params.InitialBaseFee),
}
signer = types.LatestSigner(gspec.Config)
engine = beacon.New(ethash.NewFaker())
)
genDb, blocks, receipts := GenerateChainWithGenesis(gspec, engine, 32, func(i int, block *BlockGen) {
block.SetCoinbase(common.Address{0x00})
// If the block number is multiple of 3, send a few bonus transactions to the miner
if i%3 == 2 {
for j := 0; j < i%4+1; j++ {
tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key)
if err != nil {
panic(err)
}
block.AddTx(tx)
}
}
})
chainA, receiptsA := GenerateChain(gspec.Config, blocks[len(blocks)-1], engine, genDb, 16, func(i int, gen *BlockGen) {
gen.SetCoinbase(common.Address{0: byte(0xa), 19: byte(i)})
})
chainB, receiptsB := GenerateChain(gspec.Config, blocks[len(blocks)-1], engine, genDb, 20, func(i int, gen *BlockGen) {
gen.SetCoinbase(common.Address{0: byte(0xb), 19: byte(i)})
})
db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
defer db.Close()
chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
defer chain.Stop()
if n, err := chain.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
if n, err := chain.InsertReceiptChain(chainA, types.EncodeBlockReceiptLists(receiptsA), ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
// If the common ancestor is below the ancient limit, rewind the chain head.
// It's aligned with the behavior in the snap sync
ancestor := blocks[len(blocks)-1].NumberU64()
if ancestor < ancientLimit {
rawdb.WriteLastPivotNumber(db, ancestor)
chain.SetHead(ancestor)
}
if n, err := chain.InsertReceiptChain(chainB, types.EncodeBlockReceiptLists(receiptsB), ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
head := chain.CurrentSnapBlock()
if head.Hash() != chainB[len(chainB)-1].Hash() {
t.Errorf("head snap block #%d: header mismatch: want: %v, got: %v", head.Number, chainB[len(chainB)-1].Hash(), head.Hash())
}
// Iterate over all chain data components, and cross reference
for i := 0; i < len(blocks); i++ {
num, hash := blocks[i].NumberU64(), blocks[i].Hash()
header := chain.GetHeaderByNumber(num)
if header.Hash() != hash {
t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
}
}
for i := 0; i < len(chainA); i++ {
num, hash := chainA[i].NumberU64(), chainA[i].Hash()
header := chain.GetHeaderByNumber(num)
if header == nil {
continue
}
if header.Hash() == hash {
t.Errorf("block #%d: unexpected canonical header: %v", num, hash)
}
}
for i := 0; i < len(chainB); i++ {
num, hash := chainB[i].NumberU64(), chainB[i].Hash()
header := chain.GetHeaderByNumber(num)
if header.Hash() != hash {
t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
}
}
}
// Tests the scenario that all the inserted chain segment are with the configured
// chain cutoff point. In this case the chain segment before the cutoff should
// be persisted without the receipts and bodies; chain after should be persisted
// normally.
func TestInsertChainWithCutoff(t *testing.T) {
const chainLength = 64
// Configure and generate a sample block chain
var (
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
funds = big.NewInt(1000000000000000)
gspec = &Genesis{
Config: params.TestChainConfig,
Alloc: types.GenesisAlloc{address: {Balance: funds}},
BaseFee: big.NewInt(params.InitialBaseFee),
}
signer = types.LatestSigner(gspec.Config)
engine = beacon.New(ethash.NewFaker())
)
_, blocks, receipts := GenerateChainWithGenesis(gspec, engine, chainLength, func(i int, block *BlockGen) {
block.SetCoinbase(common.Address{0x00})
tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key)
if err != nil {
panic(err)
}
block.AddTx(tx)
})
// Run the actual tests.
t.Run("cutoff-32/ancientLimit-32", func(t *testing.T) {
// cutoff = 32, ancientLimit = 32
testInsertChainWithCutoff(t, 32, 32, gspec, blocks, receipts)
})
t.Run("cutoff-32/ancientLimit-64", func(t *testing.T) {
// cutoff = 32, ancientLimit = 64 (entire chain in ancient)
testInsertChainWithCutoff(t, 32, 64, gspec, blocks, receipts)
})
t.Run("cutoff-32/ancientLimit-64", func(t *testing.T) {
// cutoff = 32, ancientLimit = 65 (64 blocks in ancient, 1 block in live)
testInsertChainWithCutoff(t, 32, 65, gspec, blocks, receipts)
})
}
func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64, genesis *Genesis, blocks []*types.Block, receipts []types.Receipts) {
// log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
// Add a known pruning point for the duration of the test.
ghash := genesis.ToBlock().Hash()
cutoffBlock := blocks[cutoff-1]
history.PrunePoints[ghash] = &history.PrunePoint{
BlockNumber: cutoffBlock.NumberU64(),
BlockHash: cutoffBlock.Hash(),
}
defer func() {
delete(history.PrunePoints, ghash)
}()
// Enable pruning in cache config.
config := DefaultCacheConfigWithScheme(rawdb.PathScheme)
config.ChainHistoryMode = history.KeepPostMerge
db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
defer db.Close()
chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), genesis, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
defer chain.Stop()
var (
headersBefore []*types.Header
blocksAfter []*types.Block
receiptsAfter []types.Receipts
)
for i, b := range blocks {
if b.NumberU64() < cutoffBlock.NumberU64() {
headersBefore = append(headersBefore, b.Header())
} else {
blocksAfter = append(blocksAfter, b)
receiptsAfter = append(receiptsAfter, receipts[i])
}
}
if n, err := chain.InsertHeadersBeforeCutoff(headersBefore); err != nil {
t.Fatalf("failed to insert headers before cutoff %d: %v", n, err)
}
if n, err := chain.InsertReceiptChain(blocksAfter, types.EncodeBlockReceiptLists(receiptsAfter), ancientLimit); err != nil {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
headSnap := chain.CurrentSnapBlock()
if headSnap.Hash() != blocks[len(blocks)-1].Hash() {
t.Errorf("head snap block #%d: header mismatch: want: %v, got: %v", headSnap.Number, blocks[len(blocks)-1].Hash(), headSnap.Hash())
}
headHeader := chain.CurrentHeader()
if headHeader.Hash() != blocks[len(blocks)-1].Hash() {
t.Errorf("head header #%d: header mismatch: want: %v, got: %v", headHeader.Number, blocks[len(blocks)-1].Hash(), headHeader.Hash())
}
headBlock := chain.CurrentBlock()
if headBlock.Hash() != ghash {
t.Errorf("head block #%d: header mismatch: want: %v, got: %v", headBlock.Number, ghash, headBlock.Hash())
}
// Iterate over all chain data components, and cross reference
for i := 0; i < len(blocks); i++ {
num, hash := blocks[i].NumberU64(), blocks[i].Hash()
// Canonical headers should be visible regardless of cutoff
header := chain.GetHeaderByNumber(num)
if header.Hash() != hash {
t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
}
tail, err := db.Tail()
if err != nil {
t.Fatalf("Failed to get chain tail, %v", err)
}
if tail != cutoffBlock.NumberU64() {
t.Fatalf("Unexpected chain tail, want: %d, got: %d", cutoffBlock.NumberU64(), tail)
}
// Block bodies and receipts before the cutoff should be non-existent
if num < cutoffBlock.NumberU64() {
body := chain.GetBody(hash)
if body != nil {
t.Fatalf("Unexpected block body: %d, cutoff: %d", num, cutoffBlock.NumberU64())
}
receipts := chain.GetReceiptsByHash(hash)
if receipts != nil {
t.Fatalf("Unexpected block receipts: %d, cutoff: %d", num, cutoffBlock.NumberU64())
}
} else {
body := chain.GetBody(hash)
if body == nil || len(body.Transactions) != 1 {
t.Fatalf("Missed block body: %d, cutoff: %d", num, cutoffBlock.NumberU64())
}
receipts := chain.GetReceiptsByHash(hash)
if receipts == nil || len(receipts) != 1 {
t.Fatalf("Missed block receipts: %d, cutoff: %d", num, cutoffBlock.NumberU64())
}
}
}
}

92
core/bloom_indexer.go
View file

@ -1,92 +0,0 @@
// Copyright 2021 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package core
import (
"context"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/bitutil"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
)
const (
// bloomThrottling is the time to wait between processing two consecutive index
// sections. It's useful during chain upgrades to prevent disk overload.
bloomThrottling = 100 * time.Millisecond
)
// BloomIndexer implements a core.ChainIndexer, building up a rotated bloom bits index
// for the Ethereum header bloom filters, permitting blazing fast filtering.
type BloomIndexer struct {
size uint64 // section size to generate bloombits for
db ethdb.Database // database instance to write index data and metadata into
gen *bloombits.Generator // generator to rotate the bloom bits crating the bloom index
section uint64 // Section is the section number being processed currently
head common.Hash // Head is the hash of the last header processed
}
// NewBloomIndexer returns a chain indexer that generates bloom bits data for the
// canonical chain for fast logs filtering.
func NewBloomIndexer(db ethdb.Database, size, confirms uint64) *ChainIndexer {
backend := &BloomIndexer{
db: db,
size: size,
}
table := rawdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix))
return NewChainIndexer(db, table, backend, size, confirms, bloomThrottling, "bloombits")
}
// Reset implements core.ChainIndexerBackend, starting a new bloombits index
// section.
func (b *BloomIndexer) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error {
gen, err := bloombits.NewGenerator(uint(b.size))
b.gen, b.section, b.head = gen, section, common.Hash{}
return err
}
// Process implements core.ChainIndexerBackend, adding a new header's bloom into
// the index.
func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error {
b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom)
b.head = header.Hash()
return nil
}
// Commit implements core.ChainIndexerBackend, finalizing the bloom section and
// writing it out into the database.
func (b *BloomIndexer) Commit() error {
batch := b.db.NewBatchWithSize((int(b.size) / 8) * types.BloomBitLength)
for i := 0; i < types.BloomBitLength; i++ {
bits, err := b.gen.Bitset(uint(i))
if err != nil {
return err
}
rawdb.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits))
}
return batch.Write()
}
// Prune returns an empty error since we don't support pruning here.
func (b *BloomIndexer) Prune(threshold uint64) error {
return nil
}

18
core/bloombits/doc.go
View file

@ -1,18 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
// Package bloombits implements bloom filtering on batches of data.
package bloombits

98
core/bloombits/generator.go
View file

@ -1,98 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bloombits
import (
"errors"
"github.com/ethereum/go-ethereum/core/types"
)
var (
// errSectionOutOfBounds is returned if the user tried to add more bloom filters
// to the batch than available space, or if tries to retrieve above the capacity.
errSectionOutOfBounds = errors.New("section out of bounds")
// errBloomBitOutOfBounds is returned if the user tried to retrieve specified
// bit bloom above the capacity.
errBloomBitOutOfBounds = errors.New("bloom bit out of bounds")
)
// Generator takes a number of bloom filters and generates the rotated bloom bits
// to be used for batched filtering.
type Generator struct {
blooms [types.BloomBitLength][]byte // Rotated blooms for per-bit matching
sections uint // Number of sections to batch together
nextSec uint // Next section to set when adding a bloom
}
// NewGenerator creates a rotated bloom generator that can iteratively fill a
// batched bloom filter's bits.
func NewGenerator(sections uint) (*Generator, error) {
if sections%8 != 0 {
return nil, errors.New("section count not multiple of 8")
}
b := &Generator{sections: sections}
for i := 0; i < types.BloomBitLength; i++ {
b.blooms[i] = make([]byte, sections/8)
}
return b, nil
}
// AddBloom takes a single bloom filter and sets the corresponding bit column
// in memory accordingly.
func (b *Generator) AddBloom(index uint, bloom types.Bloom) error {
// Make sure we're not adding more bloom filters than our capacity
if b.nextSec >= b.sections {
return errSectionOutOfBounds
}
if b.nextSec != index {
return errors.New("bloom filter with unexpected index")
}
// Rotate the bloom and insert into our collection
byteIndex := b.nextSec / 8
bitIndex := byte(7 - b.nextSec%8)
for byt := 0; byt < types.BloomByteLength; byt++ {
bloomByte := bloom[types.BloomByteLength-1-byt]
if bloomByte == 0 {
continue
}
base := 8 * byt
b.blooms[base+7][byteIndex] |= ((bloomByte >> 7) & 1) << bitIndex
b.blooms[base+6][byteIndex] |= ((bloomByte >> 6) & 1) << bitIndex
b.blooms[base+5][byteIndex] |= ((bloomByte >> 5) & 1) << bitIndex
b.blooms[base+4][byteIndex] |= ((bloomByte >> 4) & 1) << bitIndex
b.blooms[base+3][byteIndex] |= ((bloomByte >> 3) & 1) << bitIndex
b.blooms[base+2][byteIndex] |= ((bloomByte >> 2) & 1) << bitIndex
b.blooms[base+1][byteIndex] |= ((bloomByte >> 1) & 1) << bitIndex
b.blooms[base][byteIndex] |= (bloomByte & 1) << bitIndex
}
b.nextSec++
return nil
}
// Bitset returns the bit vector belonging to the given bit index after all
// blooms have been added.
func (b *Generator) Bitset(idx uint) ([]byte, error) {
if b.nextSec != b.sections {
return nil, errors.New("bloom not fully generated yet")
}
if idx >= types.BloomBitLength {
return nil, errBloomBitOutOfBounds
}
return b.blooms[idx], nil
}

100
core/bloombits/generator_test.go
View file

@ -1,100 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bloombits
import (
"bytes"
crand "crypto/rand"
"math/rand"
"testing"
"github.com/ethereum/go-ethereum/core/types"
)
// Tests that batched bloom bits are correctly rotated from the input bloom
// filters.
func TestGenerator(t *testing.T) {
// Generate the input and the rotated output
var input, output [types.BloomBitLength][types.BloomByteLength]byte
for i := 0; i < types.BloomBitLength; i++ {
for j := 0; j < types.BloomBitLength; j++ {
bit := byte(rand.Int() % 2)
input[i][j/8] |= bit << byte(7-j%8)
output[types.BloomBitLength-1-j][i/8] |= bit << byte(7-i%8)
}
}
// Crunch the input through the generator and verify the result
gen, err := NewGenerator(types.BloomBitLength)
if err != nil {
t.Fatalf("failed to create bloombit generator: %v", err)
}
for i, bloom := range input {
if err := gen.AddBloom(uint(i), bloom); err != nil {
t.Fatalf("bloom %d: failed to add: %v", i, err)
}
}
for i, want := range output {
have, err := gen.Bitset(uint(i))
if err != nil {
t.Fatalf("output %d: failed to retrieve bits: %v", i, err)
}
if !bytes.Equal(have, want[:]) {
t.Errorf("output %d: bit vector mismatch have %x, want %x", i, have, want)
}
}
}
func BenchmarkGenerator(b *testing.B) {
var input [types.BloomBitLength][types.BloomByteLength]byte
b.Run("empty", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
// Crunch the input through the generator and verify the result
gen, err := NewGenerator(types.BloomBitLength)
if err != nil {
b.Fatalf("failed to create bloombit generator: %v", err)
}
for j, bloom := range &input {
if err := gen.AddBloom(uint(j), bloom); err != nil {
b.Fatalf("bloom %d: failed to add: %v", i, err)
}
}
}
})
for i := 0; i < types.BloomBitLength; i++ {
crand.Read(input[i][:])
}
b.Run("random", func(b *testing.B) {
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
// Crunch the input through the generator and verify the result
gen, err := NewGenerator(types.BloomBitLength)
if err != nil {
b.Fatalf("failed to create bloombit generator: %v", err)
}
for j, bloom := range &input {
if err := gen.AddBloom(uint(j), bloom); err != nil {
b.Fatalf("bloom %d: failed to add: %v", i, err)
}
}
}
})
}

649
core/bloombits/matcher.go
View file

@ -1,649 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bloombits
import (
"bytes"
"context"
"errors"
"math"
"sort"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common/bitutil"
"github.com/ethereum/go-ethereum/crypto"
)
// bloomIndexes represents the bit indexes inside the bloom filter that belong
// to some key.
type bloomIndexes [3]uint
// calcBloomIndexes returns the bloom filter bit indexes belonging to the given key.
func calcBloomIndexes(b []byte) bloomIndexes {
b = crypto.Keccak256(b)
var idxs bloomIndexes
for i := 0; i < len(idxs); i++ {
idxs[i] = (uint(b[2*i])<<8)&2047 + uint(b[2*i+1])
}
return idxs
}
// partialMatches with a non-nil vector represents a section in which some sub-
// matchers have already found potential matches. Subsequent sub-matchers will
// binary AND their matches with this vector. If vector is nil, it represents a
// section to be processed by the first sub-matcher.
type partialMatches struct {
section uint64
bitset []byte
}
// Retrieval represents a request for retrieval task assignments for a given
// bit with the given number of fetch elements, or a response for such a request.
// It can also have the actual results set to be used as a delivery data struct.
//
// The context and error fields are used by the light client to terminate matching
// early if an error is encountered on some path of the pipeline.
type Retrieval struct {
Bit uint
Sections []uint64
Bitsets [][]byte
Context context.Context
Error error
}
// Matcher is a pipelined system of schedulers and logic matchers which perform
// binary AND/OR operations on the bit-streams, creating a stream of potential
// blocks to inspect for data content.
type Matcher struct {
sectionSize uint64 // Size of the data batches to filter on
filters [][]bloomIndexes // Filter the system is matching for
schedulers map[uint]*scheduler // Retrieval schedulers for loading bloom bits
retrievers chan chan uint // Retriever processes waiting for bit allocations
counters chan chan uint // Retriever processes waiting for task count reports
retrievals chan chan *Retrieval // Retriever processes waiting for task allocations
deliveries chan *Retrieval // Retriever processes waiting for task response deliveries
running atomic.Bool // Atomic flag whether a session is live or not
}
// NewMatcher creates a new pipeline for retrieving bloom bit streams and doing
// address and topic filtering on them. Setting a filter component to `nil` is
// allowed and will result in that filter rule being skipped (OR 0x11...1).
func NewMatcher(sectionSize uint64, filters [][][]byte) *Matcher {
// Create the matcher instance
m := &Matcher{
sectionSize: sectionSize,
schedulers: make(map[uint]*scheduler),
retrievers: make(chan chan uint),
counters: make(chan chan uint),
retrievals: make(chan chan *Retrieval),
deliveries: make(chan *Retrieval),
}
// Calculate the bloom bit indexes for the groups we're interested in
m.filters = nil
for _, filter := range filters {
// Gather the bit indexes of the filter rule, special casing the nil filter
if len(filter) == 0 {
continue
}
bloomBits := make([]bloomIndexes, len(filter))
for i, clause := range filter {
if clause == nil {
bloomBits = nil
break
}
bloomBits[i] = calcBloomIndexes(clause)
}
// Accumulate the filter rules if no nil rule was within
if bloomBits != nil {
m.filters = append(m.filters, bloomBits)
}
}
// For every bit, create a scheduler to load/download the bit vectors
for _, bloomIndexLists := range m.filters {
for _, bloomIndexList := range bloomIndexLists {
for _, bloomIndex := range bloomIndexList {
m.addScheduler(bloomIndex)
}
}
}
return m
}
// addScheduler adds a bit stream retrieval scheduler for the given bit index if
// it has not existed before. If the bit is already selected for filtering, the
// existing scheduler can be used.
func (m *Matcher) addScheduler(idx uint) {
if _, ok := m.schedulers[idx]; ok {
return
}
m.schedulers[idx] = newScheduler(idx)
}
// Start starts the matching process and returns a stream of bloom matches in
// a given range of blocks. If there are no more matches in the range, the result
// channel is closed.
func (m *Matcher) Start(ctx context.Context, begin, end uint64, results chan uint64) (*MatcherSession, error) {
// Make sure we're not creating concurrent sessions
if m.running.Swap(true) {
return nil, errors.New("matcher already running")
}
defer m.running.Store(false)
// Initiate a new matching round
session := &MatcherSession{
matcher: m,
quit: make(chan struct{}),
ctx: ctx,
}
for _, scheduler := range m.schedulers {
scheduler.reset()
}
sink := m.run(begin, end, cap(results), session)
// Read the output from the result sink and deliver to the user
session.pend.Add(1)
go func() {
defer session.pend.Done()
defer close(results)
for {
select {
case <-session.quit:
return
case res, ok := <-sink:
// New match result found
if !ok {
return
}
// Calculate the first and last blocks of the section
sectionStart := res.section * m.sectionSize
first := sectionStart
if begin > first {
first = begin
}
last := sectionStart + m.sectionSize - 1
if end < last {
last = end
}
// Iterate over all the blocks in the section and return the matching ones
for i := first; i <= last; i++ {
// Skip the entire byte if no matches are found inside (and we're processing an entire byte!)
next := res.bitset[(i-sectionStart)/8]
if next == 0 {
if i%8 == 0 {
i += 7
}
continue
}
// Some bit it set, do the actual submatching
if bit := 7 - i%8; next&(1<<bit) != 0 {
select {
case <-session.quit:
return
case results <- i:
}
}
}
}
}
}()
return session, nil
}
// run creates a daisy-chain of sub-matchers, one for the address set and one
// for each topic set, each sub-matcher receiving a section only if the previous
// ones have all found a potential match in one of the blocks of the section,
// then binary AND-ing its own matches and forwarding the result to the next one.
//
// The method starts feeding the section indexes into the first sub-matcher on a
// new goroutine and returns a sink channel receiving the results.
func (m *Matcher) run(begin, end uint64, buffer int, session *MatcherSession) chan *partialMatches {
// Create the source channel and feed section indexes into
source := make(chan *partialMatches, buffer)
session.pend.Add(1)
go func() {
defer session.pend.Done()
defer close(source)
for i := begin / m.sectionSize; i <= end/m.sectionSize; i++ {
select {
case <-session.quit:
return
case source <- &partialMatches{i, bytes.Repeat([]byte{0xff}, int(m.sectionSize/8))}:
}
}
}()
// Assemble the daisy-chained filtering pipeline
next := source
dist := make(chan *request, buffer)
for _, bloom := range m.filters {
next = m.subMatch(next, dist, bloom, session)
}
// Start the request distribution
session.pend.Add(1)
go m.distributor(dist, session)
return next
}
// subMatch creates a sub-matcher that filters for a set of addresses or topics, binary OR-s those matches, then
// binary AND-s the result to the daisy-chain input (source) and forwards it to the daisy-chain output.
// The matches of each address/topic are calculated by fetching the given sections of the three bloom bit indexes belonging to
// that address/topic, and binary AND-ing those vectors together.
func (m *Matcher) subMatch(source chan *partialMatches, dist chan *request, bloom []bloomIndexes, session *MatcherSession) chan *partialMatches {
// Start the concurrent schedulers for each bit required by the bloom filter
sectionSources := make([][3]chan uint64, len(bloom))
sectionSinks := make([][3]chan []byte, len(bloom))
for i, bits := range bloom {
for j, bit := range bits {
sectionSources[i][j] = make(chan uint64, cap(source))
sectionSinks[i][j] = make(chan []byte, cap(source))
m.schedulers[bit].run(sectionSources[i][j], dist, sectionSinks[i][j], session.quit, &session.pend)
}
}
process := make(chan *partialMatches, cap(source)) // entries from source are forwarded here after fetches have been initiated
results := make(chan *partialMatches, cap(source))
session.pend.Add(2)
go func() {
// Tear down the goroutine and terminate all source channels
defer session.pend.Done()
defer close(process)
defer func() {
for _, bloomSources := range sectionSources {
for _, bitSource := range bloomSources {
close(bitSource)
}
}
}()
// Read sections from the source channel and multiplex into all bit-schedulers
for {
select {
case <-session.quit:
return
case subres, ok := <-source:
// New subresult from previous link
if !ok {
return
}
// Multiplex the section index to all bit-schedulers
for _, bloomSources := range sectionSources {
for _, bitSource := range bloomSources {
select {
case <-session.quit:
return
case bitSource <- subres.section:
}
}
}
// Notify the processor that this section will become available
select {
case <-session.quit:
return
case process <- subres:
}
}
}
}()
go func() {
// Tear down the goroutine and terminate the final sink channel
defer session.pend.Done()
defer close(results)
// Read the source notifications and collect the delivered results
for {
select {
case <-session.quit:
return
case subres, ok := <-process:
// Notified of a section being retrieved
if !ok {
return
}
// Gather all the sub-results and merge them together
var orVector []byte
for _, bloomSinks := range sectionSinks {
var andVector []byte
for _, bitSink := range bloomSinks {
var data []byte
select {
case <-session.quit:
return
case data = <-bitSink:
}
if andVector == nil {
andVector = make([]byte, int(m.sectionSize/8))
copy(andVector, data)
} else {
bitutil.ANDBytes(andVector, andVector, data)
}
}
if orVector == nil {
orVector = andVector
} else {
bitutil.ORBytes(orVector, orVector, andVector)
}
}
if orVector == nil {
orVector = make([]byte, int(m.sectionSize/8))
}
if subres.bitset != nil {
bitutil.ANDBytes(orVector, orVector, subres.bitset)
}
if bitutil.TestBytes(orVector) {
select {
case <-session.quit:
return
case results <- &partialMatches{subres.section, orVector}:
}
}
}
}
}()
return results
}
// distributor receives requests from the schedulers and queues them into a set
// of pending requests, which are assigned to retrievers wanting to fulfil them.
func (m *Matcher) distributor(dist chan *request, session *MatcherSession) {
defer session.pend.Done()
var (
requests = make(map[uint][]uint64) // Per-bit list of section requests, ordered by section number
unallocs = make(map[uint]struct{}) // Bits with pending requests but not allocated to any retriever
retrievers chan chan uint // Waiting retrievers (toggled to nil if unallocs is empty)
allocs int // Number of active allocations to handle graceful shutdown requests
shutdown = session.quit // Shutdown request channel, will gracefully wait for pending requests
)
// assign is a helper method to try to assign a pending bit an actively
// listening servicer, or schedule it up for later when one arrives.
assign := func(bit uint) {
select {
case fetcher := <-m.retrievers:
allocs++
fetcher <- bit
default:
// No retrievers active, start listening for new ones
retrievers = m.retrievers
unallocs[bit] = struct{}{}
}
}
for {
select {
case <-shutdown:
// Shutdown requested. No more retrievers can be allocated,
// but we still need to wait until all pending requests have returned.
shutdown = nil
if allocs == 0 {
return
}
case req := <-dist:
// New retrieval request arrived to be distributed to some fetcher process
queue := requests[req.bit]
index := sort.Search(len(queue), func(i int) bool { return queue[i] >= req.section })
requests[req.bit] = append(queue[:index], append([]uint64{req.section}, queue[index:]...)...)
// If it's a new bit and we have waiting fetchers, allocate to them
if len(queue) == 0 {
assign(req.bit)
}
case fetcher := <-retrievers:
// New retriever arrived, find the lowest section-ed bit to assign
bit, best := uint(0), uint64(math.MaxUint64)
for idx := range unallocs {
if requests[idx][0] < best {
bit, best = idx, requests[idx][0]
}
}
// Stop tracking this bit (and alloc notifications if no more work is available)
delete(unallocs, bit)
if len(unallocs) == 0 {
retrievers = nil
}
allocs++
fetcher <- bit
case fetcher := <-m.counters:
// New task count request arrives, return number of items
fetcher <- uint(len(requests[<-fetcher]))
case fetcher := <-m.retrievals:
// New fetcher waiting for tasks to retrieve, assign
task := <-fetcher
if want := len(task.Sections); want >= len(requests[task.Bit]) {
task.Sections = requests[task.Bit]
delete(requests, task.Bit)
} else {
task.Sections = append(task.Sections[:0], requests[task.Bit][:want]...)
requests[task.Bit] = append(requests[task.Bit][:0], requests[task.Bit][want:]...)
}
fetcher <- task
// If anything was left unallocated, try to assign to someone else
if len(requests[task.Bit]) > 0 {
assign(task.Bit)
}
case result := <-m.deliveries:
// New retrieval task response from fetcher, split out missing sections and
// deliver complete ones
var (
sections = make([]uint64, 0, len(result.Sections))
bitsets = make([][]byte, 0, len(result.Bitsets))
missing = make([]uint64, 0, len(result.Sections))
)
for i, bitset := range result.Bitsets {
if len(bitset) == 0 {
missing = append(missing, result.Sections[i])
continue
}
sections = append(sections, result.Sections[i])
bitsets = append(bitsets, bitset)
}
m.schedulers[result.Bit].deliver(sections, bitsets)
allocs--
// Reschedule missing sections and allocate bit if newly available
if len(missing) > 0 {
queue := requests[result.Bit]
for _, section := range missing {
index := sort.Search(len(queue), func(i int) bool { return queue[i] >= section })
queue = append(queue[:index], append([]uint64{section}, queue[index:]...)...)
}
requests[result.Bit] = queue
if len(queue) == len(missing) {
assign(result.Bit)
}
}
// End the session when all pending deliveries have arrived.
if shutdown == nil && allocs == 0 {
return
}
}
}
}
// MatcherSession is returned by a started matcher to be used as a terminator
// for the actively running matching operation.
type MatcherSession struct {
matcher *Matcher
closer sync.Once // Sync object to ensure we only ever close once
quit chan struct{} // Quit channel to request pipeline termination
ctx context.Context // Context used by the light client to abort filtering
err error // Global error to track retrieval failures deep in the chain
errLock sync.Mutex
pend sync.WaitGroup
}
// Close stops the matching process and waits for all subprocesses to terminate
// before returning. The timeout may be used for graceful shutdown, allowing the
// currently running retrievals to complete before this time.
func (s *MatcherSession) Close() {
s.closer.Do(func() {
// Signal termination and wait for all goroutines to tear down
close(s.quit)
s.pend.Wait()
})
}
// Error returns any failure encountered during the matching session.
func (s *MatcherSession) Error() error {
s.errLock.Lock()
defer s.errLock.Unlock()
return s.err
}
// allocateRetrieval assigns a bloom bit index to a client process that can either
// immediately request and fetch the section contents assigned to this bit or wait
// a little while for more sections to be requested.
func (s *MatcherSession) allocateRetrieval() (uint, bool) {
fetcher := make(chan uint)
select {
case <-s.quit:
return 0, false
case s.matcher.retrievers <- fetcher:
bit, ok := <-fetcher
return bit, ok
}
}
// pendingSections returns the number of pending section retrievals belonging to
// the given bloom bit index.
func (s *MatcherSession) pendingSections(bit uint) int {
fetcher := make(chan uint)
select {
case <-s.quit:
return 0
case s.matcher.counters <- fetcher:
fetcher <- bit
return int(<-fetcher)
}
}
// allocateSections assigns all or part of an already allocated bit-task queue
// to the requesting process.
func (s *MatcherSession) allocateSections(bit uint, count int) []uint64 {
fetcher := make(chan *Retrieval)
select {
case <-s.quit:
return nil
case s.matcher.retrievals <- fetcher:
task := &Retrieval{
Bit: bit,
Sections: make([]uint64, count),
}
fetcher <- task
return (<-fetcher).Sections
}
}
// deliverSections delivers a batch of section bit-vectors for a specific bloom
// bit index to be injected into the processing pipeline.
func (s *MatcherSession) deliverSections(bit uint, sections []uint64, bitsets [][]byte) {
s.matcher.deliveries <- &Retrieval{Bit: bit, Sections: sections, Bitsets: bitsets}
}
// Multiplex polls the matcher session for retrieval tasks and multiplexes it into
// the requested retrieval queue to be serviced together with other sessions.
//
// This method will block for the lifetime of the session. Even after termination
// of the session, any request in-flight need to be responded to! Empty responses
// are fine though in that case.
func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan *Retrieval) {
waitTimer := time.NewTimer(wait)
defer waitTimer.Stop()
for {
// Allocate a new bloom bit index to retrieve data for, stopping when done
bit, ok := s.allocateRetrieval()
if !ok {
return
}
// Bit allocated, throttle a bit if we're below our batch limit
if s.pendingSections(bit) < batch {
waitTimer.Reset(wait)
select {
case <-s.quit:
// Session terminating, we can't meaningfully service, abort
s.allocateSections(bit, 0)
s.deliverSections(bit, []uint64{}, [][]byte{})
return
case <-waitTimer.C:
// Throttling up, fetch whatever is available
}
}
// Allocate as much as we can handle and request servicing
sections := s.allocateSections(bit, batch)
request := make(chan *Retrieval)
select {
case <-s.quit:
// Session terminating, we can't meaningfully service, abort
s.deliverSections(bit, sections, make([][]byte, len(sections)))
return
case mux <- request:
// Retrieval accepted, something must arrive before we're aborting
request <- &Retrieval{Bit: bit, Sections: sections, Context: s.ctx}
result := <-request
// Deliver a result before s.Close() to avoid a deadlock
s.deliverSections(result.Bit, result.Sections, result.Bitsets)
if result.Error != nil {
s.errLock.Lock()
s.err = result.Error
s.errLock.Unlock()
s.Close()
}
}
}
}

292
core/bloombits/matcher_test.go
View file

@ -1,292 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bloombits
import (
"context"
"math/rand"
"sync/atomic"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
)
const testSectionSize = 4096
// Tests that wildcard filter rules (nil) can be specified and are handled well.
func TestMatcherWildcards(t *testing.T) {
t.Parallel()
matcher := NewMatcher(testSectionSize, [][][]byte{
{common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard
{common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()}, // Default hash is not a wildcard
{common.Hash{0x01}.Bytes()}, // Plain rule, sanity check
{common.Hash{0x01}.Bytes(), nil}, // Wildcard suffix, drop rule
{nil, common.Hash{0x01}.Bytes()}, // Wildcard prefix, drop rule
{nil, nil}, // Wildcard combo, drop rule
{}, // Inited wildcard rule, drop rule
nil, // Proper wildcard rule, drop rule
})
if len(matcher.filters) != 3 {
t.Fatalf("filter system size mismatch: have %d, want %d", len(matcher.filters), 3)
}
if len(matcher.filters[0]) != 2 {
t.Fatalf("address clause size mismatch: have %d, want %d", len(matcher.filters[0]), 2)
}
if len(matcher.filters[1]) != 2 {
t.Fatalf("combo topic clause size mismatch: have %d, want %d", len(matcher.filters[1]), 2)
}
if len(matcher.filters[2]) != 1 {
t.Fatalf("singletone topic clause size mismatch: have %d, want %d", len(matcher.filters[2]), 1)
}
}
// Tests the matcher pipeline on a single continuous workflow without interrupts.
func TestMatcherContinuous(t *testing.T) {
t.Parallel()
testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, false, 75)
testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, false, 81)
testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, false, 36)
}
// Tests the matcher pipeline on a constantly interrupted and resumed work pattern
// with the aim of ensuring data items are requested only once.
func TestMatcherIntermittent(t *testing.T) {
t.Parallel()
testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, true, 75)
testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, true, 81)
testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, true, 36)
}
// Tests the matcher pipeline on random input to hopefully catch anomalies.
func TestMatcherRandom(t *testing.T) {
t.Parallel()
for i := 0; i < 10; i++ {
testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 0, 10000, 0)
testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 0, 10000, 0)
testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 0, 10000, 0)
testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 0, 10000, 0)
testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 0, 10000, 0)
}
}
// Tests that the matcher can properly find matches if the starting block is
// shifted from a multiple of 8. This is needed to cover an optimisation with
// bitset matching https://github.com/ethereum/go-ethereum/issues/15309.
func TestMatcherShifted(t *testing.T) {
t.Parallel()
// Block 0 always matches in the tests, skip ahead of first 8 blocks with the
// start to get a potential zero byte in the matcher bitset.
// To keep the second bitset byte zero, the filter must only match for the first
// time in block 16, so doing an all-16 bit filter should suffice.
// To keep the starting block non divisible by 8, block number 9 is the first
// that would introduce a shift and not match block 0.
testMatcherBothModes(t, [][]bloomIndexes{{{16, 16, 16}}}, 9, 64, 0)
}
// Tests that matching on everything doesn't crash (special case internally).
func TestWildcardMatcher(t *testing.T) {
t.Parallel()
testMatcherBothModes(t, nil, 0, 10000, 0)
}
// makeRandomIndexes generates a random filter system, composed of multiple filter
// criteria, each having one bloom list component for the address and arbitrarily
// many topic bloom list components.
func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {
res := make([][]bloomIndexes, len(lengths))
for i, topics := range lengths {
res[i] = make([]bloomIndexes, topics)
for j := 0; j < topics; j++ {
for k := 0; k < len(res[i][j]); k++ {
res[i][j][k] = uint(rand.Intn(max-1) + 2)
}
}
}
return res
}
// testMatcherDiffBatches runs the given matches test in single-delivery and also
// in batches delivery mode, verifying that all kinds of deliveries are handled
// correctly within.
func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) {
singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1)
batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16)
if singleton != batched {
t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in singleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched)
}
}
// testMatcherBothModes runs the given matcher test in both continuous as well as
// in intermittent mode, verifying that the request counts match each other.
func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, retrievals uint32) {
continuous := testMatcher(t, filter, start, blocks, false, retrievals, 16)
intermittent := testMatcher(t, filter, start, blocks, true, retrievals, 16)
if continuous != intermittent {
t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent)
}
}
// testMatcher is a generic tester to run the given matcher test and return the
// number of requests made for cross validation between different modes.
func testMatcher(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 {
// Create a new matcher an simulate our explicit random bitsets
matcher := NewMatcher(testSectionSize, nil)
matcher.filters = filter
for _, rule := range filter {
for _, topic := range rule {
for _, bit := range topic {
matcher.addScheduler(bit)
}
}
}
// Track the number of retrieval requests made
var requested atomic.Uint32
// Start the matching session for the filter and the retriever goroutines
quit := make(chan struct{})
matches := make(chan uint64, 16)
session, err := matcher.Start(context.Background(), start, blocks-1, matches)
if err != nil {
t.Fatalf("failed to stat matcher session: %v", err)
}
startRetrievers(session, quit, &requested, maxReqCount)
// Iterate over all the blocks and verify that the pipeline produces the correct matches
for i := start; i < blocks; i++ {
if expMatch3(filter, i) {
match, ok := <-matches
if !ok {
t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i)
return 0
}
if match != i {
t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match)
}
// If we're testing intermittent mode, abort and restart the pipeline
if intermittent {
session.Close()
close(quit)
quit = make(chan struct{})
matches = make(chan uint64, 16)
session, err = matcher.Start(context.Background(), i+1, blocks-1, matches)
if err != nil {
t.Fatalf("failed to stat matcher session: %v", err)
}
startRetrievers(session, quit, &requested, maxReqCount)
}
}
}
// Ensure the result channel is torn down after the last block
match, ok := <-matches
if ok {
t.Errorf("filter = %v blocks = %v intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match)
}
// Clean up the session and ensure we match the expected retrieval count
session.Close()
close(quit)
if retrievals != 0 && requested.Load() != retrievals {
t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested.Load(), retrievals)
}
return requested.Load()
}
// startRetrievers starts a batch of goroutines listening for section requests
// and serving them.
func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *atomic.Uint32, batch int) {
requests := make(chan chan *Retrieval)
for i := 0; i < 10; i++ {
// Start a multiplexer to test multiple threaded execution
go session.Multiplex(batch, 100*time.Microsecond, requests)
// Start a services to match the above multiplexer
go func() {
for {
// Wait for a service request or a shutdown
select {
case <-quit:
return
case request := <-requests:
task := <-request
task.Bitsets = make([][]byte, len(task.Sections))
for i, section := range task.Sections {
if rand.Int()%4 != 0 { // Handle occasional missing deliveries
task.Bitsets[i] = generateBitset(task.Bit, section)
retrievals.Add(1)
}
}
request <- task
}
}
}()
}
}
// generateBitset generates the rotated bitset for the given bloom bit and section
// numbers.
func generateBitset(bit uint, section uint64) []byte {
bitset := make([]byte, testSectionSize/8)
for i := 0; i < len(bitset); i++ {
for b := 0; b < 8; b++ {
blockIdx := section*testSectionSize + uint64(i*8+b)
bitset[i] += bitset[i]
if (blockIdx % uint64(bit)) == 0 {
bitset[i]++
}
}
}
return bitset
}
func expMatch1(filter bloomIndexes, i uint64) bool {
for _, ii := range filter {
if (i % uint64(ii)) != 0 {
return false
}
}
return true
}
func expMatch2(filter []bloomIndexes, i uint64) bool {
for _, ii := range filter {
if expMatch1(ii, i) {
return true
}
}
return false
}
func expMatch3(filter [][]bloomIndexes, i uint64) bool {
for _, ii := range filter {
if !expMatch2(ii, i) {
return false
}
}
return true
}

181
core/bloombits/scheduler.go
View file

@ -1,181 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bloombits
import (
"sync"
)
// request represents a bloom retrieval task to prioritize and pull from the local
// database or remotely from the network.
type request struct {
section uint64 // Section index to retrieve the bit-vector from
bit uint // Bit index within the section to retrieve the vector of
}
// response represents the state of a requested bit-vector through a scheduler.
type response struct {
cached []byte // Cached bits to dedup multiple requests
done chan struct{} // Channel to allow waiting for completion
}
// scheduler handles the scheduling of bloom-filter retrieval operations for
// entire section-batches belonging to a single bloom bit. Beside scheduling the
// retrieval operations, this struct also deduplicates the requests and caches
// the results to minimize network/database overhead even in complex filtering
// scenarios.
type scheduler struct {
bit uint // Index of the bit in the bloom filter this scheduler is responsible for
responses map[uint64]*response // Currently pending retrieval requests or already cached responses
lock sync.Mutex // Lock protecting the responses from concurrent access
}
// newScheduler creates a new bloom-filter retrieval scheduler for a specific
// bit index.
func newScheduler(idx uint) *scheduler {
return &scheduler{
bit: idx,
responses: make(map[uint64]*response),
}
}
// run creates a retrieval pipeline, receiving section indexes from sections and
// returning the results in the same order through the done channel. Concurrent
// runs of the same scheduler are allowed, leading to retrieval task deduplication.
func (s *scheduler) run(sections chan uint64, dist chan *request, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) {
// Create a forwarder channel between requests and responses of the same size as
// the distribution channel (since that will block the pipeline anyway).
pend := make(chan uint64, cap(dist))
// Start the pipeline schedulers to forward between user -> distributor -> user
wg.Add(2)
go s.scheduleRequests(sections, dist, pend, quit, wg)
go s.scheduleDeliveries(pend, done, quit, wg)
}
// reset cleans up any leftovers from previous runs. This is required before a
// restart to ensure the no previously requested but never delivered state will
// cause a lockup.
func (s *scheduler) reset() {
s.lock.Lock()
defer s.lock.Unlock()
for section, res := range s.responses {
if res.cached == nil {
delete(s.responses, section)
}
}
}
// scheduleRequests reads section retrieval requests from the input channel,
// deduplicates the stream and pushes unique retrieval tasks into the distribution
// channel for a database or network layer to honour.
func (s *scheduler) scheduleRequests(reqs chan uint64, dist chan *request, pend chan uint64, quit chan struct{}, wg *sync.WaitGroup) {
// Clean up the goroutine and pipeline when done
defer wg.Done()
defer close(pend)
// Keep reading and scheduling section requests
for {
select {
case <-quit:
return
case section, ok := <-reqs:
// New section retrieval requested
if !ok {
return
}
// Deduplicate retrieval requests
unique := false
s.lock.Lock()
if s.responses[section] == nil {
s.responses[section] = &response{
done: make(chan struct{}),
}
unique = true
}
s.lock.Unlock()
// Schedule the section for retrieval and notify the deliverer to expect this section
if unique {
select {
case <-quit:
return
case dist <- &request{bit: s.bit, section: section}:
}
}
select {
case <-quit:
return
case pend <- section:
}
}
}
}
// scheduleDeliveries reads section acceptance notifications and waits for them
// to be delivered, pushing them into the output data buffer.
func (s *scheduler) scheduleDeliveries(pend chan uint64, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) {
// Clean up the goroutine and pipeline when done
defer wg.Done()
defer close(done)
// Keep reading notifications and scheduling deliveries
for {
select {
case <-quit:
return
case idx, ok := <-pend:
// New section retrieval pending
if !ok {
return
}
// Wait until the request is honoured
s.lock.Lock()
res := s.responses[idx]
s.lock.Unlock()
select {
case <-quit:
return
case <-res.done:
}
// Deliver the result
select {
case <-quit:
return
case done <- res.cached:
}
}
}
}
// deliver is called by the request distributor when a reply to a request arrives.
func (s *scheduler) deliver(sections []uint64, data [][]byte) {
s.lock.Lock()
defer s.lock.Unlock()
for i, section := range sections {
if res := s.responses[section]; res != nil && res.cached == nil { // Avoid non-requests and double deliveries
res.cached = data[i]
close(res.done)
}
}
}

103
core/bloombits/scheduler_test.go
View file

@ -1,103 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package bloombits
import (
"bytes"
"math/big"
"sync"
"sync/atomic"
"testing"
)
// Tests that the scheduler can deduplicate and forward retrieval requests to
// underlying fetchers and serve responses back, irrelevant of the concurrency
// of the requesting clients or serving data fetchers.
func TestSchedulerSingleClientSingleFetcher(t *testing.T) { testScheduler(t, 1, 1, 5000) }
func TestSchedulerSingleClientMultiFetcher(t *testing.T) { testScheduler(t, 1, 10, 5000) }
func TestSchedulerMultiClientSingleFetcher(t *testing.T) { testScheduler(t, 10, 1, 5000) }
func TestSchedulerMultiClientMultiFetcher(t *testing.T) { testScheduler(t, 10, 10, 5000) }
func testScheduler(t *testing.T, clients int, fetchers int, requests int) {
t.Parallel()
f := newScheduler(0)
// Create a batch of handler goroutines that respond to bloom bit requests and
// deliver them to the scheduler.
var fetchPend sync.WaitGroup
fetchPend.Add(fetchers)
defer fetchPend.Wait()
fetch := make(chan *request, 16)
defer close(fetch)
var delivered atomic.Uint32
for i := 0; i < fetchers; i++ {
go func() {
defer fetchPend.Done()
for req := range fetch {
delivered.Add(1)
f.deliver([]uint64{
req.section + uint64(requests), // Non-requested data (ensure it doesn't go out of bounds)
req.section, // Requested data
req.section, // Duplicated data (ensure it doesn't double close anything)
}, [][]byte{
{},
new(big.Int).SetUint64(req.section).Bytes(),
new(big.Int).SetUint64(req.section).Bytes(),
})
}
}()
}
// Start a batch of goroutines to concurrently run scheduling tasks
quit := make(chan struct{})
var pend sync.WaitGroup
pend.Add(clients)
for i := 0; i < clients; i++ {
go func() {
defer pend.Done()
in := make(chan uint64, 16)
out := make(chan []byte, 16)
f.run(in, fetch, out, quit, &pend)
go func() {
for j := 0; j < requests; j++ {
in <- uint64(j)
}
close(in)
}()
b := new(big.Int)
for j := 0; j < requests; j++ {
bits := <-out
if want := b.SetUint64(uint64(j)).Bytes(); !bytes.Equal(bits, want) {
t.Errorf("vector %d: delivered content mismatch: have %x, want %x", j, bits, want)
}
}
}()
}
pend.Wait()
if have := delivered.Load(); int(have) != requests {
t.Errorf("request count mismatch: have %v, want %v", have, requests)
}
}

522
core/chain_indexer.go
View file

@ -1,522 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package core
import (
"context"
"encoding/binary"
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
)
// ChainIndexerBackend defines the methods needed to process chain segments in
// the background and write the segment results into the database. These can be
// used to create filter blooms or CHTs.
type ChainIndexerBackend interface {
// Reset initiates the processing of a new chain segment, potentially terminating
// any partially completed operations (in case of a reorg).
Reset(ctx context.Context, section uint64, prevHead common.Hash) error
// Process crunches through the next header in the chain segment. The caller
// will ensure a sequential order of headers.
Process(ctx context.Context, header *types.Header) error
// Commit finalizes the section metadata and stores it into the database.
Commit() error
// Prune deletes the chain index older than the given threshold.
Prune(threshold uint64) error
}
// ChainIndexerChain interface is used for connecting the indexer to a blockchain
type ChainIndexerChain interface {
// CurrentHeader retrieves the latest locally known header.
CurrentHeader() *types.Header
// SubscribeChainHeadEvent subscribes to new head header notifications.
SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription
}
// ChainIndexer does a post-processing job for equally sized sections of the
// canonical chain (like BlooomBits and CHT structures). A ChainIndexer is
// connected to the blockchain through the event system by starting a
// ChainHeadEventLoop in a goroutine.
//
// Further child ChainIndexers can be added which use the output of the parent
// section indexer. These child indexers receive new head notifications only
// after an entire section has been finished or in case of rollbacks that might
// affect already finished sections.
type ChainIndexer struct {
chainDb ethdb.Database // Chain database to index the data from
indexDb ethdb.Database // Prefixed table-view of the db to write index metadata into
backend ChainIndexerBackend // Background processor generating the index data content
children []*ChainIndexer // Child indexers to cascade chain updates to
active atomic.Bool // Flag whether the event loop was started
update chan struct{} // Notification channel that headers should be processed
quit chan chan error // Quit channel to tear down running goroutines
ctx context.Context
ctxCancel func()
sectionSize uint64 // Number of blocks in a single chain segment to process
confirmsReq uint64 // Number of confirmations before processing a completed segment
storedSections uint64 // Number of sections successfully indexed into the database
knownSections uint64 // Number of sections known to be complete (block wise)
cascadedHead uint64 // Block number of the last completed section cascaded to subindexers
checkpointSections uint64 // Number of sections covered by the checkpoint
checkpointHead common.Hash // Section head belonging to the checkpoint
throttling time.Duration // Disk throttling to prevent a heavy upgrade from hogging resources
log log.Logger
lock sync.Mutex
}
// NewChainIndexer creates a new chain indexer to do background processing on
// chain segments of a given size after certain number of confirmations passed.
// The throttling parameter might be used to prevent database thrashing.
func NewChainIndexer(chainDb ethdb.Database, indexDb ethdb.Database, backend ChainIndexerBackend, section, confirm uint64, throttling time.Duration, kind string) *ChainIndexer {
c := &ChainIndexer{
chainDb: chainDb,
indexDb: indexDb,
backend: backend,
update: make(chan struct{}, 1),
quit: make(chan chan error),
sectionSize: section,
confirmsReq: confirm,
throttling: throttling,
log: log.New("type", kind),
}
// Initialize database dependent fields and start the updater
c.loadValidSections()
c.ctx, c.ctxCancel = context.WithCancel(context.Background())
go c.updateLoop()
return c
}
// AddCheckpoint adds a checkpoint. Sections are never processed and the chain
// is not expected to be available before this point. The indexer assumes that
// the backend has sufficient information available to process subsequent sections.
//
// Note: knownSections == 0 and storedSections == checkpointSections until
// syncing reaches the checkpoint
func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) {
c.lock.Lock()
defer c.lock.Unlock()
// Short circuit if the given checkpoint is below than local's.
if c.checkpointSections >= section+1 || section < c.storedSections {
return
}
c.checkpointSections = section + 1
c.checkpointHead = shead
c.setSectionHead(section, shead)
c.setValidSections(section + 1)
}
// Start creates a goroutine to feed chain head events into the indexer for
// cascading background processing. Children do not need to be started, they
// are notified about new events by their parents.
func (c *ChainIndexer) Start(chain ChainIndexerChain) {
events := make(chan ChainHeadEvent, 10)
sub := chain.SubscribeChainHeadEvent(events)
go c.eventLoop(chain.CurrentHeader(), events, sub)
}
// Close tears down all goroutines belonging to the indexer and returns any error
// that might have occurred internally.
func (c *ChainIndexer) Close() error {
var errs []error
c.ctxCancel()
// Tear down the primary update loop
errc := make(chan error)
c.quit <- errc
if err := <-errc; err != nil {
errs = append(errs, err)
}
// If needed, tear down the secondary event loop
if c.active.Load() {
c.quit <- errc
if err := <-errc; err != nil {
errs = append(errs, err)
}
}
// Close all children
for _, child := range c.children {
if err := child.Close(); err != nil {
errs = append(errs, err)
}
}
// Return any failures
switch {
case len(errs) == 0:
return nil
case len(errs) == 1:
return errs[0]
default:
return fmt.Errorf("%v", errs)
}
}
// eventLoop is a secondary - optional - event loop of the indexer which is only
// started for the outermost indexer to push chain head events into a processing
// queue.
func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainHeadEvent, sub event.Subscription) {
// Mark the chain indexer as active, requiring an additional teardown
c.active.Store(true)
defer sub.Unsubscribe()
// Fire the initial new head event to start any outstanding processing
c.newHead(currentHeader.Number.Uint64(), false)
var (
prevHeader = currentHeader
prevHash = currentHeader.Hash()
)
for {
select {
case errc := <-c.quit:
// Chain indexer terminating, report no failure and abort
errc <- nil
return
case ev, ok := <-events:
// Received a new event, ensure it's not nil (closing) and update
if !ok {
errc := <-c.quit
errc <- nil
return
}
if ev.Header.ParentHash != prevHash {
// Reorg to the common ancestor if needed (might not exist in light sync mode, skip reorg then)
// TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly?
if rawdb.ReadCanonicalHash(c.chainDb, prevHeader.Number.Uint64()) != prevHash {
if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, ev.Header); h != nil {
c.newHead(h.Number.Uint64(), true)
}
}
}
c.newHead(ev.Header.Number.Uint64(), false)
prevHeader, prevHash = ev.Header, ev.Header.Hash()
}
}
}
// newHead notifies the indexer about new chain heads and/or reorgs.
func (c *ChainIndexer) newHead(head uint64, reorg bool) {
c.lock.Lock()
defer c.lock.Unlock()
// If a reorg happened, invalidate all sections until that point
if reorg {
// Revert the known section number to the reorg point
known := (head + 1) / c.sectionSize
stored := known
if known < c.checkpointSections {
known = 0
}
if stored < c.checkpointSections {
stored = c.checkpointSections
}
if known < c.knownSections {
c.knownSections = known
}
// Revert the stored sections from the database to the reorg point
if stored < c.storedSections {
c.setValidSections(stored)
}
// Update the new head number to the finalized section end and notify children
head = known * c.sectionSize
if head < c.cascadedHead {
c.cascadedHead = head
for _, child := range c.children {
child.newHead(c.cascadedHead, true)
}
}
return
}
// No reorg, calculate the number of newly known sections and update if high enough
var sections uint64
if head >= c.confirmsReq {
sections = (head + 1 - c.confirmsReq) / c.sectionSize
if sections < c.checkpointSections {
sections = 0
}
if sections > c.knownSections {
if c.knownSections < c.checkpointSections {
// syncing reached the checkpoint, verify section head
syncedHead := rawdb.ReadCanonicalHash(c.chainDb, c.checkpointSections*c.sectionSize-1)
if syncedHead != c.checkpointHead {
c.log.Error("Synced chain does not match checkpoint", "number", c.checkpointSections*c.sectionSize-1, "expected", c.checkpointHead, "synced", syncedHead)
return
}
}
c.knownSections = sections
select {
case c.update <- struct{}{}:
default:
}
}
}
}
// updateLoop is the main event loop of the indexer which pushes chain segments
// down into the processing backend.
func (c *ChainIndexer) updateLoop() {
var (
updating bool
updated time.Time
)
for {
select {
case errc := <-c.quit:
// Chain indexer terminating, report no failure and abort
errc <- nil
return
case <-c.update:
// Section headers completed (or rolled back), update the index
c.lock.Lock()
if c.knownSections > c.storedSections {
// Periodically print an upgrade log message to the user
if time.Since(updated) > 8*time.Second {
if c.knownSections > c.storedSections+1 {
updating = true
c.log.Info("Upgrading chain index", "percentage", c.storedSections*100/c.knownSections)
}
updated = time.Now()
}
// Cache the current section count and head to allow unlocking the mutex
c.verifyLastHead()
section := c.storedSections
var oldHead common.Hash
if section > 0 {
oldHead = c.SectionHead(section - 1)
}
// Process the newly defined section in the background
c.lock.Unlock()
newHead, err := c.processSection(section, oldHead)
if err != nil {
select {
case <-c.ctx.Done():
<-c.quit <- nil
return
default:
}
c.log.Error("Section processing failed", "error", err)
}
c.lock.Lock()
// If processing succeeded and no reorgs occurred, mark the section completed
if err == nil && (section == 0 || oldHead == c.SectionHead(section-1)) {
c.setSectionHead(section, newHead)
c.setValidSections(section + 1)
if c.storedSections == c.knownSections && updating {
updating = false
c.log.Info("Finished upgrading chain index")
}
c.cascadedHead = c.storedSections*c.sectionSize - 1
for _, child := range c.children {
c.log.Trace("Cascading chain index update", "head", c.cascadedHead)
child.newHead(c.cascadedHead, false)
}
} else {
// If processing failed, don't retry until further notification
c.log.Debug("Chain index processing failed", "section", section, "err", err)
c.verifyLastHead()
c.knownSections = c.storedSections
}
}
// If there are still further sections to process, reschedule
if c.knownSections > c.storedSections {
time.AfterFunc(c.throttling, func() {
select {
case c.update <- struct{}{}:
default:
}
})
}
c.lock.Unlock()
}
}
}
// processSection processes an entire section by calling backend functions while
// ensuring the continuity of the passed headers. Since the chain mutex is not
// held while processing, the continuity can be broken by a long reorg, in which
// case the function returns with an error.
func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (common.Hash, error) {
c.log.Trace("Processing new chain section", "section", section)
// Reset and partial processing
if err := c.backend.Reset(c.ctx, section, lastHead); err != nil {
c.setValidSections(0)
return common.Hash{}, err
}
for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ {
hash := rawdb.ReadCanonicalHash(c.chainDb, number)
if hash == (common.Hash{}) {
return common.Hash{}, fmt.Errorf("canonical block #%d unknown", number)
}
header := rawdb.ReadHeader(c.chainDb, hash, number)
if header == nil {
return common.Hash{}, fmt.Errorf("block #%d [%x..] not found", number, hash[:4])
} else if header.ParentHash != lastHead {
return common.Hash{}, errors.New("chain reorged during section processing")
}
if err := c.backend.Process(c.ctx, header); err != nil {
return common.Hash{}, err
}
lastHead = header.Hash()
}
if err := c.backend.Commit(); err != nil {
return common.Hash{}, err
}
return lastHead, nil
}
// verifyLastHead compares last stored section head with the corresponding block hash in the
// actual canonical chain and rolls back reorged sections if necessary to ensure that stored
// sections are all valid
func (c *ChainIndexer) verifyLastHead() {
for c.storedSections > 0 && c.storedSections > c.checkpointSections {
if c.SectionHead(c.storedSections-1) == rawdb.ReadCanonicalHash(c.chainDb, c.storedSections*c.sectionSize-1) {
return
}
c.setValidSections(c.storedSections - 1)
}
}
// Sections returns the number of processed sections maintained by the indexer
// and also the information about the last header indexed for potential canonical
// verifications.
func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) {
c.lock.Lock()
defer c.lock.Unlock()
c.verifyLastHead()
return c.storedSections, c.storedSections*c.sectionSize - 1, c.SectionHead(c.storedSections - 1)
}
// AddChildIndexer adds a child ChainIndexer that can use the output of this one
func (c *ChainIndexer) AddChildIndexer(indexer *ChainIndexer) {
if indexer == c {
panic("can't add indexer as a child of itself")
}
c.lock.Lock()
defer c.lock.Unlock()
c.children = append(c.children, indexer)
// Cascade any pending updates to new children too
sections := c.storedSections
if c.knownSections < sections {
// if a section is "stored" but not "known" then it is a checkpoint without
// available chain data so we should not cascade it yet
sections = c.knownSections
}
if sections > 0 {
indexer.newHead(sections*c.sectionSize-1, false)
}
}
// Prune deletes all chain data older than given threshold.
func (c *ChainIndexer) Prune(threshold uint64) error {
return c.backend.Prune(threshold)
}
// loadValidSections reads the number of valid sections from the index database
// and caches is into the local state.
func (c *ChainIndexer) loadValidSections() {
data, _ := c.indexDb.Get([]byte("count"))
if len(data) == 8 {
c.storedSections = binary.BigEndian.Uint64(data)
}
}
// setValidSections writes the number of valid sections to the index database
func (c *ChainIndexer) setValidSections(sections uint64) {
// Set the current number of valid sections in the database
var data [8]byte
binary.BigEndian.PutUint64(data[:], sections)
c.indexDb.Put([]byte("count"), data[:])
// Remove any reorged sections, caching the valids in the mean time
for c.storedSections > sections {
c.storedSections--
c.removeSectionHead(c.storedSections)
}
c.storedSections = sections // needed if new > old
}
// SectionHead retrieves the last block hash of a processed section from the
// index database.
func (c *ChainIndexer) SectionHead(section uint64) common.Hash {
var data [8]byte
binary.BigEndian.PutUint64(data[:], section)
hash, _ := c.indexDb.Get(append([]byte("shead"), data[:]...))
if len(hash) == len(common.Hash{}) {
return common.BytesToHash(hash)
}
return common.Hash{}
}
// setSectionHead writes the last block hash of a processed section to the index
// database.
func (c *ChainIndexer) setSectionHead(section uint64, hash common.Hash) {
var data [8]byte
binary.BigEndian.PutUint64(data[:], section)
c.indexDb.Put(append([]byte("shead"), data[:]...), hash.Bytes())
}
// removeSectionHead removes the reference to a processed section from the index
// database.
func (c *ChainIndexer) removeSectionHead(section uint64) {
var data [8]byte
binary.BigEndian.PutUint64(data[:], section)
c.indexDb.Delete(append([]byte("shead"), data[:]...))
}

246
core/chain_indexer_test.go
View file

@ -1,246 +0,0 @@
// Copyright 2017 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package core
import (
"context"
"errors"
"fmt"
"math/big"
"math/rand"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
)
// Runs multiple tests with randomized parameters.
func TestChainIndexerSingle(t *testing.T) {
for i := 0; i < 10; i++ {
testChainIndexer(t, 1)
}
}
// Runs multiple tests with randomized parameters and different number of
// chain backends.
func TestChainIndexerWithChildren(t *testing.T) {
for i := 2; i < 8; i++ {
testChainIndexer(t, i)
}
}
// testChainIndexer runs a test with either a single chain indexer or a chain of
// multiple backends. The section size and required confirmation count parameters
// are randomized.
func testChainIndexer(t *testing.T, count int) {
db := rawdb.NewMemoryDatabase()
defer db.Close()
// Create a chain of indexers and ensure they all report empty
backends := make([]*testChainIndexBackend, count)
for i := 0; i < count; i++ {
var (
sectionSize = uint64(rand.Intn(100) + 1)
confirmsReq = uint64(rand.Intn(10))
)
backends[i] = &testChainIndexBackend{t: t, processCh: make(chan uint64)}
backends[i].indexer = NewChainIndexer(db, rawdb.NewTable(db, string([]byte{byte(i)})), backends[i], sectionSize, confirmsReq, 0, fmt.Sprintf("indexer-%d", i))
if sections, _, _ := backends[i].indexer.Sections(); sections != 0 {
t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, 0)
}
if i > 0 {
backends[i-1].indexer.AddChildIndexer(backends[i].indexer)
}
}
defer backends[0].indexer.Close() // parent indexer shuts down children
// notify pings the root indexer about a new head or reorg, then expect
// processed blocks if a section is processable
notify := func(headNum, failNum uint64, reorg bool) {
backends[0].indexer.newHead(headNum, reorg)
if reorg {
for _, backend := range backends {
headNum = backend.reorg(headNum)
backend.assertSections()
}
return
}
var cascade bool
for _, backend := range backends {
headNum, cascade = backend.assertBlocks(headNum, failNum)
if !cascade {
break
}
backend.assertSections()
}
}
// inject inserts a new random canonical header into the database directly
inject := func(number uint64) {
header := &types.Header{Number: big.NewInt(int64(number)), Extra: big.NewInt(rand.Int63()).Bytes()}
if number > 0 {
header.ParentHash = rawdb.ReadCanonicalHash(db, number-1)
}
rawdb.WriteHeader(db, header)
rawdb.WriteCanonicalHash(db, header.Hash(), number)
}
// Start indexer with an already existing chain
for i := uint64(0); i <= 100; i++ {
inject(i)
}
notify(100, 100, false)
// Add new blocks one by one
for i := uint64(101); i <= 1000; i++ {
inject(i)
notify(i, i, false)
}
// Do a reorg
notify(500, 500, true)
// Create new fork
for i := uint64(501); i <= 1000; i++ {
inject(i)
notify(i, i, false)
}
for i := uint64(1001); i <= 1500; i++ {
inject(i)
}
// Failed processing scenario where less blocks are available than notified
notify(2000, 1500, false)
// Notify about a reorg (which could have caused the missing blocks if happened during processing)
notify(1500, 1500, true)
// Create new fork
for i := uint64(1501); i <= 2000; i++ {
inject(i)
notify(i, i, false)
}
}
// testChainIndexBackend implements ChainIndexerBackend
type testChainIndexBackend struct {
t *testing.T
indexer *ChainIndexer
section, headerCnt, stored uint64
processCh chan uint64
}
// assertSections verifies if a chain indexer has the correct number of section.
func (b *testChainIndexBackend) assertSections() {
// Keep trying for 3 seconds if it does not match
var sections uint64
for i := 0; i < 300; i++ {
sections, _, _ = b.indexer.Sections()
if sections == b.stored {
return
}
time.Sleep(10 * time.Millisecond)
}
b.t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, b.stored)
}
// assertBlocks expects processing calls after new blocks have arrived. If the
// failNum < headNum then we are simulating a scenario where a reorg has happened
// after the processing has started and the processing of a section fails.
func (b *testChainIndexBackend) assertBlocks(headNum, failNum uint64) (uint64, bool) {
var sections uint64
if headNum >= b.indexer.confirmsReq {
sections = (headNum + 1 - b.indexer.confirmsReq) / b.indexer.sectionSize
if sections > b.stored {
// expect processed blocks
for expectd := b.stored * b.indexer.sectionSize; expectd < sections*b.indexer.sectionSize; expectd++ {
if expectd > failNum {
// rolled back after processing started, no more process calls expected
// wait until updating is done to make sure that processing actually fails
var updating bool
for i := 0; i < 300; i++ {
b.indexer.lock.Lock()
updating = b.indexer.knownSections > b.indexer.storedSections
b.indexer.lock.Unlock()
if !updating {
break
}
time.Sleep(10 * time.Millisecond)
}
if updating {
b.t.Fatalf("update did not finish")
}
sections = expectd / b.indexer.sectionSize
break
}
select {
case <-time.After(10 * time.Second):
b.t.Fatalf("Expected processed block #%d, got nothing", expectd)
case processed := <-b.processCh:
if processed != expectd {
b.t.Errorf("Expected processed block #%d, got #%d", expectd, processed)
}
}
}
b.stored = sections
}
}
if b.stored == 0 {
return 0, false
}
return b.stored*b.indexer.sectionSize - 1, true
}
func (b *testChainIndexBackend) reorg(headNum uint64) uint64 {
firstChanged := (headNum + 1) / b.indexer.sectionSize
if firstChanged < b.stored {
b.stored = firstChanged
}
return b.stored * b.indexer.sectionSize
}
func (b *testChainIndexBackend) Reset(ctx context.Context, section uint64, prevHead common.Hash) error {
b.section = section
b.headerCnt = 0
return nil
}
func (b *testChainIndexBackend) Process(ctx context.Context, header *types.Header) error {
b.headerCnt++
if b.headerCnt > b.indexer.sectionSize {
b.t.Error("Processing too many headers")
}
//t.processCh <- header.Number.Uint64()
select {
case <-time.After(10 * time.Second):
b.t.Error("Unexpected call to Process")
// Can't use Fatal since this is not the test's goroutine.
// Returning error stops the chainIndexer's updateLoop
return errors.New("unexpected call to Process")
case b.processCh <- header.Number.Uint64():
}
return nil
}
func (b *testChainIndexBackend) Commit() error {
if b.headerCnt != b.indexer.sectionSize {
b.t.Error("Not enough headers processed")
}
return nil
}
func (b *testChainIndexBackend) Prune(threshold uint64) error {
return nil
}

15
core/chain_makers.go
View file

@ -328,9 +328,13 @@ func (b *BlockGen) collectRequests(readonly bool) (requests [][]byte) {
blockContext := NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase)
evm := vm.NewEVM(blockContext, statedb, b.cm.config, vm.Config{})
// EIP-7002
ProcessWithdrawalQueue(&requests, evm)
if err := ProcessWithdrawalQueue(&requests, evm); err != nil {
panic(fmt.Sprintf("could not process withdrawal requests: %v", err))
}
// EIP-7251
ProcessConsolidationQueue(&requests, evm)
if err := ProcessConsolidationQueue(&requests, evm); err != nil {
panic(fmt.Sprintf("could not process consolidation requests: %v", err))
}
}
return requests
}
@ -504,6 +508,13 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
if gen != nil {
gen(i, b)
}
requests := b.collectRequests(false)
if requests != nil {
reqHash := types.CalcRequestsHash(requests)
b.header.RequestsHash = &reqHash
}
body := &types.Body{
Transactions: b.txs,
Uncles: b.uncles,

2
core/error.go
View file

@ -28,8 +28,6 @@ var (
// ErrNoGenesis is returned when there is no Genesis Block.
ErrNoGenesis = errors.New("genesis not found in chain")
errSideChainReceipts = errors.New("side blocks can't be accepted as ancient chain data")
)
// List of evm-call-message pre-checking errors. All state transition messages will

81
core/filtermaps/chain_view.go
View file

@ -17,6 +17,8 @@
package filtermaps
import (
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
@ -27,6 +29,7 @@ type blockchain interface {
GetHeader(hash common.Hash, number uint64) *types.Header
GetCanonicalHash(number uint64) common.Hash
GetReceiptsByHash(hash common.Hash) types.Receipts
GetRawReceipts(hash common.Hash, number uint64) types.Receipts
}
// ChainView represents an immutable view of a chain with a block id and a set
@ -39,6 +42,7 @@ type blockchain interface {
// of the underlying blockchain, it should only possess the block headers
// and receipts up until the expected chain view head.
type ChainView struct {
lock sync.Mutex
chain blockchain
headNumber uint64
hashes []common.Hash // block hashes starting backwards from headNumber until first canonical hash
@ -55,43 +59,80 @@ func NewChainView(chain blockchain, number uint64, hash common.Hash) *ChainView
return cv
}
// getBlockHash returns the block hash belonging to the given block number.
// HeadNumber returns the head block number of the chain view.
func (cv *ChainView) HeadNumber() uint64 {
return cv.headNumber
}
// BlockHash returns the block hash belonging to the given block number.
// Note that the hash of the head block is not returned because ChainView might
// represent a view where the head block is currently being created.
func (cv *ChainView) getBlockHash(number uint64) common.Hash {
if number >= cv.headNumber {
func (cv *ChainView) BlockHash(number uint64) common.Hash {
cv.lock.Lock()
defer cv.lock.Unlock()
if number > cv.headNumber {
panic("invalid block number")
}
return cv.blockHash(number)
}
// getBlockId returns the unique block id belonging to the given block number.
// BlockId returns the unique block id belonging to the given block number.
// Note that it is currently equal to the block hash. In the future it might
// be a different id for future blocks if the log index root becomes part of
// consensus and therefore rendering the index with the new head will happen
// before the hash of that new head is available.
func (cv *ChainView) getBlockId(number uint64) common.Hash {
func (cv *ChainView) BlockId(number uint64) common.Hash {
cv.lock.Lock()
defer cv.lock.Unlock()
if number > cv.headNumber {
panic("invalid block number")
}
return cv.blockHash(number)
}
// getReceipts returns the set of receipts belonging to the block at the given
// block number.
func (cv *ChainView) getReceipts(number uint64) types.Receipts {
if number > cv.headNumber {
panic("invalid block number")
}
return cv.chain.GetReceiptsByHash(cv.blockHash(number))
// Header returns the block header at the given block number.
func (cv *ChainView) Header(number uint64) *types.Header {
return cv.chain.GetHeader(cv.BlockHash(number), number)
}
// limitedView returns a new chain view that is a truncated version of the parent view.
func (cv *ChainView) limitedView(newHead uint64) *ChainView {
if newHead >= cv.headNumber {
return cv
// Receipts returns the set of receipts belonging to the block at the given
// block number.
func (cv *ChainView) Receipts(number uint64) types.Receipts {
blockHash := cv.BlockHash(number)
if blockHash == (common.Hash{}) {
log.Error("Chain view: block hash unavailable", "number", number, "head", cv.headNumber)
return nil
}
return NewChainView(cv.chain, newHead, cv.blockHash(newHead))
return cv.chain.GetReceiptsByHash(blockHash)
}
// RawReceipts returns the set of receipts belonging to the block at the given
// block number. Does not derive the fields of the receipts, should only be
// used during creation of the filter maps, please use cv.Receipts during querying.
func (cv *ChainView) RawReceipts(number uint64) types.Receipts {
blockHash := cv.BlockHash(number)
if blockHash == (common.Hash{}) {
log.Error("Chain view: block hash unavailable", "number", number, "head", cv.headNumber)
return nil
}
return cv.chain.GetRawReceipts(blockHash, number)
}
// SharedRange returns the block range shared by two chain views.
func (cv *ChainView) SharedRange(cv2 *ChainView) common.Range[uint64] {
cv.lock.Lock()
defer cv.lock.Unlock()
if cv == nil || cv2 == nil || !cv.extendNonCanonical() || !cv2.extendNonCanonical() {
return common.Range[uint64]{}
}
var sharedLen uint64
for n := min(cv.headNumber+1-uint64(len(cv.hashes)), cv2.headNumber+1-uint64(len(cv2.hashes))); n <= cv.headNumber && n <= cv2.headNumber && cv.blockHash(n) == cv2.blockHash(n); n++ {
sharedLen = n + 1
}
return common.NewRange(0, sharedLen)
}
// equalViews returns true if the two chain views are equivalent.
@ -99,7 +140,7 @@ func equalViews(cv1, cv2 *ChainView) bool {
if cv1 == nil || cv2 == nil {
return false
}
return cv1.headNumber == cv2.headNumber && cv1.getBlockId(cv1.headNumber) == cv2.getBlockId(cv2.headNumber)
return cv1.headNumber == cv2.headNumber && cv1.BlockId(cv1.headNumber) == cv2.BlockId(cv2.headNumber)
}
// matchViews returns true if the two chain views are equivalent up until the
@ -113,9 +154,9 @@ func matchViews(cv1, cv2 *ChainView, number uint64) bool {
return false
}
if number == cv1.headNumber || number == cv2.headNumber {
return cv1.getBlockId(number) == cv2.getBlockId(number)
return cv1.BlockId(number) == cv2.BlockId(number)
}
return cv1.getBlockHash(number) == cv2.getBlockHash(number)
return cv1.BlockHash(number) == cv2.BlockHash(number)
}
// extendNonCanonical checks whether the previously known reverse list of head

3
core/filtermaps/checkpoints_holesky.json
View file

@ -17,5 +17,6 @@
{"blockNumber": 3101669, "blockId": "0xa6e66a18fed64d33178c7088f273c404be597662c34f6e6884cf2e24f3ea4ace", "firstIndex": 1073741353},
{"blockNumber": 3221725, "blockId": "0xe771f897dece48b1583cc1d1d10de8015da57407eb1fdf239fdbe46eaab85143", "firstIndex": 1140850137},
{"blockNumber": 3357164, "blockId": "0x6252d0aa54c79623b0680069c88d7b5c47983f0d5c4845b6c811b8d9b5e8ff3c", "firstIndex": 1207959453},
{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542}
{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542},
{"blockNumber": 3546397, "blockId": "0xdabdef7defa4281180a57c5af121877b82274f15ccf074ea0096146f4c246df2", "firstIndex": 1342176778}
]

9
core/filtermaps/checkpoints_mainnet.json
View file

@ -260,5 +260,12 @@
{"blockNumber": 21959188, "blockId": "0x6b9c482cc4822af2ad62a359b923062556ab6a046d1a39a8243b764848690601", "firstIndex": 17381193886},
{"blockNumber": 21994911, "blockId": "0x0d99ef9dd42dd1c62fc5249eb4f618e7672ab93fa0ba7545c77360371cf972e5", "firstIndex": 17448302795},
{"blockNumber": 22026007, "blockId": "0xf7cdc7f6694f2c2ed31fa8a607f65cfa59d0dd7d7424ab5c017f959ae2982c71", "firstIndex": 17515413036},
{"blockNumber": 22059890, "blockId": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768}
{"blockNumber": 22059890, "blockId": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768},
{"blockNumber": 22090784, "blockId": "0xf97c2eaf9a550360ac24000c0ff17ffa388a2bdd6f73f2f36718e332edfa107a", "firstIndex": 17649630983},
{"blockNumber": 22121157, "blockId": "0xa790025235db782e899f23d8b09663ec2d74ec149e4125d62989f98829b08e2d", "firstIndex": 17716724973},
{"blockNumber": 22148056, "blockId": "0xbe25ac4f1bdd89a7db5782bf1157e6c4378d80220d67a029397853ef16cd1a4c", "firstIndex": 17783838366},
{"blockNumber": 22168652, "blockId": "0x6ae43618c915e636794e2cc2d75dde9992766881c7405fe6479c045ed4bee57e", "firstIndex": 17850956277},
{"blockNumber": 22190975, "blockId": "0x9437121647899a4b7b84d67fbea7cc6ff967481c2eab4328ccd86e2cefe19420", "firstIndex": 17918066140},
{"blockNumber": 22234357, "blockId": "0x036030830134f9224160d5a0b62da35ec7813dc8855d554bd22e9d38545243ed", "firstIndex": 17985175075},
{"blockNumber": 22276736, "blockId": "0x5ceb96d98aa1b4c1c2f2fa253ae9cdb1b04e0420c11bf795400e8762c0a1635c", "firstIndex": 18052284344}
]

6
core/filtermaps/checkpoints_sepolia.json
View file

@ -64,5 +64,9 @@
{"blockNumber": 7730079, "blockId": "0x08e0d511a93e2a9c004b3456d77d687ae86247417ada1cdd1f85332a62dfed1d", "firstIndex": 4227846795},
{"blockNumber": 7777515, "blockId": "0xeba8faed2e12bb9ed5e7fad19dd82662f15f6f4f512a0d232eedf1e676712428", "firstIndex": 4294966082},
{"blockNumber": 7828860, "blockId": "0x62c054bd24be351dc4777460ee922a75fdcea5c9830455cbac6fa29552719c85", "firstIndex": 4362076087},
{"blockNumber": 7869890, "blockId": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267}
{"blockNumber": 7869890, "blockId": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267},
{"blockNumber": 7911722, "blockId": "0x9a85e48e3135c97c51fc1786f2af0596c802e021b6c53cfca65a129cafcd23ed", "firstIndex": 4496287265},
{"blockNumber": 7960147, "blockId": "0xc9359cc76d7090e1c8a031108f0ab7a8935d971efd4325fe53612a1d99562f6f", "firstIndex": 4563402388},
{"blockNumber": 8030418, "blockId": "0x21867e68cd8327aed2da2601399d60f7f9e41dca4a4f2f9be982e5a2b9304a88", "firstIndex": 4630511616},
{"blockNumber": 8087701, "blockId": "0x0fa8c8d7549cc9a8d308262706fe248efe759f8b63511efb1e7f3926e9af2dcb", "firstIndex": 4697614758}
]

Some files were not shown because too many files have changed in this diff Show more