resolve merge conflict

.gitea/workflows/release-azure-cleanup.yml

@@ -0,0 +1,22 @@
+on:
+  schedule:
+    - cron: '0 14 * * *'
+  workflow_dispatch:
+
+jobs:
+  azure-cleanup:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.24
+        cache: false
+
+    - name: Run cleanup script
+      run: |
+        go run build/ci.go purge -store gethstore/builds -days 14
+      env:
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}

.gitea/workflows/release-ppa.yml

@@ -0,0 +1,36 @@
+on:
+  schedule:
+    - cron: '0 15 * * *'
+  push:
+    branches:
+      - "release/*"
+  workflow_dispatch:
+
+jobs:
+  ppa:
+    name: PPA Upload
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.24
+        cache: false
+
+    - name: Install deb toolchain
+      run: |
+        apt-get update
+        apt-get -yq --no-install-suggests --no-install-recommends install devscripts debhelper dput fakeroot
+
+    - name: Add launchpad to known_hosts
+      run: |
+        echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
+
+    - name: Run ci.go
+      run: |
+        go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
+      env:
+        PPA_SIGNING_KEY: ${{ secrets.PPA_SIGNING_KEY }}
+        PPA_SSH_KEY: ${{ secrets.PPA_SSH_KEY }}

.gitea/workflows/release.yml

@@ -0,0 +1,147 @@
+on:
+  push:
+    branches:
+      - "master"
+    tags:
+      - "v*"
+
+jobs:
+  linux-intel:
+    name: Linux Build
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.24
+        cache: false
+
+    - name: Install cross toolchain
+      run: |
+        apt-get update
+        apt-get -yq --no-install-suggests --no-install-recommends install gcc-multilib
+
+    - name: Build (amd64)
+      run: |
+        go run build/ci.go install -arch amd64 -dlgo
+
+    - name: Create/upload archive (amd64)
+      run: |
+        go run build/ci.go archive -arch amd64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        rm -f build/bin/*
+      env:
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
+
+    - name: Build (386)
+      run: |
+        go run build/ci.go install -arch 386 -dlgo
+
+    - name: Create/upload archive (386)
+      run: |
+        go run build/ci.go archive -arch 386 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        rm -f build/bin/*
+      env:
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
+
+  linux-arm:
+    name: Linux Build (arm)
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.24
+        cache: false
+
+    - name: Install cross toolchain
+      run: |
+        apt-get update
+        apt-get -yq --no-install-suggests --no-install-recommends install gcc-arm-linux-gnueabi libc6-dev-armel-cross gcc-arm-linux-gnueabihf libc6-dev-armhf-cross gcc-aarch64-linux-gnu libc6-dev-arm64-cross
+        ln -s /usr/include/asm-generic /usr/include/asm
+
+    - name: Build (arm64)
+      run: |
+        go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
+
+    - name: Create/upload archive (arm64)
+      run: |
+        go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        rm -fr build/bin/*
+      env:
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
+
+    - name: Run build (arm5)
+      run: |
+        go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
+      env:
+        GOARM: "5"
+
+    - name: Create/upload archive (arm5)
+      run: |
+        go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+      env:
+        GOARM: "5"
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
+
+    - name: Run build (arm6)
+      run: |
+        go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
+      env:
+        GOARM: "6"
+
+    - name: Create/upload archive (arm6)
+      run: |
+        go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        rm -fr build/bin/*
+      env:
+        GOARM: "6"
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
+
+    - name: Run build (arm7)
+      run: |
+        go run build/ci.go install -dlgo -arch arm -cc arm-linux-gnueabi-gcc
+      env:
+        GOARM: "7"
+
+    - name: Create/upload archive (arm7)
+      run: |
+        go run build/ci.go archive -arch arm -type tar -signer LINUX_SIGNING_KEY -upload gethstore/builds
+        rm -fr build/bin/*
+      env:
+        GOARM: "7"
+        LINUX_SIGNING_KEY: ${{ secrets.LINUX_SIGNING_KEY }}
+        AZURE_BLOBSTORE_TOKEN: ${{ secrets.AZURE_BLOBSTORE_TOKEN }}
+
+  docker:
+    name: Docker Image
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: 1.24
+        cache: false
+
+    - name: Run docker build
+      env:
+        DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+        DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
+      run: |
+        go run build/ci.go dockerx -platform linux/amd64,linux/arm64,linux/riscv64 -upload

.github/CODEOWNERS

@@ -9,12 +9,11 @@ beacon/light/                   @zsfelfoldi
 beacon/merkle/                  @zsfelfoldi 
 beacon/types/                   @zsfelfoldi @fjl
 beacon/params/                  @zsfelfoldi @fjl
-cmd/clef/                       @holiman
+cmd/evm/                        @MariusVanDerWijden @lightclient
-cmd/evm/                        @holiman @MariusVanDerWijden @lightclient
+core/state/                     @rjl493456442
-core/state/                     @rjl493456442 @holiman
+crypto/                         @gballet @jwasinger @fjl
-crypto/                         @gballet @jwasinger @holiman @fjl
+core/                           @rjl493456442
-core/                           @holiman @rjl493456442
+eth/                            @rjl493456442
-eth/                            @holiman @rjl493456442
 eth/catalyst/                   @MariusVanDerWijden @lightclient @fjl @jwasinger
 eth/tracers/                    @s1na
 ethclient/                      @fjl
@@ -26,11 +25,9 @@ core/tracing/                   @s1na
 graphql/                        @s1na
 internal/ethapi/                @fjl @s1na @lightclient
 internal/era/                   @lightclient
-metrics/                        @holiman
+miner/                          @MariusVanDerWijden @fjl @rjl493456442
-miner/                          @MariusVanDerWijden @holiman @fjl @rjl493456442
 node/                           @fjl
 p2p/                            @fjl @zsfelfoldi
 rlp/                            @fjl
-params/                         @fjl @holiman @karalabe @gballet @rjl493456442 @zsfelfoldi
+params/                         @fjl @karalabe @gballet @rjl493456442 @zsfelfoldi
-rpc/                            @fjl @holiman
+rpc/                            @fjl
-signer/                         @holiman

.gitignore

@@ -43,3 +43,16 @@ profile.cov
 .vscode
 
 tests/spec-tests/
+
+# binaries
+cmd/abidump/abidump
+cmd/abigen/abigen
+cmd/blsync/blsync
+cmd/clef/clef
+cmd/devp2p/devp2p
+cmd/era/era
+cmd/ethkey/ethkey
+cmd/evm/evm
+cmd/geth/geth
+cmd/rlpdump/rlpdump
+cmd/workload/workload

.golangci.yml

@@ -1,31 +1,25 @@
 # This file configures github.com/golangci/golangci-lint.
-
+version: '2'
 run:
-  timeout: 20m
   tests: true
-
 linters:
-  disable-all: true
+  default: none
   enable:
-    - goimports
-    - gosimple
-    - govet
-    - ineffassign
-    - misspell
-    - unconvert
-    - typecheck
-    - unused
-    - staticcheck
     - bidichk
-    - durationcheck
     - copyloopvar
-    - whitespace
-    - revive # only certain checks enabled
     - durationcheck
     - gocheckcompilerdirectives
-    - reassign
+    - govet
+    - ineffassign
     - mirror
+    - misspell
+    - reassign
+    - revive # only certain checks enabled
+    - staticcheck
+    - unconvert
+    - unused
     - usetesting
+    - whitespace
     ### linters we tried and will not be using:
     ###
     # - structcheck # lots of false positives
@@ -36,10 +30,11 @@ linters:
     # - exhaustive # silly check
     # - makezero # false positives
     # - nilerr # several intentional
-
+  settings:
-linters-settings:
+    staticcheck:
-  gofmt:
+      checks:
-    simplify: true
+        # disable Quickfixes
+        - -QF1*
     revive:
       enable-all-rules: false
       # here we enable specific useful rules
@@ -48,22 +43,23 @@ linters-settings:
         - name: receiver-naming
           severity: warning
           disabled: false
-        exclude: [""]
+          exclude:
-
+            - ''
-issues:
+  exclusions:
-  # default is true. Enables skipping of directories:
+    generated: lax
-  #   vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+    presets:
-  exclude-dirs-use-default: true
+      - comments
-  exclude-files:
+      - common-false-positives
-    - core/genesis_alloc.go
+      - legacy
-  exclude-rules:
+      - std-error-handling
-    - path: crypto/bn256/cloudflare/optate.go
+    rules:
-      linters:
+      - linters:
           - deadcode
           - staticcheck
-    - path: crypto/bn256/
+        path: crypto/bn256/cloudflare/optate.go
-      linters:
+      - linters:
           - revive
+        path: crypto/bn256/
       - path: cmd/utils/flags.go
         text: "SA1019: cfg.TxLookupLimit is deprecated: use 'TransactionHistory' instead."
       - path: cmd/utils/flags.go
@@ -72,8 +68,29 @@ issues:
         text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.'
       - path: core/vm/contracts.go
         text: 'SA1019: "golang.org/x/crypto/ripemd160" is deprecated: RIPEMD-160 is a legacy hash and should not be used for new applications.'
-  exclude:
+      - path: (.+)\.go$
-    - 'SA1019: event.TypeMux is deprecated: use Feed'
+        text: 'SA1019: event.TypeMux is deprecated: use Feed'
-    - 'SA1019: strings.Title is deprecated'
+      - path: (.+)\.go$
-    - 'SA1019: strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead.'
+        text: 'SA1019: strings.Title is deprecated'
-    - 'SA1029: should not use built-in type string as key for value'
+      - path: (.+)\.go$
+        text: 'SA1019: strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead.'
+      - path: (.+)\.go$
+        text: 'SA1029: should not use built-in type string as key for value'
+    paths:
+      - core/genesis_alloc.go
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - goimports
+  settings:
+    gofmt:
+      simplify: true
+  exclusions:
+    generated: lax
+    paths:
+      - core/genesis_alloc.go
+      - third_party$
+      - builtin$
+      - examples$

.mailmap

@@ -50,6 +50,10 @@ Boqin Qin <bobbqqin@bupt.edu.cn> <Bobbqqin@gmail.com>
 
 Casey Detrio <cdetrio@gmail.com>
 
+Charlotte <tqpcharlie@proton.me>
+Charlotte <tqpcharlie@proton.me> <Zachinquarantine@protonmail.com>
+Charlotte <tqpcharlie@proton.me> <zachinquarantine@yahoo.com>
+
 Cheng Li <lob4tt@gmail.com>
 
 Chris Ziogas <ziogaschr@gmail.com>
@@ -301,9 +305,6 @@ Yohann Léon <sybiload@gmail.com>
 yzb <335357057@qq.com>
 yzb <335357057@qq.com> <flyingyzb@gmail.com>
 
-Zachinquarantine <Zachinquarantine@protonmail.com>
-Zachinquarantine <Zachinquarantine@protonmail.com> <zachinquarantine@yahoo.com>
-
 Ziyuan Zhong <zzy.albert@163.com>
 
 Zsolt Felföldi <zsfelfoldi@gmail.com>

.travis.yml

@@ -2,16 +2,10 @@ language: go
 go_import_path: github.com/ethereum/go-ethereum
 sudo: false
 jobs:
-  allow_failures:
-    - stage: build
-      os: osx
-      env:
-        - azure-osx
-
   include:
     # This builder create and push the Docker images for all architectures
     - stage: build
-      if: type = push
+      if: type = push && tag ~= /^v[0-9]/
       os: linux
       arch: amd64
       dist: focal
@@ -27,9 +21,25 @@ jobs:
       script:
         - go run build/ci.go dockerx -platform "linux/amd64,linux/arm64,linux/riscv64" -hub ethereum/client-go -upload
 
+    # This builder does the Ubuntu PPA nightly uploads
+    - stage: build
+      if: type = push && tag ~= /^v[0-9]/
+      os: linux
+      dist: focal
+      go: 1.24.x
+      env:
+        - ubuntu-ppa
+      git:
+        submodules: false # avoid cloning ethereum/tests
+      before_install:
+        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install devscripts debhelper dput fakeroot
+      script:
+        - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
+        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
+
     # This builder does the Linux Azure uploads
     - stage: build
-      if: type = push
+      if: type = push && tag ~= /^v[0-9]/
       os: linux
       dist: focal
       sudo: required
@@ -62,57 +72,6 @@ jobs:
         - go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
 
-    # This builder does the OSX Azure uploads
-    - stage: build
-      if: type = push
-      os: osx
-      osx_image: xcode14.2
-      go: 1.23.1 # See https://github.com/ethereum/go-ethereum/pull/30478
-      env:
-        - azure-osx
-      git:
-        submodules: false # avoid cloning ethereum/tests
-      script:
-        - ln -sf /Users/travis/gopath/bin/go1.23.1 /usr/local/bin/go # Work around travis go-setup bug
-        - go run build/ci.go install -dlgo
-        - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
-        - go run build/ci.go install -dlgo -arch arm64
-        - go run build/ci.go archive -arch arm64 -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
-
-    # These builders run the tests
-    - stage: build
-      if: type = push
-      os: linux
-      arch: amd64
-      dist: focal
-      go: 1.24.x
-      script:
-        - travis_wait 45 go run build/ci.go test $TEST_PACKAGES
-
-    - stage: build
-      if: type = push
-      os: linux
-      dist: focal
-      go: 1.23.x
-      script:
-        - travis_wait 45 go run build/ci.go test $TEST_PACKAGES
-
-    # This builder does the Ubuntu PPA nightly uploads
-    - stage: build
-      if: type = cron || (type = push && tag ~= /^v[0-9]/)
-      os: linux
-      dist: focal
-      go: 1.24.x
-      env:
-        - ubuntu-ppa
-      git:
-        submodules: false # avoid cloning ethereum/tests
-      before_install:
-        - sudo -E apt-get -yq --no-install-suggests --no-install-recommends install devscripts debhelper dput fakeroot
-      script:
-        - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts
-        - go run build/ci.go debsrc -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder <geth-ci@ethereum.org>"
-
     # This builder does the Azure archive purges to avoid accumulating junk
     - stage: build
       if: type = cron
@@ -125,14 +84,3 @@ jobs:
         submodules: false # avoid cloning ethereum/tests
       script:
         - go run build/ci.go purge -store gethstore/builds -days 14
-
-    # This builder executes race tests
-    - stage: build
-      if: type = cron
-      os: linux
-      dist: focal
-      go: 1.24.x
-      env:
-        - racetests
-      script:
-        - travis_wait 60 go run build/ci.go test -race $TEST_PACKAGES

AUTHORS

@@ -123,6 +123,7 @@ Ceyhun Onur <ceyhun.onur@avalabs.org>
 chabashilah <doumodoumo@gmail.com>
 changhong <changhong.yu@shanbay.com>
 Charles Cooper <cooper.charles.m@gmail.com>
+Charlotte <tqpcharlie@proton.me>
 Chase Wright <mysticryuujin@gmail.com>
 Chawin Aiemvaravutigul <nick41746@hotmail.com>
 Chen Quan <terasum@163.com>
@@ -839,7 +840,6 @@ ywzqwwt <39263032+ywzqwwt@users.noreply.github.com>
 yzb <335357057@qq.com>
 zaccoding <zaccoding725@gmail.com>
 Zach <zach.ramsay@gmail.com>
-Zachinquarantine <Zachinquarantine@protonmail.com>
 zah <zahary@gmail.com>
 Zahoor Mohamed <zahoor@zahoor.in>
 Zak Cole <zak@beattiecole.com>

Makefile

@@ -2,7 +2,7 @@
 # with Go source code. If you know what GOPATH is then you probably
 # don't need to bother with make.
 
-.PHONY: geth all test lint fmt clean devtools help
+.PHONY: geth evm all test lint fmt clean devtools help
 
 GOBIN = ./build/bin
 GO ?= latest
@@ -14,6 +14,12 @@ geth:
 	@echo "Done building."
 	@echo "Run \"$(GOBIN)/geth\" to launch geth."
 
+#? evm: Build evm.
+evm:
+	$(GORUN) build/ci.go install ./cmd/evm
+	@echo "Done building."
+	@echo "Run \"$(GOBIN)/evm\" to launch evm."
+
 #? all: Build all packages and executables.
 all:
 	$(GORUN) build/ci.go install

README.md

@@ -162,7 +162,7 @@ accessible from the outside.
 
 As a developer, sooner rather than later you'll want to start interacting with `geth` and the
 Ethereum network via your own programs and not manually through the console. To aid
-this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://ethereum.github.io/execution-apis/api-documentation/)
+this, `geth` has built-in support for a JSON-RPC based APIs ([standard APIs](https://ethereum.org/en/developers/docs/apis/json-rpc/)
 and [`geth` specific APIs](https://geth.ethereum.org/docs/interacting-with-geth/rpc)).
 These can be exposed via HTTP, WebSockets and IPC (UNIX sockets on UNIX based
 platforms, and named pipes on Windows).

accounts/abi/abigen/bind_test.go

@@ -939,6 +939,7 @@ var bindTests = []struct {
 					if _, err := eventer.RaiseSimpleEvent(auth, common.Address{byte(j)}, [32]byte{byte(j)}, true, big.NewInt(int64(10*i+j))); err != nil {
 						t.Fatalf("block %d, event %d: raise failed: %v", i, j, err)
 					}
+					time.Sleep(time.Millisecond * 200)
 				}
 				sim.Commit()
 			}
@@ -1495,7 +1496,7 @@ var bindTests = []struct {
 			if n != 3 {
 				t.Fatalf("Invalid bar0 event")
 			}
-		case <-time.NewTimer(3 * time.Second).C:
+		case <-time.NewTimer(10 * time.Second).C:
 			t.Fatalf("Wait bar0 event timeout")
 		}
 
@@ -1506,7 +1507,7 @@ var bindTests = []struct {
 			if n != 1 {
 				t.Fatalf("Invalid bar event")
 			}
-		case <-time.NewTimer(3 * time.Second).C:
+		case <-time.NewTimer(10 * time.Second).C:
 			t.Fatalf("Wait bar event timeout")
 		}
 		close(stopCh)
@@ -1806,7 +1807,9 @@ var bindTests = []struct {
 		[]string{"608060405234801561001057600080fd5b50610113806100206000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806324ec1d3f14602d575b600080fd5b60336035565b005b7fb4b2ff75e30cb4317eaae16dd8a187dd89978df17565104caa6c2797caae27d460405180604001604052806001815260200160028152506040516078919060ba565b60405180910390a1565b6040820160008201516096600085018260ad565b50602082015160a7602085018260ad565b50505050565b60b48160d3565b82525050565b600060408201905060cd60008301846082565b92915050565b600081905091905056fea26469706673582212208823628796125bf9941ce4eda18da1be3cf2931b231708ab848e1bd7151c0c9a64736f6c63430008070033"},
 		[]string{`[{"anonymous":false,"inputs":[{"components":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"uint256","name":"b","type":"uint256"}],"indexed":false,"internalType":"struct Test.MyStruct","name":"s","type":"tuple"}],"name":"StructEvent","type":"event"},{"inputs":[],"name":"TestEvent","outputs":[],"stateMutability":"nonpayable","type":"function"}]`},
 		`
+			"context"
 			"math/big"
+			"time"
 
 			"github.com/ethereum/go-ethereum/accounts/abi/bind"
 			"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
@@ -1828,12 +1831,23 @@ var bindTests = []struct {
 			}
 			sim.Commit()
 
-			_, err = d.TestEvent(user)
+			tx, err := d.TestEvent(user)
 			if err != nil {
 				t.Fatalf("Failed to call contract %v", err)
 			}
 			sim.Commit()
 
+			// Wait for the transaction to be mined
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+			receipt, err := bind.WaitMined(ctx, sim, tx)
+			if err != nil {
+				t.Fatalf("Failed to wait for tx to be mined: %v", err)
+			}
+			if receipt.Status != types.ReceiptStatusSuccessful {
+				t.Fatal("Transaction failed")
+			}
+
 			it, err := d.FilterStructEvent(nil)
 			if err != nil {
 				t.Fatalf("Failed to filter contract event %v", err)

accounts/abi/abigen/testdata/v2/structs-abi.go.txt

@@ -1,116 +0,0 @@
-// Code generated via abigen V2 - DO NOT EDIT.
-// This file is a generated binding and any manual changes will be lost.
-
-package v1bindtests
-
-import (
-	"bytes"
-	"errors"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/accounts/abi"
-	"github.com/ethereum/go-ethereum/accounts/abi/bind/v2"
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var (
-	_ = bytes.Equal
-	_ = errors.New
-	_ = big.NewInt
-	_ = common.Big1
-	_ = types.BloomLookup
-	_ = abi.ConvertType
-)
-
-// Struct0 is an auto generated low-level Go binding around an user-defined struct.
-type Struct0 struct {
-	B [32]byte
-}
-
-// StructsMetaData contains all meta data concerning the Structs contract.
-var StructsMetaData = bind.MetaData{
-	ABI: "[{\"inputs\":[],\"name\":\"F\",\"outputs\":[{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"B\",\"type\":\"bytes32\"}],\"internalType\":\"structStructs.A[]\",\"name\":\"a\",\"type\":\"tuple[]\"},{\"internalType\":\"uint256[]\",\"name\":\"c\",\"type\":\"uint256[]\"},{\"internalType\":\"bool[]\",\"name\":\"d\",\"type\":\"bool[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"},{\"inputs\":[],\"name\":\"G\",\"outputs\":[{\"components\":[{\"internalType\":\"bytes32\",\"name\":\"B\",\"type\":\"bytes32\"}],\"internalType\":\"structStructs.A[]\",\"name\":\"a\",\"type\":\"tuple[]\"}],\"stateMutability\":\"view\",\"type\":\"function\"}]",
-	ID:  "Structs",
-}
-
-// Structs is an auto generated Go binding around an Ethereum contract.
-type Structs struct {
-	abi abi.ABI
-}
-
-// NewStructs creates a new instance of Structs.
-func NewStructs() *Structs {
-	parsed, err := StructsMetaData.ParseABI()
-	if err != nil {
-		panic(errors.New("invalid ABI: " + err.Error()))
-	}
-	return &Structs{abi: *parsed}
-}
-
-// Instance creates a wrapper for a deployed contract instance at the given address.
-<<<<<<< HEAD
-// Use this to create the instance object passed to abigen v2 library functions Call, Transact, etc.
-func (c *Structs) Instance(backend bind.ContractBackend, addr common.Address) *bind.BoundContract {
-	return bind.NewBoundContract(addr, c.abi, backend, backend, backend)
-=======
-// Use this to create the instance object passed to abigen v2 library functions Call,
-// Transact, etc.
-func (c *Structs) Instance(backend bind.ContractBackend, addr common.Address) bind.BoundContract {
-	return bind.NewBoundContract(backend, addr, c.abi)
->>>>>>> 854c25e086 (accounts/abi/abigen: improve v2 template)
-}
-
-// F is the Go binding used to pack the parameters required for calling
-// the contract method 0x28811f59.
-//
-// Solidity: function F() view returns((bytes32)[] a, uint256[] c, bool[] d)
-func (structs *Structs) PackF() ([]byte, error) {
-	return structs.abi.Pack("F")
-}
-
-// FOutput serves as a container for the return parameters of contract
-// method F.
-type FOutput struct {
-	A []Struct0
-	C []*big.Int
-	D []bool
-}
-
-// UnpackF is the Go binding that unpacks the parameters returned
-// from invoking the contract method with ID 0x28811f59.
-//
-// Solidity: function F() view returns((bytes32)[] a, uint256[] c, bool[] d)
-func (structs *Structs) UnpackF(data []byte) (*FOutput, error) {
-	out, err := structs.abi.Unpack("F", data)
-	if err != nil {
-		return nil, err
-	}
-	ret := new(FOutput)
-	ret.A = *abi.ConvertType(out[0], new([]Struct0)).(*[]Struct0)
-	ret.C = *abi.ConvertType(out[1], new([]*big.Int)).(*[]*big.Int)
-	ret.D = *abi.ConvertType(out[2], new([]bool)).(*[]bool)
-	return ret, nil
-}
-
-// G is the Go binding used to pack the parameters required for calling
-// the contract method 0x6fecb623.
-//
-// Solidity: function G() view returns((bytes32)[] a)
-func (structs *Structs) PackG() ([]byte, error) {
-	return structs.abi.Pack("G")
-}
-
-// UnpackG is the Go binding that unpacks the parameters returned
-// from invoking the contract method with ID 0x6fecb623.
-//
-// Solidity: function G() view returns((bytes32)[] a)
-func (structs *Structs) UnpackG(data []byte) (*[]Struct0, error) {
-	out, err := structs.abi.Unpack("G", data)
-	if err != nil {
-		return nil, err
-	}
-	out0 := *abi.ConvertType(out[0], new([]Struct0)).(*[]Struct0)
-	return &out0, nil
-}

accounts/abi/bind/v2/base.go

@@ -390,6 +390,7 @@ func (c *BoundContract) estimateGasLimit(opts *TransactOpts, contract *common.Ad
 		GasFeeCap:  gasFeeCap,
 		Value:      value,
 		Data:       input,
+		AccessList: opts.AccessList,
 	}
 	return c.transactor.EstimateGas(ensureContext(opts.Context), msg)
 }

accounts/abi/error_handling.go

@@ -32,6 +32,7 @@ var (
 	errBadInt16    = errors.New("abi: improperly encoded int16 value")
 	errBadInt32    = errors.New("abi: improperly encoded int32 value")
 	errBadInt64    = errors.New("abi: improperly encoded int64 value")
+	errInvalidSign = errors.New("abi: negatively-signed value cannot be packed into uint parameter")
 )
 
 // formatSliceString formats the reflection kind with the given slice size

accounts/abi/pack.go

@@ -37,7 +37,16 @@ func packBytesSlice(bytes []byte, l int) []byte {
 // t.
 func packElement(t Type, reflectValue reflect.Value) ([]byte, error) {
 	switch t.T {
-	case IntTy, UintTy:
+	case UintTy:
+		// make sure to not pack a negative value into a uint type.
+		if reflectValue.Kind() == reflect.Ptr {
+			val := new(big.Int).Set(reflectValue.Interface().(*big.Int))
+			if val.Sign() == -1 {
+				return nil, errInvalidSign
+			}
+		}
+		return packNum(reflectValue), nil
+	case IntTy:
 		return packNum(reflectValue), nil
 	case StringTy:
 		return packBytesSlice([]byte(reflectValue.String()), reflectValue.Len()), nil

accounts/abi/pack_test.go

@@ -177,6 +177,11 @@ func TestMethodPack(t *testing.T) {
 	if !bytes.Equal(packed, sig) {
 		t.Errorf("expected %x got %x", sig, packed)
 	}
+
+	// test that we can't pack a negative value for a parameter that is specified as a uint
+	if _, err := abi.Pack("send", big.NewInt(-1)); err == nil {
+		t.Fatal("expected error when trying to pack negative big.Int into uint256 value")
+	}
 }
 
 func TestPackNumber(t *testing.T) {

accounts/abi/unpack_test.go

@@ -1014,128 +1014,134 @@ func TestPackAndUnpackIncompatibleNumber(t *testing.T) {
 	cases := []struct {
 		decodeType  string
 		inputValue  *big.Int
-		err         error
+		unpackErr   error
+		packErr     error
 		expectValue interface{}
 	}{
 		{
 			decodeType: "uint8",
 			inputValue: big.NewInt(math.MaxUint8 + 1),
-			err:        errBadUint8,
+			unpackErr:  errBadUint8,
 		},
 		{
 			decodeType:  "uint8",
 			inputValue:  big.NewInt(math.MaxUint8),
-			err:         nil,
+			unpackErr:   nil,
 			expectValue: uint8(math.MaxUint8),
 		},
 		{
 			decodeType: "uint16",
 			inputValue: big.NewInt(math.MaxUint16 + 1),
-			err:        errBadUint16,
+			unpackErr:  errBadUint16,
 		},
 		{
 			decodeType:  "uint16",
 			inputValue:  big.NewInt(math.MaxUint16),
-			err:         nil,
+			unpackErr:   nil,
 			expectValue: uint16(math.MaxUint16),
 		},
 		{
 			decodeType: "uint32",
 			inputValue: big.NewInt(math.MaxUint32 + 1),
-			err:        errBadUint32,
+			unpackErr:  errBadUint32,
 		},
 		{
 			decodeType:  "uint32",
 			inputValue:  big.NewInt(math.MaxUint32),
-			err:         nil,
+			unpackErr:   nil,
 			expectValue: uint32(math.MaxUint32),
 		},
 		{
 			decodeType: "uint64",
 			inputValue: maxU64Plus1,
-			err:        errBadUint64,
+			unpackErr:  errBadUint64,
 		},
 		{
 			decodeType:  "uint64",
 			inputValue:  maxU64,
-			err:         nil,
+			unpackErr:   nil,
 			expectValue: uint64(math.MaxUint64),
 		},
 		{
 			decodeType:  "uint256",
 			inputValue:  maxU64Plus1,
-			err:         nil,
+			unpackErr:   nil,
 			expectValue: maxU64Plus1,
 		},
 		{
 			decodeType: "int8",
 			inputValue: big.NewInt(math.MaxInt8 + 1),
-			err:        errBadInt8,
+			unpackErr:  errBadInt8,
 		},
 		{
-			decodeType: "int8",
 			inputValue: big.NewInt(math.MinInt8 - 1),
-			err:        errBadInt8,
+			packErr:    errInvalidSign,
 		},
 		{
 			decodeType:  "int8",
 			inputValue:  big.NewInt(math.MaxInt8),
-			err:         nil,
+			unpackErr:   nil,
 			expectValue: int8(math.MaxInt8),
 		},
 		{
 			decodeType: "int16",
 			inputValue: big.NewInt(math.MaxInt16 + 1),
-			err:        errBadInt16,
+			unpackErr:  errBadInt16,
 		},
 		{
-			decodeType: "int16",
 			inputValue: big.NewInt(math.MinInt16 - 1),
-			err:        errBadInt16,
+			packErr:    errInvalidSign,
 		},
 		{
 			decodeType:  "int16",
 			inputValue:  big.NewInt(math.MaxInt16),
-			err:         nil,
+			unpackErr:   nil,
 			expectValue: int16(math.MaxInt16),
 		},
 		{
 			decodeType: "int32",
 			inputValue: big.NewInt(math.MaxInt32 + 1),
-			err:        errBadInt32,
+			unpackErr:  errBadInt32,
 		},
 		{
-			decodeType: "int32",
 			inputValue: big.NewInt(math.MinInt32 - 1),
-			err:        errBadInt32,
+			packErr:    errInvalidSign,
 		},
 		{
 			decodeType:  "int32",
 			inputValue:  big.NewInt(math.MaxInt32),
-			err:         nil,
+			unpackErr:   nil,
 			expectValue: int32(math.MaxInt32),
 		},
 		{
 			decodeType: "int64",
 			inputValue: new(big.Int).Add(big.NewInt(math.MaxInt64), big.NewInt(1)),
-			err:        errBadInt64,
+			unpackErr:  errBadInt64,
 		},
 		{
-			decodeType: "int64",
 			inputValue: new(big.Int).Sub(big.NewInt(math.MinInt64), big.NewInt(1)),
-			err:        errBadInt64,
+			packErr:    errInvalidSign,
 		},
 		{
 			decodeType:  "int64",
 			inputValue:  big.NewInt(math.MaxInt64),
-			err:         nil,
+			unpackErr:   nil,
 			expectValue: int64(math.MaxInt64),
 		},
 	}
 	for i, testCase := range cases {
 		packed, err := encodeABI.Pack(testCase.inputValue)
-		if err != nil {
+		if testCase.packErr != nil {
-			panic(err)
+			if err == nil {
+				t.Fatalf("expected packing of testcase input value to fail")
+			}
+			if err != testCase.packErr {
+				t.Fatalf("expected error '%v', got '%v'", testCase.packErr, err)
+			}
+			continue
+		}
+		if err != nil && err != testCase.packErr {
+			panic(fmt.Errorf("unexpected error packing test-case input: %v", err))
 		}
 		ty, err := NewType(testCase.decodeType, "", nil)
 		if err != nil {
@@ -1145,8 +1151,8 @@ func TestPackAndUnpackIncompatibleNumber(t *testing.T) {
 			{Type: ty},
 		}
 		decoded, err := decodeABI.Unpack(packed)
-		if err != testCase.err {
+		if err != testCase.unpackErr {
-			t.Fatalf("Expected error %v, actual error %v. case %d", testCase.err, err, i)
+			t.Fatalf("Expected error %v, actual error %v. case %d", testCase.unpackErr, err, i)
 		}
 		if err != nil {
 			continue

accounts/usbwallet/trezor.go

@@ -25,6 +25,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/accounts"
@@ -249,7 +250,11 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
 		}
 	}
 	// Extract the Ethereum signature and do a sanity validation
-	if len(response.GetSignatureR()) == 0 || len(response.GetSignatureS()) == 0 || response.GetSignatureV() == 0 {
+	if len(response.GetSignatureR()) == 0 || len(response.GetSignatureS()) == 0 {
+		return common.Address{}, nil, errors.New("reply lacks signature")
+	} else if response.GetSignatureV() == 0 && int(chainID.Int64()) <= (math.MaxUint32-36)/2 {
+		// for chainId >= (MaxUint32-36)/2, Trezor returns signature bit only
+		// https://github.com/trezor/trezor-mcu/pull/399
 		return common.Address{}, nil, errors.New("reply lacks signature")
 	}
 	signature := append(append(response.GetSignatureR(), response.GetSignatureS()...), byte(response.GetSignatureV()))
@@ -261,8 +266,12 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction
 	} else {
 		// Trezor backend does not support typed transactions yet.
 		signer = types.NewEIP155Signer(chainID)
+		// if chainId is above (MaxUint32 - 36) / 2 then the final v values is returned
+		// directly. Otherwise, the returned value is 35 + chainid * 2.
+		if signature[64] > 1 && int(chainID.Int64()) <= (math.MaxUint32-36)/2 {
 			signature[64] -= byte(chainID.Uint64()*2 + 35)
 		}
+	}
 
 	// Inject the final signature into the transaction and sanity check the sender
 	signed, err := tx.WithSignature(signer, signature)

beacon/blsync/client.go

@@ -23,9 +23,11 @@ import (
 	"github.com/ethereum/go-ethereum/beacon/light/sync"
 	"github.com/ethereum/go-ethereum/beacon/params"
 	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/mclock"
 	"github.com/ethereum/go-ethereum/ethdb/memorydb"
 	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
@@ -46,7 +48,13 @@ func NewClient(config params.ClientConfig) *Client {
 	var (
 		db             = memorydb.New()
 		committeeChain = light.NewCommitteeChain(db, &config.ChainConfig, config.Threshold, !config.NoFilter)
-		headTracker    = light.NewHeadTracker(committeeChain, config.Threshold)
+		headTracker    = light.NewHeadTracker(committeeChain, config.Threshold, func(checkpoint common.Hash) {
+			if saved, err := config.SaveCheckpointToFile(checkpoint); saved {
+				log.Debug("Saved beacon checkpoint", "file", config.CheckpointFile, "checkpoint", checkpoint)
+			} else if err != nil {
+				log.Error("Failed to save beacon checkpoint", "file", config.CheckpointFile, "checkpoint", checkpoint, "error", err)
+			}
+		})
 	)
 	headSync := sync.NewHeadSync(headTracker, committeeChain)
 

beacon/blsync/engineclient.go

@@ -26,6 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/beacon/params"
 	"github.com/ethereum/go-ethereum/beacon/types"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	ctypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
@@ -104,7 +105,11 @@ func (ec *engineClient) callNewPayload(fork string, event types.ChainHeadEvent)
 		method = "engine_newPayloadV4"
 		parentBeaconRoot := event.BeaconHead.ParentRoot
 		blobHashes := collectBlobHashes(event.Block)
-		params = append(params, blobHashes, parentBeaconRoot, event.ExecRequests)
+		hexRequests := make([]hexutil.Bytes, len(event.ExecRequests))
+		for i := range event.ExecRequests {
+			hexRequests[i] = hexutil.Bytes(event.ExecRequests[i])
+		}
+		params = append(params, blobHashes, parentBeaconRoot, hexRequests)
 	case "deneb":
 		method = "engine_newPayloadV3"
 		parentBeaconRoot := event.BeaconHead.ParentRoot

beacon/engine/types.go

@@ -123,6 +123,11 @@ type BlobAndProofV1 struct {
 	Proof hexutil.Bytes `json:"proof"`
 }
 
+type BlobAndProofV2 struct {
+	Blob       hexutil.Bytes   `json:"blob"`
+	CellProofs []hexutil.Bytes `json:"proofs"`
+}
+
 // JSON type overrides for ExecutionPayloadEnvelope.
 type executionPayloadEnvelopeMarshaling struct {
 	BlockValue *hexutil.Big
@@ -131,7 +136,7 @@ type executionPayloadEnvelopeMarshaling struct {
 
 type PayloadStatusV1 struct {
 	Status          string         `json:"status"`
-	Witness         *hexutil.Bytes `json:"witness"`
+	Witness         *hexutil.Bytes `json:"witness,omitempty"`
 	LatestValidHash *common.Hash   `json:"latestValidHash"`
 	ValidationError *string        `json:"validationError"`
 }
@@ -331,7 +336,9 @@ func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types.
 		for j := range sidecar.Blobs {
 			bundle.Blobs = append(bundle.Blobs, hexutil.Bytes(sidecar.Blobs[j][:]))
 			bundle.Commitments = append(bundle.Commitments, hexutil.Bytes(sidecar.Commitments[j][:]))
-			bundle.Proofs = append(bundle.Proofs, hexutil.Bytes(sidecar.Proofs[j][:]))
+		}
+		for _, proof := range sidecar.Proofs {
+			bundle.Proofs = append(bundle.Proofs, hexutil.Bytes(proof[:]))
 		}
 	}
 

beacon/engine/types_test.go

@@ -0,0 +1,56 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package engine
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto/kzg4844"
+)
+
+func TestBlobs(t *testing.T) {
+	var (
+		emptyBlob          = new(kzg4844.Blob)
+		emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob)
+		emptyBlobProof, _  = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit)
+		emptyCellProof, _  = kzg4844.ComputeCellProofs(emptyBlob)
+	)
+	header := types.Header{}
+	block := types.NewBlock(&header, &types.Body{}, nil, nil)
+
+	sidecarWithoutCellProofs := &types.BlobTxSidecar{
+		Blobs:       []kzg4844.Blob{*emptyBlob},
+		Commitments: []kzg4844.Commitment{emptyBlobCommit},
+		Proofs:      []kzg4844.Proof{emptyBlobProof},
+	}
+	env := BlockToExecutableData(block, common.Big0, []*types.BlobTxSidecar{sidecarWithoutCellProofs}, nil)
+	if len(env.BlobsBundle.Proofs) != 1 {
+		t.Fatalf("Expect 1 proof in blobs bundle, got %v", len(env.BlobsBundle.Proofs))
+	}
+
+	sidecarWithCellProofs := &types.BlobTxSidecar{
+		Blobs:       []kzg4844.Blob{*emptyBlob},
+		Commitments: []kzg4844.Commitment{emptyBlobCommit},
+		Proofs:      emptyCellProof,
+	}
+	env = BlockToExecutableData(block, common.Big0, []*types.BlobTxSidecar{sidecarWithCellProofs}, nil)
+	if len(env.BlobsBundle.Proofs) != 128 {
+		t.Fatalf("Expect 128 proofs in blobs bundle, got %v", len(env.BlobsBundle.Proofs))
+	}
+}

beacon/light/head_tracker.go

@@ -21,7 +21,9 @@ import (
 	"sync"
 	"time"
 
+	"github.com/ethereum/go-ethereum/beacon/params"
 	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/log"
 )
 
@@ -38,13 +40,15 @@ type HeadTracker struct {
 	hasFinalityUpdate   bool
 	prefetchHead        types.HeadInfo
 	changeCounter       uint64
+	saveCheckpoint      func(common.Hash)
 }
 
 // NewHeadTracker creates a new HeadTracker.
-func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int) *HeadTracker {
+func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int, saveCheckpoint func(common.Hash)) *HeadTracker {
 	return &HeadTracker{
 		committeeChain: committeeChain,
 		minSignerCount: minSignerCount,
+		saveCheckpoint: saveCheckpoint,
 	}
 }
 
@@ -100,6 +104,9 @@ func (h *HeadTracker) ValidateFinality(update types.FinalityUpdate) (bool, error
 	if replace {
 		h.finalityUpdate, h.hasFinalityUpdate = update, true
 		h.changeCounter++
+		if h.saveCheckpoint != nil && update.Finalized.Slot%params.EpochLength == 0 {
+			h.saveCheckpoint(update.Finalized.Hash())
+		}
 	}
 	return replace, err
 }

beacon/params/checkpoint_holesky.hex

@@ -0,0 +1 @@
+0xd60e5310c5d52ced44cfb13be4e9f22a1e6a6dc56964c3cccd429182d26d72d0

beacon/params/checkpoint_mainnet.hex

@@ -0,0 +1 @@
+0x02f0bb348b0d45f95a9b7e2bb5705768ad06548876cee03d880a2c9dabb1ff88

beacon/params/checkpoint_sepolia.hex

@@ -0,0 +1 @@
+0xa0dad451a230c01be6f2492980ec5bb412d8cf33351a75e8b172b5b84a5fd03a

beacon/params/config.go

@@ -54,6 +54,7 @@ type ChainConfig struct {
 	GenesisValidatorsRoot common.Hash // Root hash of the genesis validator set, used for signature domain calculation
 	Forks                 Forks
 	Checkpoint            common.Hash
+	CheckpointFile        string
 }
 
 // ForkAtEpoch returns the latest active fork at the given epoch.
@@ -211,3 +212,36 @@ func (f Forks) Less(i, j int) bool {
 	}
 	return f[i].knownIndex < f[j].knownIndex
 }
+
+// SetCheckpointFile sets the checkpoint import/export file name and attempts to
+// read the checkpoint from the file if it already exists. It returns true if
+// a checkpoint has been loaded.
+func (c *ChainConfig) SetCheckpointFile(checkpointFile string) (bool, error) {
+	c.CheckpointFile = checkpointFile
+	file, err := os.ReadFile(checkpointFile)
+	if os.IsNotExist(err) {
+		return false, nil // did not load checkpoint
+	}
+	if err != nil {
+		return false, fmt.Errorf("failed to read beacon checkpoint file: %v", err)
+	}
+	cp, err := hexutil.Decode(string(file))
+	if err != nil {
+		return false, fmt.Errorf("failed to decode hex string in beacon checkpoint file: %v", err)
+	}
+	if len(cp) != 32 {
+		return false, fmt.Errorf("invalid hex string length in beacon checkpoint file: %d", len(cp))
+	}
+	copy(c.Checkpoint[:len(cp)], cp)
+	return true, nil
+}
+
+// SaveCheckpointToFile saves the given checkpoint to file if a checkpoint
+// import/export file has been specified.
+func (c *ChainConfig) SaveCheckpointToFile(checkpoint common.Hash) (bool, error) {
+	if c.CheckpointFile == "" {
+		return false, nil
+	}
+	err := os.WriteFile(c.CheckpointFile, []byte(checkpoint.Hex()), 0600)
+	return err == nil, err
+}

beacon/params/networks.go

@@ -17,25 +17,37 @@
 package params
 
 import (
+	_ "embed"
+
 	"github.com/ethereum/go-ethereum/common"
 )
 
+//go:embed checkpoint_mainnet.hex
+var checkpointMainnet string
+
+//go:embed checkpoint_sepolia.hex
+var checkpointSepolia string
+
+//go:embed checkpoint_holesky.hex
+var checkpointHolesky string
+
 var (
 	MainnetLightConfig = (&ChainConfig{
 		GenesisValidatorsRoot: common.HexToHash("0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95"),
 		GenesisTime:           1606824023,
-		Checkpoint:            common.HexToHash("0x6509b691f4de4f7b083f2784938fd52f0e131675432b3fd85ea549af9aebd3d0"),
+		Checkpoint:            common.HexToHash(checkpointMainnet),
 	}).
 		AddFork("GENESIS", 0, []byte{0, 0, 0, 0}).
 		AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}).
 		AddFork("BELLATRIX", 144896, []byte{2, 0, 0, 0}).
 		AddFork("CAPELLA", 194048, []byte{3, 0, 0, 0}).
-		AddFork("DENEB", 269568, []byte{4, 0, 0, 0})
+		AddFork("DENEB", 269568, []byte{4, 0, 0, 0}).
+		AddFork("ELECTRA", 364032, []byte{5, 0, 0, 0})
 
 	SepoliaLightConfig = (&ChainConfig{
 		GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"),
 		GenesisTime:           1655733600,
-		Checkpoint:            common.HexToHash("0x456e85f5608afab3465a0580bff8572255f6d97af0c5f939e3f7536b5edb2d3f"),
+		Checkpoint:            common.HexToHash(checkpointSepolia),
 	}).
 		AddFork("GENESIS", 0, []byte{144, 0, 0, 105}).
 		AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}).
@@ -47,7 +59,7 @@ var (
 	HoleskyLightConfig = (&ChainConfig{
 		GenesisValidatorsRoot: common.HexToHash("0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1"),
 		GenesisTime:           1695902400,
-		Checkpoint:            common.HexToHash("0x6456a1317f54d4b4f2cb5bc9d153b5af0988fe767ef0609f0236cf29030bcff7"),
+		Checkpoint:            common.HexToHash(checkpointHolesky),
 	}).
 		AddFork("GENESIS", 0, []byte{1, 1, 112, 0}).
 		AddFork("ALTAIR", 0, []byte{2, 1, 112, 0}).

build/checksums.txt

@@ -1,113 +1,113 @@
 # This file contains sha256 checksums of optional build dependencies.
 
-# version:spec-tests pectra-devnet-6@v1.0.0
+# version:spec-tests v4.5.0
 # https://github.com/ethereum/execution-spec-tests/releases
-# https://github.com/ethereum/execution-spec-tests/releases/download/pectra-devnet-6%40v1.0.0/
+# https://github.com/ethereum/execution-spec-tests/releases/download/v4.5.0/
-b69211752a3029083c020dc635fe12156ca1a6725a08559da540a0337586a77e  fixtures_pectra-devnet-6.tar.gz
+58afb92a0075a2cb7c4dec1281f7cb88b21b02afbedad096b580f3f8cc14c54c  fixtures_develop.tar.gz
 
-# version:golang 1.24.1
+# version:golang 1.24.3
 # https://go.dev/dl/
-8244ebf46c65607db10222b5806aeb31c1fcf8979c1b6b12f60c677e9a3c0656  go1.24.1.src.tar.gz
+229c08b600b1446798109fae1f569228102c8473caba8104b6418cb5bc032878  go1.24.3.src.tar.gz
-8d627dc163a4bffa2b1887112ad6194af175dce108d606ed1714a089fb806033  go1.24.1.aix-ppc64.tar.gz
+6f6901497547db3b77c14f7f953fbcef9fa5fb84199ee2ee14a5686e66bed5a6  go1.24.3.aix-ppc64.tar.gz
-addbfce2056744962e2d7436313ab93486660cf7a2e066d171b9d6f2da7c7abe  go1.24.1.darwin-amd64.tar.gz
+a05fa7e4043a4fec66897135219e3b8ab2202b5ef351c60c2fbb531dfb8f2900  go1.24.3.darwin-amd64.pkg
-58d529334561cff11087cd4ab18fe0b46d8d5aad88f45c02b9645f847e014512  go1.24.1.darwin-amd64.pkg
+13e6fe3fcf65689d77d40e633de1e31c6febbdbcb846eb05fc2434ed2213e92b  go1.24.3.darwin-amd64.tar.gz
-295581b5619acc92f5106e5bcb05c51869337eb19742fdfa6c8346c18e78ff88  go1.24.1.darwin-arm64.tar.gz
+97055ff4214043b39dc32e043fdd5c565df7c0a4e2fc0174e779a134c347ae0e  go1.24.3.darwin-arm64.pkg
-78b0fc8ddc344eb499f1a952c687cb84cbd28ba2b739cfa0d4eb042f07e44e82  go1.24.1.darwin-arm64.pkg
+64a3fa22142f627e78fac3018ce3d4aeace68b743eff0afda8aae0411df5e4fb  go1.24.3.darwin-arm64.tar.gz
-e70053f56f7eb93806d80cbd5726f78509a0a467602f7bea0e2c4ee8ed7c3968  go1.24.1.dragonfly-amd64.tar.gz
+32de3fd44d5055973978436a7f1f0ffbaae85c1b603ec6105e5c38d8a674c721  go1.24.3.dragonfly-amd64.tar.gz
-3595e2674ed8fe72e604ca59c964d3e5277aafb08475c2b1aaca2d2fd69c24fc  go1.24.1.freebsd-386.tar.gz
+9fe6101b3797919bd7337ee5ce591954f85d59db7ae88983904db29fd64c3dd1  go1.24.3.freebsd-386.tar.gz
-47d7de8bb64d5c3ee7b6723aa62d5ecb11e3568ef2249bbe1d4bbd432d37c00c  go1.24.1.freebsd-amd64.tar.gz
+6ccf4cca287e90cc28cd7954b6172f5d177a17e20b072b65f7f39636c325e2fb  go1.24.3.freebsd-amd64.tar.gz
-04eec3bcfaa14c1370cdf98e8307fac7e4853496c3045afb9c3124a29cbca205  go1.24.1.freebsd-arm.tar.gz
+ce45ebf389066f82a7b056b66dd650efb51fde6f8bf92a2a3ab6990f02788ebf  go1.24.3.freebsd-arm.tar.gz
-51aa70146e40cfdc20927424083dc86e6223f85dc08089913a1651973b55665b  go1.24.1.freebsd-arm64.tar.gz
+8f6494a12a874d0ea57c67987829359e016960ce3ba0673273609d6ac2af589a  go1.24.3.freebsd-arm64.tar.gz
-3c131d8e3fc285a1340f87813153e24226d3ddbd6e54f3facbd6e4c46a84655e  go1.24.1.freebsd-riscv64.tar.gz
+f9db392560cf0851f0bc8f2190e1978e01b4603038c27fecfc8658a695b71616  go1.24.3.freebsd-riscv64.tar.gz
-201d09da737ba39d5367f87d4e8b31edaeeb3dc9b9c407cb8cfb40f90c5a727a  go1.24.1.illumos-amd64.tar.gz
+01717fff64c5d98457272002fa825d0a15e307bf6e189f2b0c23817fa033b61c  go1.24.3.illumos-amd64.tar.gz
-8c530ecedbc17e42ce10177bea07ccc96a3e77c792ea1ea72173a9675d16ffa5  go1.24.1.linux-386.tar.gz
+41b1051063e68cbd2b919bf12326764fe33937cf1d32b5c529dd1a4f43dce578  go1.24.3.linux-386.tar.gz
-cb2396bae64183cdccf81a9a6df0aea3bce9511fc21469fb89a0c00470088073  go1.24.1.linux-amd64.tar.gz
+3333f6ea53afa971e9078895eaa4ac7204a8c6b5c68c10e6bc9a33e8e391bdd8  go1.24.3.linux-amd64.tar.gz
-8df5750ffc0281017fb6070fba450f5d22b600a02081dceef47966ffaf36a3af  go1.24.1.linux-arm64.tar.gz
+a463cb59382bd7ae7d8f4c68846e73c4d589f223c589ac76871b66811ded7836  go1.24.3.linux-arm64.tar.gz
-6d95f8d7884bfe2364644c837f080f2b585903d0b771eb5b06044e226a4f120a  go1.24.1.linux-armv6l.tar.gz
+17a392d7e826625dd12a32099df0b00b85c32d8132ed86fe917183ee5c3f88ed  go1.24.3.linux-armv6l.tar.gz
-19304a4a56e46d04604547d2d83235dc4f9b192c79832560ce337d26cc7b835a  go1.24.1.linux-loong64.tar.gz
+e4b003c04c902edc140153d279b42167f1ad7c229f48f1f729bbef5e65e88d1f  go1.24.3.linux-loong64.tar.gz
-6347be77fa5359c12a5308c8ab87147c1fc4717b0c216493d1706c3b9fcde22d  go1.24.1.linux-mips.tar.gz
+1c79d89edf835edf9d4336ccea7cb89bc5c0ca82b12b36b218d599a5400d60fe  go1.24.3.linux-mips.tar.gz
-1647df415f7030b82d4105670192aa7e8910e18563bb0d505192d72800cc2d21  go1.24.1.linux-mips64.tar.gz
+0b64fe147d69f4d681d8e8a035c760477531432f83d831f18d37cb9bf3652488  go1.24.3.linux-mips64.tar.gz
-762da594e4ec0f9cf6defae6ef971f5f7901203ee6a2d979e317adec96657317  go1.24.1.linux-mips64le.tar.gz
+396b784c255b64512dc00c302c053e43a3cbfc77518664c6ac5569aafad4d1e6  go1.24.3.linux-mips64le.tar.gz
-9d8133c7b23a557399fab870b5cf464079c2b623a43b214a7567cf11c254a444  go1.24.1.linux-mipsle.tar.gz
+93898313887f14e8efbe9d7386d5da4792b2d6c492bee562993fd4c9daa75c6d  go1.24.3.linux-mipsle.tar.gz
-132f10999abbaccbada47fa85462db30c423955913b14d6c692de25f4636c766  go1.24.1.linux-ppc64.tar.gz
+873ae3a6a6655a7b6f820e095d9965507e8dfd3cf76bc92d75c564ecbca385f6  go1.24.3.linux-ppc64.tar.gz
-0fb522efcefabae6e37e69bdc444094e75bfe824ea6d4cc3cbc70c7ae1b16858  go1.24.1.linux-ppc64le.tar.gz
+341a749d168f47b1d4dad25e32cae70849b7ceed7c290823b853c9e6b0df0856  go1.24.3.linux-ppc64le.tar.gz
-eaef4323d5467ff97fb1979c8491764060dade19f02f3275a9313f9a0da3b9c0  go1.24.1.linux-riscv64.tar.gz
+fa482f53ccb4ba280316b8c5751ea67291507280d9166f2a38fe4d9b5d5fb64b  go1.24.3.linux-riscv64.tar.gz
-6c05e14d8f11094cb56a1c50f390b6b658bed8a7cbd8d1a57e926581b7eabfce  go1.24.1.linux-s390x.tar.gz
+a87b0c2a079a0bece1620fb29a00e02b4dba17507850f837e754af7d57cda282  go1.24.3.linux-s390x.tar.gz
-5dbb287d343ea00d58a70b11629f32ee716dc50a6875c459ea2018df0f294cd8  go1.24.1.netbsd-386.tar.gz
+63155382308db1306200aff7821aa26bf2a2dda23537dd637a9704b485b6ddf0  go1.24.3.netbsd-386.tar.gz
-617aa3faee50ce84c343db0888e9a210c310a7203666b4ed620f31030c9fb32f  go1.24.1.netbsd-amd64.tar.gz
+fe2c5c79482958b867c08a4fc2a10a998de9c0206b08d5b3ebcb2232e8d2777c  go1.24.3.netbsd-amd64.tar.gz
-59a928b7080c4a6ac985946274b7c65ce1cecc0b468ecd992d17b7c12fab9296  go1.24.1.netbsd-arm.tar.gz
+e8ff77aef21521b5dd94e44282a3243309b80717414cf12f72835a45886a049f  go1.24.3.netbsd-arm.tar.gz
-28daa8d0feb4aef2af60cefa3305bb9314de7e8a05cbca41ac548964cdfe89b7  go1.24.1.netbsd-arm64.tar.gz
+b337fbaf82822685940ffaa76fbcf4be5d2f0258bc819cd80bc408b491f45c04  go1.24.3.netbsd-arm64.tar.gz
-b7382b2f5d99813aeac14db482faa3bfbd47a68880b607fa2a7e669e26bab9cd  go1.24.1.openbsd-386.tar.gz
+c1bb9dd8418480aa7f65452b08de3759da3bf89702be71b5a9fc084836b24ad5  go1.24.3.openbsd-386.tar.gz
-2513b6537c45deead5e641c7ce7502913e7d5e6f0b21c52542fb11f81578690f  go1.24.1.openbsd-amd64.tar.gz
+531218de748b0caaf6d1ad18921206fc12baaa89bf483a0a5e60a571c206fe6f  go1.24.3.openbsd-amd64.tar.gz
-853c1917d4fc7b144c55a02842aa48542d5cc798dde8db96dc0fdbc263200e04  go1.24.1.openbsd-arm.tar.gz
+bcd0dc959986fc346969b5d4111c3c8031882d8bf8d87a2c2ecf1328962a91f2  go1.24.3.openbsd-arm.tar.gz
-6bc207b91e6f6ae3347fb54616a8fb2f5c11983713846a4cef111ff3f4f94d14  go1.24.1.openbsd-arm64.tar.gz
+00ee6f8f1c41fd2e28ad386bd7e39acce7cab84af6de835855b29d1c597335c4  go1.24.3.openbsd-arm64.tar.gz
-4279260e2f2b94ee94e81470d13db7367f4393b061fee60985528fa0fa430df4  go1.24.1.openbsd-ppc64.tar.gz
+9f4ec0a9203ed3c54ce1a2a390ad3d45838cdb7efd85baeff857e37dfde04edd  go1.24.3.openbsd-ppc64.tar.gz
-6fc4023a0a339ee0778522364a127d94c78e62122288d47d820dba703f81dc07  go1.24.1.openbsd-riscv64.tar.gz
+da4d6f80e2373250d8c31c32dcd1e08775c327c0d610923604660cc0e07e8cba  go1.24.3.openbsd-riscv64.tar.gz
-b5eb9fafd77146e7e1f748acfd95559580ecc8d2f15abf432a20f58c929c7cd2  go1.24.1.plan9-386.tar.gz
+f5d02149132eedda6c2d46b360d7da462b8a5f9e3f8567db100c2d7bff0ddcd7  go1.24.3.plan9-386.tar.gz
-24dcad6361b141fc8cced15b092351e12a99d2e58d7013204a3013c50daf9fdd  go1.24.1.plan9-amd64.tar.gz
+175f3d79f4762a3c545d2c6393bf6b8bac24e838026869dafab06b930735c94f  go1.24.3.plan9-amd64.tar.gz
-a026ac3b55aa1e6fdc2aaab30207a117eafbe965ed81d3aa0676409f280ddc37  go1.24.1.plan9-arm.tar.gz
+d1e4ac15095da1611659261c2228c2058756cf87d61d9fad262f76755ef26849  go1.24.3.plan9-arm.tar.gz
-8e4f6a77388dc6e5aa481efd5abdb3b9f5c9463bb82f4db074494e04e5c84992  go1.24.1.solaris-amd64.tar.gz
+e644220a6ced3c07a7acc1364193cb709a97737dd8b6792a07a8ec6d9996713e  go1.24.3.solaris-amd64.tar.gz
-b799f4ab264eef12a014c759383ed934056608c483e0f73e34ea6caf9f1df5f9  go1.24.1.windows-386.zip
+0d7e7dc0a31ba0cdd487415709d03b02fc9490ef111e8dfd22788a6d63316f37  go1.24.3.windows-386.msi
-db128981033ac82a64688a123f631e61297b6b8f52ca913145e57caa8ce94cc3  go1.24.1.windows-386.msi
+c27c463a61ab849266baa0c17a6c5c4256a574ab642f609ba25c96ec965dc184  go1.24.3.windows-386.zip
-95666b551453209a2b8869d29d177285ff9573af10f085d961d7ae5440f645ce  go1.24.1.windows-amd64.zip
+d5b7637e7e138be877d96a4501709d480e050d86a8f402bc950e72112b5aedc5  go1.24.3.windows-amd64.msi
-5968e7adcf26e68a54f1cd41ad561275a670a8e2ca5263bc375b524638557dfb  go1.24.1.windows-amd64.msi
+be9787cb08998b1860fe3513e48a5fe5b96302d358a321b58e651184fa9638b3  go1.24.3.windows-amd64.zip
-e28c4e6d0b913955765b46157ab88ae59bb636acaa12d7bec959aa6900f1cebd  go1.24.1.windows-arm64.zip
+7efde2e5e8468e9caf2c7fc94f4da78a726a5031a1ed63acff7899527cdddff6  go1.24.3.windows-arm64.msi
-6d352c1f154a102a5b90c480cc64bab205ccf2681e34e78a3a4d3f1ddfbc81e4  go1.24.1.windows-arm64.msi
+eec9fa736056b54dd88ecb669db2bfad39b0c48f6f9080f036dfa1ca42dc4bb5  go1.24.3.windows-arm64.zip
 
-# version:golangci 1.64.6
+# version:golangci 2.0.2
 # https://github.com/golangci/golangci-lint/releases/
-# https://github.com/golangci/golangci-lint/releases/download/v1.64.6/
+# https://github.com/golangci/golangci-lint/releases/download/v2.0.2/
-08f9459e7125fed2612abd71596e04d172695921aff82120d1c1e5e9b6fff2a3  golangci-lint-1.64.6-darwin-amd64.tar.gz
+a88cbdc86b483fe44e90bf2dcc3fec2af8c754116e6edf0aa6592cac5baa7a0e  golangci-lint-2.0.2-darwin-amd64.tar.gz
-8c10d0c7c3935b8c2269d628b6a06a8f48d8fb4cc31af02fe4ce0cd18b0704c1  golangci-lint-1.64.6-darwin-arm64.tar.gz
+664550e7954f5f4451aae99b4f7382c1a47039c66f39ca605f5d9af1a0d32b49  golangci-lint-2.0.2-darwin-arm64.tar.gz
-c07fcabb55deb8d2c4d390025568e76162d7f91b1d14bd2311be45d8d440a624  golangci-lint-1.64.6-freebsd-386.tar.gz
+bda0f0f27d300502faceda8428834a76ca25986f6d9fc2bd41d201c3ed73f08e  golangci-lint-2.0.2-freebsd-386.tar.gz
-8ed1ef1102e1a42983ffcaae8e06de6a540334c3a69e205c610b8a7c92d469cd  golangci-lint-1.64.6-freebsd-amd64.tar.gz
+1cbd0c7ade3fb027d61d38a646ec1b51be5846952b4b04a5330e7f4687f2270c  golangci-lint-2.0.2-freebsd-amd64.tar.gz
-8f8dda66d1b4a85cc8a1daf1b69af6559c3eb0a41dd8033148a9ad85cfc0e1d9  golangci-lint-1.64.6-freebsd-armv6.tar.gz
+1e828a597726198b2e35acdbcc5f3aad85244d79846d2d2bdb05241c5a535f9e  golangci-lint-2.0.2-freebsd-armv6.tar.gz
-59e8ca1fa254661b2a55e96515b14a10cd02221d443054deac5b28c3c3e12d6b  golangci-lint-1.64.6-freebsd-armv7.tar.gz
+848b49315dc5cddd0c9ce35e96ab33d584db0ea8fb57bcbf9784f1622bec0269  golangci-lint-2.0.2-freebsd-armv7.tar.gz
-e3d323d5f132e9b7593141bfe1d19c8460e65a55cea1008ec96945fed563f981  golangci-lint-1.64.6-illumos-amd64.tar.gz
+cabf9a6beab574c7f98581eb237919e580024759e3cdc05c4d516b044dce6770  golangci-lint-2.0.2-illumos-amd64.tar.gz
-5ce910f2a864c5fbeb32a30cbd506e1b2ef215f7a0f422cd5be6370b13db6f03  golangci-lint-1.64.6-linux-386.deb
+2fde80d15ed6527791f106d606120620e913c3a663c90a8596861d0a4461169e  golangci-lint-2.0.2-linux-386.deb
-2caab682a26b9a5fb94ba24e3a7e1404fa9eab2c12e36ae1c5548d80a1be190c  golangci-lint-1.64.6-linux-386.rpm
+804bc6e350a8c613aaa0a33d8d45414a80157b0ba1b2c2335ac859f85ad98ebd  golangci-lint-2.0.2-linux-386.rpm
-2d82d0a4716e6d9b70c95103054855cb4b5f20f7bbdee42297f0189955bd14b6  golangci-lint-1.64.6-linux-386.tar.gz
+e64beb72fecf581e57d88ae3adb1c9d4bf022694b6bd92e3c8e460910bbdc37d  golangci-lint-2.0.2-linux-386.tar.gz
-9cd82503e9841abcaa57663fc899587fe90363c26d86a793a98d3080fd25e907  golangci-lint-1.64.6-linux-amd64.deb
+9c55aed174d7a52bb1d4006b36e7edee9023631f6b814a80cb39c9860d6f75c3  golangci-lint-2.0.2-linux-amd64.deb
-981aaca5e5202d4fbb162ec7080490eb67ef5d32add5fb62fb02210597acc9da  golangci-lint-1.64.6-linux-amd64.rpm
+c55a2ef741a687b4c679696931f7fd4a467babd64c9457cf17bb9632fd1cecd1  golangci-lint-2.0.2-linux-amd64.rpm
-71e290acbacb7b3ba4f68f0540fb74dc180c4336ac8a6f3bdcd7fcc48b15148d  golangci-lint-1.64.6-linux-amd64.tar.gz
+89cc8a7810dc63b9a37900da03e37c3601caf46d42265d774e0f1a5d883d53e2  golangci-lint-2.0.2-linux-amd64.tar.gz
-718016bb06c1f598a8d23c7964e2643de621dbe5808688cb38857eb0bb773c84  golangci-lint-1.64.6-linux-arm64.deb
+a3e78583c4e7ea1b63e82559f126bb3a5b12788676f158526752d53e67824b99  golangci-lint-2.0.2-linux-arm64.deb
-ddc0e7b4b10b47cf894aea157e89e3674bbb60f8f5c480110c21c49cc2c1634d  golangci-lint-1.64.6-linux-arm64.rpm
+bd5dd52b5c9f18aa7a2904eda9a9f91c628e98623fe70b7afcbb847e2de84422  golangci-lint-2.0.2-linux-arm64.rpm
-99a7ff13dec7a8781a68408b6ecb8a1c5e82786cba3189eaa91d5cdcc24362ce  golangci-lint-1.64.6-linux-arm64.tar.gz
+789d5b91219ac68c2336f77d41cd7e33a910420594780f455893f8453d09595b  golangci-lint-2.0.2-linux-arm64.tar.gz
-90e360f89c244394912b8709fb83a900b6d56cf19141df2afaf9e902ef3057b1  golangci-lint-1.64.6-linux-armv6.deb
+534cd4c464a66178714ed68152c1ed7aa73e5700bf409e4ed1a8363adf96afca  golangci-lint-2.0.2-linux-armv6.deb
-46546ff7c98d873ffcdbee06b39dc1024fc08db4fbf1f6309360a44cf976b5c2  golangci-lint-1.64.6-linux-armv6.rpm
+cf7d02905a5fc80b96c9a64621693b4cc7337b1ce29986c19fd72608dafe66c5  golangci-lint-2.0.2-linux-armv6.rpm
-e45c1a42868aca0b0ee54d14fb89da216f3b4409367cd7bce3b5f36788b4fc92  golangci-lint-1.64.6-linux-armv6.tar.gz
+a0d81cb527d8fe878377f2356b5773e219b0b91832a6b59e7b9bcf9a90fe0b0e  golangci-lint-2.0.2-linux-armv6.tar.gz
-3cf6ddbbbf358db3de4b64a24f9683bbe2da3f003cfdee86cb610124b57abafb  golangci-lint-1.64.6-linux-armv7.deb
+dedd5be7fff8cba8fe15b658a59347ea90d7d02a9fff87f09c7687e6da05a8b6  golangci-lint-2.0.2-linux-armv7.deb
-508b6712710da10f11aab9ea5e63af415c932dfe7fff718e649d3100b838f797  golangci-lint-1.64.6-linux-armv7.rpm
+85521b6f3ad2f5a2bc9bfe14b9b08623f764964048f75ed6dfcfaf8eb7d57cc1  golangci-lint-2.0.2-linux-armv7.rpm
-da9a8bbee86b4dfee73fbc17e0856ec84c5b04919eb09bf3dd5904c39ce41753  golangci-lint-1.64.6-linux-armv7.tar.gz
+96471046c7780dda4ea680f65e92c2ef56ff58d40bcffaf6cfe9d6d48e3c27aa  golangci-lint-2.0.2-linux-armv7.tar.gz
-ad496a58284e1e9c8ac6f993fec429dcd96c85a8c4715dbb6530a5af0dae7fbd  golangci-lint-1.64.6-linux-loong64.deb
+815d914a7738e4362466b2d11004e8618b696b49e8ace13df2c2b25f28fb1e17  golangci-lint-2.0.2-linux-loong64.deb
-3bd70fa737b224750254dce616d9a188570e49b894b0cdb2fd04155e2c061350  golangci-lint-1.64.6-linux-loong64.rpm
+f16381e3d8a0f011b95e086d83d620248432b915d01f4beab4d29cfe4dc388b0  golangci-lint-2.0.2-linux-loong64.rpm
-a535af973499729f2215a84303eb0de61399057aad6901ddea1b4f73f68d5f2c  golangci-lint-1.64.6-linux-loong64.tar.gz
+1bd8d7714f9c92db6a0f23bae89f39c85ba047bec8eeb42b8748d30ae3228d18  golangci-lint-2.0.2-linux-loong64.tar.gz
-6ad2a1cd37ca30303a488abfca2de52aff57519901c6d8d2608656fe9fb05292  golangci-lint-1.64.6-linux-mips64.deb
+ea6e9d4aabb526aa298e47e8b026d8893d918c5eb919ba0ab403e315def74cc5  golangci-lint-2.0.2-linux-mips64.deb
-5f18369f0ca010a02c267352ebe6e3e0514c6debff49899c9e5520906c0da287  golangci-lint-1.64.6-linux-mips64.rpm
+519d8d53af83fdc9c25cc3fba8b663331ac22ef68131d4b0084cb6f425b6f79a  golangci-lint-2.0.2-linux-mips64.rpm
-3449d6c13307b91b0e8817f8911c07c3412cdb6931b8d101e42db3e9762e91ad  golangci-lint-1.64.6-linux-mips64.tar.gz
+80d655a0a1ac1b19dcef4b58fa2a7dadb646cc50ad08d460b5c53cdb421165e4  golangci-lint-2.0.2-linux-mips64.tar.gz
-d4712a348f65dcaf9f6c58f1cfd6d0984d54a902873dca5e76f0d686f5c59499  golangci-lint-1.64.6-linux-mips64le.deb
+aa0e75384bb482c865d4dfc95d23ceb25666bf20461b67a832f0eea6670312ec  golangci-lint-2.0.2-linux-mips64le.deb
-57cea4538894558cb8c956d6b69c5509e4304546abe4a467478fc9ada0c736d6  golangci-lint-1.64.6-linux-mips64le.rpm
+f2a8b500fb69bdea1b01df6267aaa5218fa4a58aeb781c1a20d0d802fe465a52  golangci-lint-2.0.2-linux-mips64le.rpm
-bc030977d44535b2152fddb2732f056d193c043992fe638ddecea21a40ef09fe  golangci-lint-1.64.6-linux-mips64le.tar.gz
+e66a0c0c9a275f02d27a7caa9576112622306f001d73dfc082cf1ae446fc1242  golangci-lint-2.0.2-linux-mips64le.tar.gz
-1ceb4e492efa527d246c61798c581f49113756fab8c39bb3eefdb1fa97041f92  golangci-lint-1.64.6-linux-ppc64le.deb
+e85ad51aac6428be2d8a37000d053697371a538a5bcbc1644caa7c5e77f6d0af  golangci-lint-2.0.2-linux-ppc64le.deb
-454e1c2b3583a8644e0c33a970fb4ce35b8f11848e1a770d9095d99d159ad0ab  golangci-lint-1.64.6-linux-ppc64le.rpm
+906798365eac1944af2a9b9a303e6fd49ec9043307bc681b7a96277f7f8beea5  golangci-lint-2.0.2-linux-ppc64le.rpm
-fddf30d35923eb6c7bb57520d8645768f802bf86c4cbf5a3a13049efb9e71b82  golangci-lint-1.64.6-linux-ppc64le.tar.gz
+f7f1a271b0af274d6c9ce000f5dc6e1fb194350c67bcc62494f96f791882ba92  golangci-lint-2.0.2-linux-ppc64le.tar.gz
-bd75f0dd6f65bee5821c433803b28f3c427ef3582764c3d0e7f7fae1c9d468b6  golangci-lint-1.64.6-linux-riscv64.deb
+eea8bf643a42bf05de9780530db22923e5ab0d588f0e173594dc6518f2a25d2a  golangci-lint-2.0.2-linux-riscv64.deb
-58457207c225cbd5340c8dcb75d9a44aa890d604e28464115a3a9762febaed04  golangci-lint-1.64.6-linux-riscv64.rpm
+4ff40f9fe2954400836e2a011ba4744d00ffab5068a51368552dfce6aba3b81b  golangci-lint-2.0.2-linux-riscv64.rpm
-82639518a613a6705b7e2de5b28c278e875d782a5c97e9c1b2ac10b4ecd7852f  golangci-lint-1.64.6-linux-riscv64.tar.gz
+531d8f225866674977d630afbf0533eb02f9bec607fb13895f7a2cd7b2e0a648  golangci-lint-2.0.2-linux-riscv64.tar.gz
-131d69204f8ced200b1437731e987cc886edac30dc87e6e1dcbd4f833d351713  golangci-lint-1.64.6-linux-s390x.deb
+6f827647046c603f40d97ea5aadc6f48cd0bb5d19f7a3d56500c3b833d2a0342  golangci-lint-2.0.2-linux-s390x.deb
-ca6caf28ca7a1df7cff720f8fd6ac4b8f2eac10c8cbfe7d2eade70876aded570  golangci-lint-1.64.6-linux-s390x.rpm
+387a090e9576d19ca86aac738172e58e07c19f2784a13bb387f4f0d75fb9c8d3  golangci-lint-2.0.2-linux-s390x.rpm
-ed966d28a6512bc2b1120029a9f78ed77f2015e330b589a731d67d59be30b0b1  golangci-lint-1.64.6-linux-s390x.tar.gz
+57de1fb7722a9feb2d11ed0a007a93959d05b9db5929a392abc222e30012467e  golangci-lint-2.0.2-linux-s390x.tar.gz
-b181132b41943fc77ace7f9f5523085d12d3613f27774a0e35e17dd5e65ba589  golangci-lint-1.64.6-netbsd-386.tar.gz
+ed95e0492ea86bf79eb661f0334474b2a4255093685ff587eccd797c5a54db7e  golangci-lint-2.0.2-netbsd-386.tar.gz
-f7b81c426006e3c699dc8665797a462aecba8c2cd23ac4971472e55184d95bc9  golangci-lint-1.64.6-netbsd-amd64.tar.gz
+eab81d729778166415d349a80e568b2f2b3a781745a9be3212a92abb1e732daf  golangci-lint-2.0.2-netbsd-amd64.tar.gz
-663d6fb4c3bef57b8aacdb1b1a4634075e55d294ed170724b443374860897ca6  golangci-lint-1.64.6-netbsd-arm64.tar.gz
+d20add73f7c2de2c3b01ed4fd7b63ffcf0a6597d5ea228d1699e92339a3cd047  golangci-lint-2.0.2-netbsd-arm64.tar.gz
-8c7a76ee822568cc19352bbb9b2b0863dc5e185eb07782adbbaf648afd02ebcd  golangci-lint-1.64.6-netbsd-armv6.tar.gz
+4e4f44e6057879cd62424ff1800a767d25a595c0e91d6d48809eea9186b4c739  golangci-lint-2.0.2-netbsd-armv6.tar.gz
-0ce26d56ce78e516529e087118ba3f1bcd71d311b4c5b2bde6ec24a6e8966d85  golangci-lint-1.64.6-netbsd-armv7.tar.gz
+51ec17b16d8743ae4098a0171f04f0ed4d64561e3051b982778b0e6c306a1b03  golangci-lint-2.0.2-netbsd-armv7.tar.gz
-135d5d998168683f46e4fab308cef5aa3c282025b7de6b258f976aadfb820db7  golangci-lint-1.64.6-source.tar.gz
+5482cf27b93fae1765c70ee2a95d4074d038e9dee61bdd61d017ce8893d3a4a8  golangci-lint-2.0.2-source.tar.gz
-ccdb0cc249531a205304a76308cfa7cda830083d083d557884416a2461148263  golangci-lint-1.64.6-windows-386.zip
+a35d8fdf3e14079a10880dbbb7586b46faec89be96f086b244b3e565aac80313  golangci-lint-2.0.2-windows-386.zip
-2d88f282e08e1853c70bc7c914b5f58beaa6509903d56098aeb9bc3df30ea9d5  golangci-lint-1.64.6-windows-amd64.zip
+fe4b946cc01366b989001215687003a9c4a7098589921f75e6228d6d8cffc15c  golangci-lint-2.0.2-windows-amd64.zip
-6a3c6e7afc6916392679d7d6b95ac239827644e3e50ec4e8ca6ab1e4e6db65fe  golangci-lint-1.64.6-windows-arm64.zip
+646bd9250ef8c771d85cd22fe8e6f2397ae39599179755e3bbfa9ef97ad44090  golangci-lint-2.0.2-windows-arm64.zip
-9c9c1ef9687651566987f93e76252f7526c386901d7d8aeee044ca88115da4b1  golangci-lint-1.64.6-windows-armv6.zip
+ce1dc0bad6f8a61d64e6b3779eeb773479c175125d6f686b0e67ef9c8432d16e  golangci-lint-2.0.2-windows-armv6.zip
-4f0df114155791799dfde8cd8cb6307fecfb17fed70b44205486ec925e2f39cf  golangci-lint-1.64.6-windows-armv7.zip
+92684a48faabe792b11ac27ca8b25551eff940b0a1e84ad7244e98b4994962db  golangci-lint-2.0.2-windows-armv7.zip
 
 # This is the builder on PPA that will build Go itself (inception-y), don't modify!
 #

build/ci.go

@@ -59,6 +59,7 @@ import (
 	"github.com/cespare/cp"
 	"github.com/ethereum/go-ethereum/crypto/signify"
 	"github.com/ethereum/go-ethereum/internal/build"
+	"github.com/ethereum/go-ethereum/internal/download"
 	"github.com/ethereum/go-ethereum/internal/version"
 )
 
@@ -190,7 +191,7 @@ func doInstall(cmdline []string) {
 	// Configure the toolchain.
 	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
 	if *dlgo {
-		csdb := build.MustLoadChecksums("build/checksums.txt")
+		csdb := download.MustLoadChecksums("build/checksums.txt")
 		tc.Root = build.DownloadGo(csdb)
 	}
 	// Disable CLI markdown doc generation in release builds.
@@ -285,7 +286,7 @@ func doTest(cmdline []string) {
 	flag.CommandLine.Parse(cmdline)
 
 	// Get test fixtures.
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
 	downloadSpecTestFixtures(csdb, *cachedir)
 
 	// Configure the toolchain.
@@ -329,16 +330,11 @@ func doTest(cmdline []string) {
 }
 
 // downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
-func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string {
+func downloadSpecTestFixtures(csdb *download.ChecksumDB, cachedir string) string {
-	executionSpecTestsVersion, err := build.Version(csdb, "spec-tests")
-	if err != nil {
-		log.Fatal(err)
-	}
 	ext := ".tar.gz"
-	base := "fixtures_pectra-devnet-6" // TODO(s1na) rename once the version becomes part of the filename
+	base := "fixtures_develop"
-	url := fmt.Sprintf("https://github.com/ethereum/execution-spec-tests/releases/download/%s/%s%s", executionSpecTestsVersion, base, ext)
 	archivePath := filepath.Join(cachedir, base+ext)
-	if err := csdb.DownloadFile(url, archivePath); err != nil {
+	if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
 		log.Fatal(err)
 	}
 	if err := build.ExtractArchive(archivePath, executionSpecTestsDir); err != nil {
@@ -444,14 +440,13 @@ func doLint(cmdline []string) {
 
 // downloadLinter downloads and unpacks golangci-lint.
 func downloadLinter(cachedir string) string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
-	version, err := build.Version(csdb, "golangci")
+	version, err := csdb.FindVersion("golangci")
 	if err != nil {
 		log.Fatal(err)
 	}
 	arch := runtime.GOARCH
 	ext := ".tar.gz"
-
 	if runtime.GOOS == "windows" {
 		ext = ".zip"
 	}
@@ -459,9 +454,8 @@ func downloadLinter(cachedir string) string {
 		arch += "v" + os.Getenv("GOARM")
 	}
 	base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, arch)
-	url := fmt.Sprintf("https://github.com/golangci/golangci-lint/releases/download/v%s/%s%s", version, base, ext)
 	archivePath := filepath.Join(cachedir, base+ext)
-	if err := csdb.DownloadFile(url, archivePath); err != nil {
+	if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
 		log.Fatal(err)
 	}
 	if err := build.ExtractArchive(archivePath, cachedir); err != nil {
@@ -497,8 +491,8 @@ func protocArchiveBaseName() (string, error) {
 // in the generate command.  It returns the full path of the directory
 // containing the 'protoc-gen-go' executable.
 func downloadProtocGenGo(cachedir string) string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
-	version, err := build.Version(csdb, "protoc-gen-go")
+	version, err := csdb.FindVersion("protoc-gen-go")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -510,10 +504,8 @@ func downloadProtocGenGo(cachedir string) string {
 		archiveName += ".tar.gz"
 	}
 
-	url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf-go/releases/download/v%s/%s", version, archiveName)
-
 	archivePath := path.Join(cachedir, archiveName)
-	if err := csdb.DownloadFile(url, archivePath); err != nil {
+	if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
 		log.Fatal(err)
 	}
 	extractDest := filepath.Join(cachedir, baseName)
@@ -531,8 +523,8 @@ func downloadProtocGenGo(cachedir string) string {
 // files as a CI step.  It returns the full path to the directory containing
 // the protoc executable.
 func downloadProtoc(cachedir string) string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
-	version, err := build.Version(csdb, "protoc")
+	version, err := csdb.FindVersion("protoc")
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -543,10 +535,8 @@ func downloadProtoc(cachedir string) string {
 
 	fileName := fmt.Sprintf("protoc-%s-%s", version, baseName)
 	archiveFileName := fileName + ".zip"
-	url := fmt.Sprintf("https://github.com/protocolbuffers/protobuf/releases/download/v%s/%s", version, archiveFileName)
 	archivePath := filepath.Join(cachedir, archiveFileName)
-
+	if err := csdb.DownloadFileFromKnownURL(archivePath); err != nil {
-	if err := csdb.DownloadFile(url, archivePath); err != nil {
 		log.Fatal(err)
 	}
 	extractDest := filepath.Join(cachedir, fileName)
@@ -826,18 +816,17 @@ func doDebianSource(cmdline []string) {
 // downloadGoBootstrapSources downloads the Go source tarball(s) that will be used
 // to bootstrap the builder Go.
 func downloadGoBootstrapSources(cachedir string) []string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
 
 	var bundles []string
 	for _, booter := range []string{"ppa-builder-1.19", "ppa-builder-1.21", "ppa-builder-1.23"} {
-		gobootVersion, err := build.Version(csdb, booter)
+		gobootVersion, err := csdb.FindVersion(booter)
 		if err != nil {
 			log.Fatal(err)
 		}
 		file := fmt.Sprintf("go%s.src.tar.gz", gobootVersion)
-		url := "https://dl.google.com/go/" + file
 		dst := filepath.Join(cachedir, file)
-		if err := csdb.DownloadFile(url, dst); err != nil {
+		if err := csdb.DownloadFileFromKnownURL(dst); err != nil {
 			log.Fatal(err)
 		}
 		bundles = append(bundles, dst)
@@ -847,15 +836,14 @@ func downloadGoBootstrapSources(cachedir string) []string {
 
 // downloadGoSources downloads the Go source tarball.
 func downloadGoSources(cachedir string) string {
-	csdb := build.MustLoadChecksums("build/checksums.txt")
+	csdb := download.MustLoadChecksums("build/checksums.txt")
-	dlgoVersion, err := build.Version(csdb, "golang")
+	dlgoVersion, err := csdb.FindVersion("golang")
 	if err != nil {
 		log.Fatal(err)
 	}
 	file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion)
-	url := "https://dl.google.com/go/" + file
 	dst := filepath.Join(cachedir, file)
-	if err := csdb.DownloadFile(url, dst); err != nil {
+	if err := csdb.DownloadFileFromKnownURL(dst); err != nil {
 		log.Fatal(err)
 	}
 	return dst
@@ -1181,5 +1169,6 @@ func doPurge(cmdline []string) {
 }
 
 func doSanityCheck() {
-	build.DownloadAndVerifyChecksums(build.MustLoadChecksums("build/checksums.txt"))
+	csdb := download.MustLoadChecksums("build/checksums.txt")
+	csdb.DownloadAndVerifyAll()
 }

cmd/blsync/main.go

@@ -43,6 +43,7 @@ func main() {
 		utils.BeaconGenesisRootFlag,
 		utils.BeaconGenesisTimeFlag,
 		utils.BeaconCheckpointFlag,
+		utils.BeaconCheckpointFileFlag,
 		//TODO datadir for optional permanent database
 		utils.MainnetFlag,
 		utils.SepoliaFlag,

cmd/clef/README.md

@@ -1,6 +1,6 @@
 # Clef
 
-Clef can be used to sign transactions and data and is meant as a(n eventual) replacement for Geth's account management. This allows DApps to not depend on Geth's account management. When a DApp wants to sign data (or a transaction), it can send the content to Clef, which will then provide the user with context and asks for permission to sign the content. If the users grants the signing request, Clef will send the signature back to the DApp.
+Clef can be used to sign transactions and data and is meant as a(n eventual) replacement for Geth's account management. This allows DApps to not depend on Geth's account management. When a DApp wants to sign data (or a transaction), it can send the content to Clef, which will then provide the user with context and ask for permission to sign the content. If the user grants the signing request, Clef will send the signature back to the DApp.
 
 This setup allows a DApp to connect to a remote Ethereum node and send transactions that are locally signed. This can help in situations when a DApp is connected to an untrusted remote Ethereum node, because a local one is not available, not synchronized with the chain, or is a node that has no built-in (or limited) account management.
 
@@ -123,7 +123,7 @@ The External API is **untrusted**: it does not accept credentials, nor does it e
 
 Clef has one native console-based UI, for operation without any standalone tools. However, there is also an API to communicate with an external UI. To enable that UI, the signer needs to be executed with the `--stdio-ui` option, which allocates `stdin` / `stdout` for the UI API.
 
-An example (insecure) proof-of-concept of has been implemented in `pythonsigner.py`.
+An example (insecure) proof-of-concept has been implemented in `pythonsigner.py`.
 
 The model is as follows:
 
@@ -335,7 +335,7 @@ Bash example:
 
 #### Arguments
   - content type [string]: type of signed data
-     - `text/validator`: hex data with custom validator defined in a contract
+     - `text/validator`: hex data with a custom validator defined in a contract
      - `application/clique`: [clique](https://github.com/ethereum/EIPs/issues/225) headers
      - `text/plain`: simple hex data validated by `account_ecRecover`
   - account [address]: account to sign with

cmd/devp2p/internal/ethtest/conn.go

@@ -66,10 +66,9 @@ func (s *Suite) dialAs(key *ecdsa.PrivateKey) (*Conn, error) {
 		return nil, err
 	}
 	conn.caps = []p2p.Cap{
-		{Name: "eth", Version: 67},
+		{Name: "eth", Version: 69},
-		{Name: "eth", Version: 68},
 	}
-	conn.ourHighestProtoVersion = 68
+	conn.ourHighestProtoVersion = 69
 	return &conn, nil
 }
 
@@ -156,7 +155,7 @@ func (c *Conn) ReadEth() (any, error) {
 		var msg any
 		switch int(code) {
 		case eth.StatusMsg:
-			msg = new(eth.StatusPacket)
+			msg = new(eth.StatusPacket69)
 		case eth.GetBlockHeadersMsg:
 			msg = new(eth.GetBlockHeadersPacket)
 		case eth.BlockHeadersMsg:
@@ -229,9 +228,21 @@ func (c *Conn) ReadSnap() (any, error) {
 	}
 }
 
+// dialAndPeer creates a peer connection and runs the handshake.
+func (s *Suite) dialAndPeer(status *eth.StatusPacket69) (*Conn, error) {
+	c, err := s.dial()
+	if err != nil {
+		return nil, err
+	}
+	if err = c.peer(s.chain, status); err != nil {
+		c.Close()
+	}
+	return c, err
+}
+
 // peer performs both the protocol handshake and the status message
 // exchange with the node in order to peer with it.
-func (c *Conn) peer(chain *Chain, status *eth.StatusPacket) error {
+func (c *Conn) peer(chain *Chain, status *eth.StatusPacket69) error {
 	if err := c.handshake(); err != nil {
 		return fmt.Errorf("handshake failed: %v", err)
 	}
@@ -304,7 +315,7 @@ func (c *Conn) negotiateEthProtocol(caps []p2p.Cap) {
 }
 
 // statusExchange performs a `Status` message exchange with the given node.
-func (c *Conn) statusExchange(chain *Chain, status *eth.StatusPacket) error {
+func (c *Conn) statusExchange(chain *Chain, status *eth.StatusPacket69) error {
 loop:
 	for {
 		code, data, err := c.Read()
@@ -313,11 +324,15 @@ loop:
 		}
 		switch code {
 		case eth.StatusMsg + protoOffset(ethProto):
-			msg := new(eth.StatusPacket)
+			msg := new(eth.StatusPacket69)
 			if err := rlp.DecodeBytes(data, &msg); err != nil {
 				return fmt.Errorf("error decoding status packet: %w", err)
 			}
-			if have, want := msg.Head, chain.blocks[chain.Len()-1].Hash(); have != want {
+			if have, want := msg.LatestBlock, chain.blocks[chain.Len()-1].NumberU64(); have != want {
+				return fmt.Errorf("wrong head block in status, want: %d, have %d",
+					want, have)
+			}
+			if have, want := msg.LatestBlockHash, chain.blocks[chain.Len()-1].Hash(); have != want {
 				return fmt.Errorf("wrong head block in status, want: %#x (block %d) have %#x",
 					want, chain.blocks[chain.Len()-1].NumberU64(), have)
 			}
@@ -348,13 +363,14 @@ loop:
 	}
 	if status == nil {
 		// default status message
-		status = &eth.StatusPacket{
+		status = &eth.StatusPacket69{
 			ProtocolVersion: uint32(c.negotiatedProtoVersion),
 			NetworkID:       chain.config.ChainID.Uint64(),
-			TD:              chain.TD(),
-			Head:            chain.blocks[chain.Len()-1].Hash(),
 			Genesis:         chain.blocks[0].Hash(),
 			ForkID:          chain.ForkID(),
+			EarliestBlock:   0,
+			LatestBlock:     chain.blocks[chain.Len()-1].NumberU64(),
+			LatestBlockHash: chain.blocks[chain.Len()-1].Hash(),
 		}
 	}
 	if err := c.Write(ethProto, eth.StatusMsg, status); err != nil {

cmd/devp2p/internal/ethtest/protocol.go

@@ -32,7 +32,7 @@ const (
 // Unexported devp2p protocol lengths from p2p package.
 const (
 	baseProtoLen = 16
-	ethProtoLen  = 17
+	ethProtoLen  = 18
 	snapProtoLen = 8
 )
 

cmd/devp2p/internal/ethtest/suite.go

@@ -68,15 +68,19 @@ func (s *Suite) EthTests() []utesting.Test {
 	return []utesting.Test{
 		// status
 		{Name: "Status", Fn: s.TestStatus},
+		{Name: "MaliciousHandshake", Fn: s.TestMaliciousHandshake},
+		{Name: "BlockRangeUpdateExpired", Fn: s.TestBlockRangeUpdateHistoryExp},
+		{Name: "BlockRangeUpdateFuture", Fn: s.TestBlockRangeUpdateFuture},
+		{Name: "BlockRangeUpdateInvalid", Fn: s.TestBlockRangeUpdateInvalid},
 		// get block headers
 		{Name: "GetBlockHeaders", Fn: s.TestGetBlockHeaders},
+		{Name: "GetNonexistentBlockHeaders", Fn: s.TestGetNonexistentBlockHeaders},
 		{Name: "SimultaneousRequests", Fn: s.TestSimultaneousRequests},
 		{Name: "SameRequestID", Fn: s.TestSameRequestID},
 		{Name: "ZeroRequestID", Fn: s.TestZeroRequestID},
-		// get block bodies
+		// get history
 		{Name: "GetBlockBodies", Fn: s.TestGetBlockBodies},
-		// // malicious handshakes + status
+		{Name: "GetReceipts", Fn: s.TestGetReceipts},
-		{Name: "MaliciousHandshake", Fn: s.TestMaliciousHandshake},
 		// test transactions
 		{Name: "LargeTxRequest", Fn: s.TestLargeTxRequest, Slow: true},
 		{Name: "Transaction", Fn: s.TestTransaction},
@@ -100,15 +104,11 @@ func (s *Suite) SnapTests() []utesting.Test {
 
 func (s *Suite) TestStatus(t *utesting.T) {
 	t.Log(`This test is just a sanity check. It performs an eth protocol handshake.`)
-
+	conn, err := s.dialAndPeer(nil)
-	conn, err := s.dial()
 	if err != nil {
-		t.Fatalf("dial failed: %v", err)
+		t.Fatal("peering failed:", err)
-	}
-	defer conn.Close()
-	if err := conn.peer(s.chain, nil); err != nil {
-		t.Fatalf("peering failed: %v", err)
 	}
+	conn.Close()
 }
 
 // headersMatch returns whether the received headers match the given request
@@ -118,15 +118,12 @@ func headersMatch(expected []*types.Header, headers []*types.Header) bool {
 
 func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
 	t.Log(`This test requests block headers from the node.`)
-
+	conn, err := s.dialAndPeer(nil)
-	conn, err := s.dial()
 	if err != nil {
-		t.Fatalf("dial failed: %v", err)
-	}
-	defer conn.Close()
-	if err = conn.peer(s.chain, nil); err != nil {
 		t.Fatalf("peering failed: %v", err)
 	}
+	defer conn.Close()
+
 	// Send headers request.
 	req := &eth.GetBlockHeadersPacket{
 		RequestId: 33,
@@ -158,18 +155,51 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
 	}
 }
 
+func (s *Suite) TestGetNonexistentBlockHeaders(t *utesting.T) {
+	t.Log(`This test sends GetBlockHeaders requests for nonexistent blocks (using max uint64 value)
+to check if the node disconnects after receiving multiple invalid requests.`)
+	conn, err := s.dialAndPeer(nil)
+	if err != nil {
+		t.Fatalf("peering failed: %v", err)
+	}
+	defer conn.Close()
+
+	// Create request with max uint64 value for a nonexistent block
+	badReq := &eth.GetBlockHeadersPacket{
+		GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
+			Origin:  eth.HashOrNumber{Number: ^uint64(0)},
+			Amount:  1,
+			Skip:    0,
+			Reverse: false,
+		},
+	}
+
+	// Send request 10 times. Some clients are lient on the first few invalids.
+	for i := 0; i < 10; i++ {
+		badReq.RequestId = uint64(i)
+		if err := conn.Write(ethProto, eth.GetBlockHeadersMsg, badReq); err != nil {
+			if err == errDisc {
+				t.Fatalf("peer disconnected after %d requests", i+1)
+			}
+			t.Fatalf("write failed: %v", err)
+		}
+	}
+
+	// Check if peer disconnects at the end.
+	code, _, err := conn.Read()
+	if err == errDisc || code == discMsg {
+		t.Fatal("peer improperly disconnected")
+	}
+}
+
 func (s *Suite) TestSimultaneousRequests(t *utesting.T) {
 	t.Log(`This test requests blocks headers from the node, performing two requests
 concurrently, with different request IDs.`)
-
+	conn, err := s.dialAndPeer(nil)
-	conn, err := s.dial()
 	if err != nil {
-		t.Fatalf("dial failed: %v", err)
-	}
-	defer conn.Close()
-	if err := conn.peer(s.chain, nil); err != nil {
 		t.Fatalf("peering failed: %v", err)
 	}
+	defer conn.Close()
 
 	// Create two different requests.
 	req1 := &eth.GetBlockHeadersPacket{
@@ -235,15 +265,11 @@ concurrently, with different request IDs.`)
 func (s *Suite) TestSameRequestID(t *utesting.T) {
 	t.Log(`This test requests block headers, performing two concurrent requests with the
 same request ID. The node should handle the request by responding to both requests.`)
-
+	conn, err := s.dialAndPeer(nil)
-	conn, err := s.dial()
 	if err != nil {
-		t.Fatalf("dial failed: %v", err)
-	}
-	defer conn.Close()
-	if err := conn.peer(s.chain, nil); err != nil {
 		t.Fatalf("peering failed: %v", err)
 	}
+	defer conn.Close()
 
 	// Create two different requests with the same ID.
 	reqID := uint64(1234)
@@ -306,15 +332,12 @@ same request ID. The node should handle the request by responding to both reques
 func (s *Suite) TestZeroRequestID(t *utesting.T) {
 	t.Log(`This test sends a GetBlockHeaders message with a request-id of zero,
 and expects a response.`)
-
+	conn, err := s.dialAndPeer(nil)
-	conn, err := s.dial()
 	if err != nil {
-		t.Fatalf("dial failed: %v", err)
-	}
-	defer conn.Close()
-	if err := conn.peer(s.chain, nil); err != nil {
 		t.Fatalf("peering failed: %v", err)
 	}
+	defer conn.Close()
+
 	req := &eth.GetBlockHeadersPacket{
 		GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
 			Origin: eth.HashOrNumber{Number: 0},
@@ -341,15 +364,12 @@ and expects a response.`)
 
 func (s *Suite) TestGetBlockBodies(t *utesting.T) {
 	t.Log(`This test sends GetBlockBodies requests to the node for known blocks in the test chain.`)
-
+	conn, err := s.dialAndPeer(nil)
-	conn, err := s.dial()
 	if err != nil {
-		t.Fatalf("dial failed: %v", err)
-	}
-	defer conn.Close()
-	if err := conn.peer(s.chain, nil); err != nil {
 		t.Fatalf("peering failed: %v", err)
 	}
+	defer conn.Close()
+
 	// Create block bodies request.
 	req := &eth.GetBlockBodiesPacket{
 		RequestId: 55,
@@ -375,6 +395,47 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) {
 	}
 }
 
+func (s *Suite) TestGetReceipts(t *utesting.T) {
+	t.Log(`This test sends GetReceipts requests to the node for known blocks in the test chain.`)
+	conn, err := s.dialAndPeer(nil)
+	if err != nil {
+		t.Fatalf("peering failed: %v", err)
+	}
+	defer conn.Close()
+
+	// Find some blocks containing receipts.
+	var hashes = make([]common.Hash, 0, 3)
+	for i := range s.chain.Len() {
+		block := s.chain.GetBlock(i)
+		if len(block.Transactions()) > 0 {
+			hashes = append(hashes, block.Hash())
+		}
+		if len(hashes) == cap(hashes) {
+			break
+		}
+	}
+
+	// Create block bodies request.
+	req := &eth.GetReceiptsPacket{
+		RequestId:          66,
+		GetReceiptsRequest: (eth.GetReceiptsRequest)(hashes),
+	}
+	if err := conn.Write(ethProto, eth.GetReceiptsMsg, req); err != nil {
+		t.Fatalf("could not write to connection: %v", err)
+	}
+	// Wait for response.
+	resp := new(eth.ReceiptsPacket[*eth.ReceiptList69])
+	if err := conn.ReadMsg(ethProto, eth.ReceiptsMsg, &resp); err != nil {
+		t.Fatalf("error reading block bodies msg: %v", err)
+	}
+	if got, want := resp.RequestId, req.RequestId; got != want {
+		t.Fatalf("unexpected request id in respond", got, want)
+	}
+	if len(resp.List) != len(req.GetReceiptsRequest) {
+		t.Fatalf("wrong bodies in response: expected %d bodies, got %d", len(req.GetReceiptsRequest), len(resp.List))
+	}
+}
+
 // randBuf makes a random buffer size kilobytes large.
 func randBuf(size int) []byte {
 	buf := make([]byte, size*1024)
@@ -457,6 +518,97 @@ func (s *Suite) TestMaliciousHandshake(t *utesting.T) {
 	}
 }
 
+func (s *Suite) TestBlockRangeUpdateInvalid(t *utesting.T) {
+	t.Log(`This test sends an invalid BlockRangeUpdate message to the node and expects to be disconnected.`)
+	conn, err := s.dialAndPeer(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+
+	conn.Write(ethProto, eth.BlockRangeUpdateMsg, &eth.BlockRangeUpdatePacket{
+		EarliestBlock:   10,
+		LatestBlock:     8,
+		LatestBlockHash: s.chain.GetBlock(8).Hash(),
+	})
+
+	if code, _, err := conn.Read(); err != nil {
+		t.Fatalf("expected disconnect, got err: %v", err)
+	} else if code != discMsg {
+		t.Fatalf("expected disconnect message, got msg code %d", code)
+	}
+}
+
+func (s *Suite) TestBlockRangeUpdateFuture(t *utesting.T) {
+	t.Log(`This test sends a BlockRangeUpdate that is beyond the chain head.
+The node should accept the update and should not disonnect.`)
+	conn, err := s.dialAndPeer(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+
+	head := s.chain.Head().NumberU64()
+	var hash common.Hash
+	rand.Read(hash[:])
+	conn.Write(ethProto, eth.BlockRangeUpdateMsg, &eth.BlockRangeUpdatePacket{
+		EarliestBlock:   head + 10,
+		LatestBlock:     head + 50,
+		LatestBlockHash: hash,
+	})
+
+	// Ensure the node does not disconnect us.
+	// Just send a few ping messages.
+	for range 10 {
+		time.Sleep(100 * time.Millisecond)
+		if err := conn.Write(baseProto, pingMsg, []any{}); err != nil {
+			t.Fatal("write error:", err)
+		}
+		code, _, err := conn.Read()
+		switch {
+		case err != nil:
+			t.Fatal("read error:", err)
+		case code == discMsg:
+			t.Fatal("got disconnect")
+		case code == pongMsg:
+		}
+	}
+}
+
+func (s *Suite) TestBlockRangeUpdateHistoryExp(t *utesting.T) {
+	t.Log(`This test sends a BlockRangeUpdate announcing incomplete (expired) history.
+The node should accept the update and should not disonnect.`)
+	conn, err := s.dialAndPeer(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+
+	head := s.chain.Head()
+	conn.Write(ethProto, eth.BlockRangeUpdateMsg, &eth.BlockRangeUpdatePacket{
+		EarliestBlock:   head.NumberU64() - 10,
+		LatestBlock:     head.NumberU64(),
+		LatestBlockHash: head.Hash(),
+	})
+
+	// Ensure the node does not disconnect us.
+	// Just send a few ping messages.
+	for range 10 {
+		time.Sleep(100 * time.Millisecond)
+		if err := conn.Write(baseProto, pingMsg, []any{}); err != nil {
+			t.Fatal("write error:", err)
+		}
+		code, _, err := conn.Read()
+		switch {
+		case err != nil:
+			t.Fatal("read error:", err)
+		case code == discMsg:
+			t.Fatal("got disconnect")
+		case code == pongMsg:
+		}
+	}
+}
+
 func (s *Suite) TestTransaction(t *utesting.T) {
 	t.Log(`This test sends a valid transaction to the node and checks if the
 transaction gets propagated.`)

cmd/devp2p/internal/v5test/discv5tests.go

@@ -59,8 +59,9 @@ func (s *Suite) AllTests() []utesting.Test {
 	}
 }
 
-// TestPing sends PING and expects a PONG response.
 func (s *Suite) TestPing(t *utesting.T) {
+	t.Log(`This test is just a sanity check. It sends PING and expects a PONG response.`)
+
 	conn, l1 := s.listen1(t)
 	defer conn.close()
 
@@ -85,9 +86,10 @@ func checkPong(t *utesting.T, pong *v5wire.Pong, ping *v5wire.Ping, c net.Packet
 	}
 }
 
-// TestPingLargeRequestID sends PING with a 9-byte request ID, which isn't allowed by the spec.
-// The remote node should not respond.
 func (s *Suite) TestPingLargeRequestID(t *utesting.T) {
+	t.Log(`This test sends PING with a 9-byte request ID, which isn't allowed by the spec.
+The remote node should not respond.`)
+
 	conn, l1 := s.listen1(t)
 	defer conn.close()
 
@@ -104,10 +106,11 @@ func (s *Suite) TestPingLargeRequestID(t *utesting.T) {
 	}
 }
 
-// TestPingMultiIP establishes a session from one IP as usual. The session is then reused
-// on another IP, which shouldn't work. The remote node should respond with WHOAREYOU for
-// the attempt from a different IP.
 func (s *Suite) TestPingMultiIP(t *utesting.T) {
+	t.Log(`This test establishes a session from one IP as usual. The session is then reused
+on another IP, which shouldn't work. The remote node should respond with WHOAREYOU for
+the attempt from a different IP.`)
+
 	conn, l1, l2 := s.listen2(t)
 	defer conn.close()
 
@@ -120,6 +123,7 @@ func (s *Suite) TestPingMultiIP(t *utesting.T) {
 	checkPong(t, resp.(*v5wire.Pong), ping, l1)
 
 	// Send on l2. This reuses the session because there is only one codec.
+	t.Log("sending ping from alternate IP", l2.LocalAddr())
 	ping2 := &v5wire.Ping{ReqID: conn.nextReqID()}
 	conn.write(l2, ping2, nil)
 	switch resp := conn.read(l2).(type) {
@@ -158,6 +162,10 @@ func (s *Suite) TestPingMultiIP(t *utesting.T) {
 // packet instead of a handshake message packet. The remote node should respond with
 // another WHOAREYOU challenge for the second packet.
 func (s *Suite) TestPingHandshakeInterrupted(t *utesting.T) {
+	t.Log(`TestPingHandshakeInterrupted starts a handshake, but doesn't finish it and sends a second ordinary message
+packet instead of a handshake message packet. The remote node should respond with
+another WHOAREYOU challenge for the second packet.`)
+
 	conn, l1 := s.listen1(t)
 	defer conn.close()
 
@@ -181,8 +189,10 @@ func (s *Suite) TestPingHandshakeInterrupted(t *utesting.T) {
 	}
 }
 
-// TestTalkRequest sends TALKREQ and expects an empty TALKRESP response.
 func (s *Suite) TestTalkRequest(t *utesting.T) {
+	t.Log(`This test sends some examples of TALKREQ with a protocol-id of "test-protocol"
+and expects an empty TALKRESP response.`)
+
 	conn, l1 := s.listen1(t)
 	defer conn.close()
 
@@ -202,6 +212,7 @@ func (s *Suite) TestTalkRequest(t *utesting.T) {
 	}
 
 	// Empty request ID.
+	t.Log("sending TALKREQ with empty request-id")
 	resp = conn.reqresp(l1, &v5wire.TalkRequest{Protocol: "test-protocol"})
 	switch resp := resp.(type) {
 	case *v5wire.TalkResponse:
@@ -216,8 +227,9 @@ func (s *Suite) TestTalkRequest(t *utesting.T) {
 	}
 }
 
-// TestFindnodeZeroDistance checks that the remote node returns itself for FINDNODE with distance zero.
 func (s *Suite) TestFindnodeZeroDistance(t *utesting.T) {
+	t.Log(`This test checks that the remote node returns itself for FINDNODE with distance zero.`)
+
 	conn, l1 := s.listen1(t)
 	defer conn.close()
 
@@ -233,9 +245,11 @@ func (s *Suite) TestFindnodeZeroDistance(t *utesting.T) {
 	}
 }
 
-// TestFindnodeResults pings the node under test from multiple nodes. After waiting for them to be
-// accepted into the remote table, the test checks that they are returned by FINDNODE.
 func (s *Suite) TestFindnodeResults(t *utesting.T) {
+	t.Log(`This test pings the node under test from multiple other endpoints and node identities
+(the 'bystanders'). After waiting for them to be accepted into the remote table, the test checks
+that they are returned by FINDNODE.`)
+
 	// Create bystanders.
 	nodes := make([]*bystander, 5)
 	added := make(chan enode.ID, len(nodes))
@@ -273,6 +287,7 @@ func (s *Suite) TestFindnodeResults(t *utesting.T) {
 	}
 
 	// Send FINDNODE for all distances.
+	t.Log("requesting nodes")
 	conn, l1 := s.listen1(t)
 	defer conn.close()
 	foundNodes, err := conn.findnode(l1, dists)

cmd/evm/README.md

@@ -16,7 +16,7 @@ which can
 1. Take a prestate, including
    - Accounts,
    - Block context information,
-   - Previous blockshashes (*optional)
+   - Previous block hashes (*optional)
 2. Apply a set of transactions,
 3. Apply a mining-reward (*optional),
 4. And generate a post-state, including

cmd/evm/eest.go

@@ -22,7 +22,7 @@ import "regexp"
 // within its filename by the execution spec test (EEST).
 type testMetadata struct {
 	fork       string
-	module     string // which python module gnerated the test, e.g. eip7702
+	module     string // which python module generated the test, e.g. eip7702
 	file       string // exact file the test came from, e.g. test_gas.py
 	function   string // func that created the test, e.g. test_valid_mcopy_operations
 	parameters string // the name of the parameters which were used to fill the test, e.g. zero_inputs

cmd/evm/eofparse.go

@@ -1,228 +0,0 @@
-// Copyright 2024 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-import (
-	"bufio"
-	"encoding/hex"
-	"encoding/json"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/urfave/cli/v2"
-)
-
-var jt vm.JumpTable
-
-const initcode = "INITCODE"
-
-func init() {
-	jt = vm.NewEOFInstructionSetForTesting()
-}
-
-var (
-	hexFlag = &cli.StringFlag{
-		Name:  "hex",
-		Usage: "Single container data parse and validation",
-	}
-	refTestFlag = &cli.StringFlag{
-		Name:  "test",
-		Usage: "Path to EOF validation reference test.",
-	}
-	eofParseCommand = &cli.Command{
-		Name:    "eofparse",
-		Aliases: []string{"eof"},
-		Usage:   "Parses hex eof container and returns validation errors (if any)",
-		Action:  eofParseAction,
-		Flags: []cli.Flag{
-			hexFlag,
-			refTestFlag,
-		},
-	}
-	eofDumpCommand = &cli.Command{
-		Name:   "eofdump",
-		Usage:  "Parses hex eof container and prints out human-readable representation of the container.",
-		Action: eofDumpAction,
-		Flags: []cli.Flag{
-			hexFlag,
-		},
-	}
-)
-
-func eofParseAction(ctx *cli.Context) error {
-	// If `--test` is set, parse and validate the reference test at the provided path.
-	if ctx.IsSet(refTestFlag.Name) {
-		var (
-			file          = ctx.String(refTestFlag.Name)
-			executedTests int
-			passedTests   int
-		)
-		err := filepath.Walk(file, func(path string, info fs.FileInfo, err error) error {
-			if err != nil {
-				return err
-			}
-			if info.IsDir() {
-				return nil
-			}
-			log.Debug("Executing test", "name", info.Name())
-			passed, tot, err := executeTest(path)
-			passedTests += passed
-			executedTests += tot
-			return err
-		})
-		if err != nil {
-			return err
-		}
-		log.Info("Executed tests", "passed", passedTests, "total executed", executedTests)
-		return nil
-	}
-	// If `--hex` is set, parse and validate the hex string argument.
-	if ctx.IsSet(hexFlag.Name) {
-		if _, err := parseAndValidate(ctx.String(hexFlag.Name), false); err != nil {
-			return fmt.Errorf("err: %w", err)
-		}
-		fmt.Println("OK")
-		return nil
-	}
-	// If neither are passed in, read input from stdin.
-	scanner := bufio.NewScanner(os.Stdin)
-	scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
-	for scanner.Scan() {
-		l := strings.TrimSpace(scanner.Text())
-		if strings.HasPrefix(l, "#") || l == "" {
-			continue
-		}
-		if _, err := parseAndValidate(l, false); err != nil {
-			fmt.Printf("err: %v\n", err)
-		} else {
-			fmt.Println("OK")
-		}
-	}
-	if err := scanner.Err(); err != nil {
-		fmt.Println(err.Error())
-	}
-	return nil
-}
-
-type refTests struct {
-	Vectors map[string]eOFTest `json:"vectors"`
-}
-
-type eOFTest struct {
-	Code          string              `json:"code"`
-	Results       map[string]etResult `json:"results"`
-	ContainerKind string              `json:"containerKind"`
-}
-
-type etResult struct {
-	Result    bool   `json:"result"`
-	Exception string `json:"exception,omitempty"`
-}
-
-func executeTest(path string) (int, int, error) {
-	src, err := os.ReadFile(path)
-	if err != nil {
-		return 0, 0, err
-	}
-	var testsByName map[string]refTests
-	if err := json.Unmarshal(src, &testsByName); err != nil {
-		return 0, 0, err
-	}
-	passed, total := 0, 0
-	for testsName, tests := range testsByName {
-		for name, tt := range tests.Vectors {
-			for fork, r := range tt.Results {
-				total++
-				_, err := parseAndValidate(tt.Code, tt.ContainerKind == initcode)
-				if r.Result && err != nil {
-					log.Error("Test failure, expected validation success", "name", testsName, "idx", name, "fork", fork, "err", err)
-					continue
-				}
-				if !r.Result && err == nil {
-					log.Error("Test failure, expected validation error", "name", testsName, "idx", name, "fork", fork, "have err", r.Exception, "err", err)
-					continue
-				}
-				passed++
-			}
-		}
-	}
-	return passed, total, nil
-}
-
-func parseAndValidate(s string, isInitCode bool) (*vm.Container, error) {
-	if len(s) >= 2 && strings.HasPrefix(s, "0x") {
-		s = s[2:]
-	}
-	b, err := hex.DecodeString(s)
-	if err != nil {
-		return nil, fmt.Errorf("unable to decode data: %w", err)
-	}
-	return parse(b, isInitCode)
-}
-
-func parse(b []byte, isInitCode bool) (*vm.Container, error) {
-	var c vm.Container
-	if err := c.UnmarshalBinary(b, isInitCode); err != nil {
-		return nil, err
-	}
-	if err := c.ValidateCode(&jt, isInitCode); err != nil {
-		return nil, err
-	}
-	return &c, nil
-}
-
-func eofDumpAction(ctx *cli.Context) error {
-	// If `--hex` is set, parse and validate the hex string argument.
-	if ctx.IsSet(hexFlag.Name) {
-		return eofDump(ctx.String(hexFlag.Name))
-	}
-	// Otherwise read from stdin
-	scanner := bufio.NewScanner(os.Stdin)
-	scanner.Buffer(make([]byte, 1024*1024), 10*1024*1024)
-	for scanner.Scan() {
-		l := strings.TrimSpace(scanner.Text())
-		if strings.HasPrefix(l, "#") || l == "" {
-			continue
-		}
-		if err := eofDump(l); err != nil {
-			return err
-		}
-		fmt.Println("")
-	}
-	return scanner.Err()
-}
-
-func eofDump(hexdata string) error {
-	if len(hexdata) >= 2 && strings.HasPrefix(hexdata, "0x") {
-		hexdata = hexdata[2:]
-	}
-	b, err := hex.DecodeString(hexdata)
-	if err != nil {
-		return fmt.Errorf("unable to decode data: %w", err)
-	}
-	var c vm.Container
-	if err := c.UnmarshalBinary(b, false); err != nil {
-		return err
-	}
-	fmt.Println(c.String())
-	return nil
-}

cmd/evm/eofparse_test.go

@@ -1,182 +0,0 @@
-// Copyright 2024 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/hex"
-	"fmt"
-	"os"
-	"strings"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/vm"
-)
-
-func FuzzEofParsing(f *testing.F) {
-	// Seed with corpus from execution-spec-tests
-	for i := 0; ; i++ {
-		fname := fmt.Sprintf("testdata/eof/eof_corpus_%d.txt", i)
-		corpus, err := os.Open(fname)
-		if err != nil {
-			break
-		}
-		f.Logf("Reading seed data from %v", fname)
-		scanner := bufio.NewScanner(corpus)
-		scanner.Buffer(make([]byte, 1024), 10*1024*1024)
-		for scanner.Scan() {
-			s := scanner.Text()
-			if len(s) >= 2 && strings.HasPrefix(s, "0x") {
-				s = s[2:]
-			}
-			b, err := hex.DecodeString(s)
-			if err != nil {
-				panic(err) // rotten corpus
-			}
-			f.Add(b)
-		}
-		corpus.Close()
-		if err := scanner.Err(); err != nil {
-			panic(err) // rotten corpus
-		}
-	}
-	// And do the fuzzing
-	f.Fuzz(func(t *testing.T, data []byte) {
-		var (
-			jt = vm.NewEOFInstructionSetForTesting()
-			c  vm.Container
-		)
-		cpy := common.CopyBytes(data)
-		if err := c.UnmarshalBinary(data, true); err == nil {
-			c.ValidateCode(&jt, true)
-			if have := c.MarshalBinary(); !bytes.Equal(have, data) {
-				t.Fatal("Unmarshal-> Marshal failure!")
-			}
-		}
-		if err := c.UnmarshalBinary(data, false); err == nil {
-			c.ValidateCode(&jt, false)
-			if have := c.MarshalBinary(); !bytes.Equal(have, data) {
-				t.Fatal("Unmarshal-> Marshal failure!")
-			}
-		}
-		if !bytes.Equal(cpy, data) {
-			panic("data modified during unmarshalling")
-		}
-	})
-}
-
-func TestEofParseInitcode(t *testing.T) {
-	testEofParse(t, true, "testdata/eof/results.initcode.txt")
-}
-
-func TestEofParseRegular(t *testing.T) {
-	testEofParse(t, false, "testdata/eof/results.regular.txt")
-}
-
-func testEofParse(t *testing.T, isInitCode bool, wantFile string) {
-	var wantFn func() string
-	var wantLoc = 0
-	{ // Configure the want-reader
-		wants, err := os.Open(wantFile)
-		if err != nil {
-			t.Fatal(err)
-		}
-		scanner := bufio.NewScanner(wants)
-		scanner.Buffer(make([]byte, 1024), 10*1024*1024)
-		wantFn = func() string {
-			if scanner.Scan() {
-				wantLoc++
-				return scanner.Text()
-			}
-			return "end of file reached"
-		}
-	}
-
-	for i := 0; ; i++ {
-		fname := fmt.Sprintf("testdata/eof/eof_corpus_%d.txt", i)
-		corpus, err := os.Open(fname)
-		if err != nil {
-			break
-		}
-		t.Logf("# Reading seed data from %v", fname)
-		scanner := bufio.NewScanner(corpus)
-		scanner.Buffer(make([]byte, 1024), 10*1024*1024)
-		line := 1
-		for scanner.Scan() {
-			s := scanner.Text()
-			if len(s) >= 2 && strings.HasPrefix(s, "0x") {
-				s = s[2:]
-			}
-			b, err := hex.DecodeString(s)
-			if err != nil {
-				panic(err) // rotten corpus
-			}
-			have := "OK"
-			if _, err := parse(b, isInitCode); err != nil {
-				have = fmt.Sprintf("ERR: %v", err)
-			}
-			if false { // Change this to generate the want-output
-				fmt.Printf("%v\n", have)
-			} else {
-				want := wantFn()
-				if have != want {
-					if len(want) > 100 {
-						want = want[:100]
-					}
-					if len(b) > 100 {
-						b = b[:100]
-					}
-					t.Errorf("%v:%d\n%v\ninput %x\nisInit: %v\nhave: %q\nwant: %q\n",
-						fname, line, fmt.Sprintf("%v:%d", wantFile, wantLoc), b, isInitCode, have, want)
-				}
-			}
-			line++
-		}
-		corpus.Close()
-	}
-}
-
-func BenchmarkEofParse(b *testing.B) {
-	corpus, err := os.Open("testdata/eof/eof_benches.txt")
-	if err != nil {
-		b.Fatal(err)
-	}
-	defer corpus.Close()
-	scanner := bufio.NewScanner(corpus)
-	scanner.Buffer(make([]byte, 1024), 10*1024*1024)
-	line := 1
-	for scanner.Scan() {
-		s := scanner.Text()
-		if len(s) >= 2 && strings.HasPrefix(s, "0x") {
-			s = s[2:]
-		}
-		data, err := hex.DecodeString(s)
-		if err != nil {
-			b.Fatal(err) // rotten corpus
-		}
-		b.Run(fmt.Sprintf("test-%d", line), func(b *testing.B) {
-			b.ReportAllocs()
-			b.SetBytes(int64(len(data)))
-			for i := 0; i < b.N; i++ {
-				_, _ = parse(data, false)
-			}
-		})
-		line++
-	}
-}

cmd/evm/internal/t8ntool/execution.go

@@ -303,7 +303,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 			}
 
 			// Set the receipt logs and create the bloom filter.
-			receipt.Logs = statedb.GetLogs(tx.Hash(), vmContext.BlockNumber.Uint64(), blockHash)
+			receipt.Logs = statedb.GetLogs(tx.Hash(), vmContext.BlockNumber.Uint64(), blockHash, vmContext.Time)
 			receipt.Bloom = types.CreateBloom(receipt)
 
 			// These three are non-consensus fields:
@@ -363,9 +363,13 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 			return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not parse requests logs: %v", err))
 		}
 		// EIP-7002
-		core.ProcessWithdrawalQueue(&requests, evm)
+		if err := core.ProcessWithdrawalQueue(&requests, evm); err != nil {
+			return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not process withdrawal requests: %v", err))
+		}
 		// EIP-7251
-		core.ProcessConsolidationQueue(&requests, evm)
+		if err := core.ProcessConsolidationQueue(&requests, evm); err != nil {
+			return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not process consolidation requests: %v", err))
+		}
 	}
 
 	// Commit block

cmd/evm/internal/t8ntool/tx_iterator.go

@@ -102,7 +102,7 @@ func signUnsignedTransactions(txs []*txWithKey, signer types.Signer) (types.Tran
 		if tx.protected {
 			signed, err = types.SignTx(tx.tx, signer, tx.key)
 		} else {
-			signed, err = types.SignTx(tx.tx, types.FrontierSigner{}, tx.key)
+			signed, err = types.SignTx(tx.tx, types.HomesteadSigner{}, tx.key)
 		}
 		if err != nil {
 			return nil, NewError(ErrorJson, fmt.Errorf("tx %d: failed to sign tx: %v", i, err))

cmd/evm/main.go

@@ -210,8 +210,6 @@ func init() {
 		stateTransitionCommand,
 		transactionCommand,
 		blockBuilderCommand,
-		eofParseCommand,
-		eofDumpCommand,
 	}
 	app.Before = func(ctx *cli.Context) error {
 		flags.MigrateGlobalFlags(ctx)

cmd/geth/chaincmd.go

@@ -21,9 +21,12 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
+	"regexp"
 	"runtime"
 	"slices"
 	"strconv"
+	"strings"
 	"sync/atomic"
 	"time"
 
@@ -31,13 +34,18 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/history"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/internal/era"
+	"github.com/ethereum/go-ethereum/internal/era/eradl"
+	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/urfave/cli/v2"
 )
@@ -65,7 +73,7 @@ It expects the genesis file as argument.`,
 		Name:      "dumpgenesis",
 		Usage:     "Dumps genesis block JSON configuration to stdout",
 		ArgsUsage: "",
-		Flags:     append([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags...),
+		Flags:     slices.Concat([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags),
 		Description: `
 The dumpgenesis command prints the genesis configuration of the network preset
 if one is set.  Otherwise it prints the genesis from the datadir.`,
@@ -76,12 +84,16 @@ if one is set.  Otherwise it prints the genesis from the datadir.`,
 		Usage:     "Import a blockchain file",
 		ArgsUsage: "<filename> (<filename 2> ... <filename N>) ",
 		Flags: slices.Concat([]cli.Flag{
-			utils.CacheFlag,
-			utils.SyncModeFlag,
 			utils.GCModeFlag,
 			utils.SnapshotFlag,
+			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
+			utils.CacheTrieFlag,
 			utils.CacheGCFlag,
+			utils.CacheSnapshotFlag,
+			utils.CacheNoPrefetchFlag,
+			utils.CachePreimagesFlag,
+			utils.NoCompactionFlag,
 			utils.MetricsEnabledFlag,
 			utils.MetricsEnabledExpensiveFlag,
 			utils.MetricsHTTPFlag,
@@ -104,7 +116,11 @@ if one is set.  Otherwise it prints the genesis from the datadir.`,
 			utils.LogNoHistoryFlag,
 			utils.LogExportCheckpointsFlag,
 			utils.StateHistoryFlag,
-		}, utils.DatabaseFlags),
+		}, utils.DatabaseFlags, debug.Flags),
+		Before: func(ctx *cli.Context) error {
+			flags.MigrateGlobalFlags(ctx)
+			return debug.Setup(ctx)
+		},
 		Description: `
 The import command allows the import of blocks from an RLP-encoded format. This format can be a single file
 containing multiple RLP-encoded blocks, or multiple files can be given.
@@ -118,10 +134,7 @@ to import successfully.`,
 		Name:      "export",
 		Usage:     "Export blockchain into file",
 		ArgsUsage: "<filename> [<blockNumFirst> <blockNumLast>]",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:     slices.Concat([]cli.Flag{utils.CacheFlag}, utils.DatabaseFlags),
-			utils.CacheFlag,
-			utils.SyncModeFlag,
-		}, utils.DatabaseFlags),
 		Description: `
 Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
@@ -134,12 +147,7 @@ be gzipped.`,
 		Name:      "import-history",
 		Usage:     "Import an Era archive",
 		ArgsUsage: "<dir>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:     slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag}, utils.DatabaseFlags, utils.NetworkFlags),
-			utils.TxLookupLimitFlag,
-		},
-			utils.DatabaseFlags,
-			utils.NetworkFlags,
-		),
 		Description: `
 The import-history command will import blocks and their corresponding receipts
 from Era archives.
@@ -150,7 +158,7 @@ from Era archives.
 		Name:      "export-history",
 		Usage:     "Export blockchain history to Era archives",
 		ArgsUsage: "<dir> <first> <last>",
-		Flags:     slices.Concat(utils.DatabaseFlags),
+		Flags:     utils.DatabaseFlags,
 		Description: `
 The export-history command will export blocks and their corresponding receipts
 into Era archives. Eras are typically packaged in steps of 8192 blocks.
@@ -161,10 +169,7 @@ into Era archives. Eras are typically packaged in steps of 8192 blocks.
 		Name:      "import-preimages",
 		Usage:     "Import the preimage database from an RLP stream",
 		ArgsUsage: "<datafile>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:     slices.Concat([]cli.Flag{utils.CacheFlag}, utils.DatabaseFlags),
-			utils.CacheFlag,
-			utils.SyncModeFlag,
-		}, utils.DatabaseFlags),
 		Description: `
 The import-preimages command imports hash preimages from an RLP encoded stream.
 It's deprecated, please use "geth db import" instead.
@@ -189,6 +194,54 @@ It's deprecated, please use "geth db import" instead.
 This command dumps out the state for a given block (or latest, if none provided).
 `,
 	}
+
+	pruneHistoryCommand = &cli.Command{
+		Action:    pruneHistory,
+		Name:      "prune-history",
+		Usage:     "Prune blockchain history (block bodies and receipts) up to the merge block",
+		ArgsUsage: "",
+		Flags:     utils.DatabaseFlags,
+		Description: `
+The prune-history command removes historical block bodies and receipts from the
+blockchain database up to the merge block, while preserving block headers. This
+helps reduce storage requirements for nodes that don't need full historical data.`,
+	}
+
+	downloadEraCommand = &cli.Command{
+		Action:    downloadEra,
+		Name:      "download-era",
+		Usage:     "Fetches era1 files (pre-merge history) from an HTTP endpoint",
+		ArgsUsage: "",
+		Flags: slices.Concat(
+			utils.DatabaseFlags,
+			utils.NetworkFlags,
+			[]cli.Flag{
+				eraBlockFlag,
+				eraEpochFlag,
+				eraAllFlag,
+				eraServerFlag,
+			},
+		),
+	}
+)
+
+var (
+	eraBlockFlag = &cli.StringFlag{
+		Name:  "block",
+		Usage: "Block number to fetch. (can also be a range <start>-<end>)",
+	}
+	eraEpochFlag = &cli.StringFlag{
+		Name:  "epoch",
+		Usage: "Epoch number to fetch (can also be a range <start>-<end>)",
+	}
+	eraAllFlag = &cli.BoolFlag{
+		Name:  "all",
+		Usage: "Download all available era1 files",
+	}
+	eraServerFlag = &cli.StringFlag{
+		Name:  "server",
+		Usage: "era1 server URL",
+	}
 )
 
 // initGenesis will initialise the given JSON format genesis file and writes it as
@@ -225,19 +278,19 @@ func initGenesis(ctx *cli.Context) error {
 		overrides.OverrideVerkle = &v
 	}
 
-	chaindb, err := stack.OpenDatabaseWithFreezer("chaindata", 0, 0, ctx.String(utils.AncientFlag.Name), "", false)
+	chaindb := utils.MakeChainDatabase(ctx, stack, false)
-	if err != nil {
-		utils.Fatalf("Failed to open database: %v", err)
-	}
 	defer chaindb.Close()
 
 	triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle())
 	defer triedb.Close()
 
-	_, hash, _, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides)
+	_, hash, compatErr, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides)
 	if err != nil {
 		utils.Fatalf("Failed to write genesis block: %v", err)
 	}
+	if compatErr != nil {
+		utils.Fatalf("Failed to write chain config: %v", compatErr)
+	}
 	log.Info("Successfully wrote genesis state", "database", "chaindata", "hash", hash)
 
 	return nil
@@ -262,7 +315,7 @@ func dumpGenesis(ctx *cli.Context) error {
 	// dump whatever already exists in the datadir
 	stack, _ := makeConfigNode(ctx)
 
-	db, err := stack.OpenDatabase("chaindata", 0, 0, "", true)
+	db, err := stack.OpenDatabaseWithOptions("chaindata", node.DatabaseOptions{ReadOnly: true})
 	if err != nil {
 		return err
 	}
@@ -422,6 +475,10 @@ func importHistory(ctx *cli.Context) error {
 			network = "mainnet"
 		case ctx.Bool(utils.SepoliaFlag.Name):
 			network = "sepolia"
+		case ctx.Bool(utils.HoleskyFlag.Name):
+			network = "holesky"
+		case ctx.Bool(utils.HoodiFlag.Name):
+			network = "hoodi"
 		}
 	} else {
 		// No network flag set, try to determine network based on files
@@ -445,7 +502,7 @@ func importHistory(ctx *cli.Context) error {
 		network = networks[0]
 	}
 
-	if err := utils.ImportHistory(chain, db, dir, network); err != nil {
+	if err := utils.ImportHistory(chain, dir, network); err != nil {
 		return err
 	}
 	fmt.Printf("Import done in %v\n", time.Since(start))
@@ -598,3 +655,135 @@ func hashish(x string) bool {
 	_, err := strconv.Atoi(x)
 	return err != nil
 }
+
+func pruneHistory(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	// Open the chain database
+	chain, chaindb := utils.MakeChain(ctx, stack, false)
+	defer chaindb.Close()
+	defer chain.Stop()
+
+	// Determine the prune point. This will be the first PoS block.
+	prunePoint, ok := history.PrunePoints[chain.Genesis().Hash()]
+	if !ok || prunePoint == nil {
+		return errors.New("prune point not found")
+	}
+	var (
+		mergeBlock     = prunePoint.BlockNumber
+		mergeBlockHash = prunePoint.BlockHash.Hex()
+	)
+
+	// Check we're far enough past merge to ensure all data is in freezer
+	currentHeader := chain.CurrentHeader()
+	if currentHeader == nil {
+		return errors.New("current header not found")
+	}
+	if currentHeader.Number.Uint64() < mergeBlock+params.FullImmutabilityThreshold {
+		return fmt.Errorf("chain not far enough past merge block, need %d more blocks",
+			mergeBlock+params.FullImmutabilityThreshold-currentHeader.Number.Uint64())
+	}
+
+	// Double-check the prune block in db has the expected hash.
+	hash := rawdb.ReadCanonicalHash(chaindb, mergeBlock)
+	if hash != common.HexToHash(mergeBlockHash) {
+		return fmt.Errorf("merge block hash mismatch: got %s, want %s", hash.Hex(), mergeBlockHash)
+	}
+
+	log.Info("Starting history pruning", "head", currentHeader.Number, "tail", mergeBlock, "tailHash", mergeBlockHash)
+	start := time.Now()
+	rawdb.PruneTransactionIndex(chaindb, mergeBlock)
+	if _, err := chaindb.TruncateTail(mergeBlock); err != nil {
+		return fmt.Errorf("failed to truncate ancient data: %v", err)
+	}
+	log.Info("History pruning completed", "tail", mergeBlock, "elapsed", common.PrettyDuration(time.Since(start)))
+
+	// TODO(s1na): what if there is a crash between the two prune operations?
+
+	return nil
+}
+
+// downladEra is the era1 file downloader tool.
+func downloadEra(ctx *cli.Context) error {
+	flags.CheckExclusive(ctx, eraBlockFlag, eraEpochFlag, eraAllFlag)
+
+	// Resolve the network.
+	var network = "mainnet"
+	if utils.IsNetworkPreset(ctx) {
+		switch {
+		case ctx.IsSet(utils.MainnetFlag.Name):
+		case ctx.IsSet(utils.SepoliaFlag.Name):
+			network = "sepolia"
+		default:
+			return fmt.Errorf("unsupported network, no known era1 checksums")
+		}
+	}
+
+	// Resolve the destination directory.
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	ancients := stack.ResolveAncient("chaindata", "")
+	dir := filepath.Join(ancients, rawdb.ChainFreezerName, "era")
+	if ctx.IsSet(utils.EraFlag.Name) {
+		dir = filepath.Join(ancients, ctx.String(utils.EraFlag.Name))
+	}
+
+	baseURL := ctx.String(eraServerFlag.Name)
+	if baseURL == "" {
+		return fmt.Errorf("need --%s flag to download", eraServerFlag.Name)
+	}
+
+	l, err := eradl.New(baseURL, network)
+	if err != nil {
+		return err
+	}
+	switch {
+	case ctx.IsSet(eraAllFlag.Name):
+		return l.DownloadAll(dir)
+
+	case ctx.IsSet(eraBlockFlag.Name):
+		s := ctx.String(eraBlockFlag.Name)
+		start, end, ok := parseRange(s)
+		if !ok {
+			return fmt.Errorf("invalid block range: %q", s)
+		}
+		return l.DownloadBlockRange(start, end, dir)
+
+	case ctx.IsSet(eraEpochFlag.Name):
+		s := ctx.String(eraEpochFlag.Name)
+		start, end, ok := parseRange(s)
+		if !ok {
+			return fmt.Errorf("invalid epoch range: %q", s)
+		}
+		return l.DownloadEpochRange(start, end, dir)
+
+	default:
+		return fmt.Errorf("specify one of --%s, --%s, or --%s to download", eraAllFlag.Name, eraBlockFlag.Name, eraEpochFlag.Name)
+	}
+}
+
+func parseRange(s string) (start uint64, end uint64, ok bool) {
+	if m, _ := regexp.MatchString("[0-9]+", s); m {
+		start, err := strconv.ParseUint(s, 10, 64)
+		if err != nil {
+			return 0, 0, false
+		}
+		end = start
+		return start, end, true
+	}
+	if m, _ := regexp.MatchString("[0-9]+-[0-9]+", s); m {
+		s1, s2, _ := strings.Cut(s, "-")
+		start, err := strconv.ParseUint(s1, 10, 64)
+		if err != nil {
+			return 0, 0, false
+		}
+		end, err = strconv.ParseUint(s2, 10, 64)
+		if err != nil {
+			return 0, 0, false
+		}
+		return start, end, true
+	}
+	return 0, 0, false
+}

cmd/geth/config.go

@@ -36,6 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/catalyst"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/internal/flags"
@@ -180,6 +181,45 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) {
 	return stack, cfg
 }
 
+// constructs the disclaimer text block which will be printed in the logs upon
+// startup when Geth is running in dev mode.
+func constructDevModeBanner(ctx *cli.Context, cfg gethConfig) string {
+	devModeBanner := `You are running Geth in --dev mode. Please note the following:
+
+  1. This mode is only intended for fast, iterative development without assumptions on
+     security or persistence.
+  2. The database is created in memory unless specified otherwise. Therefore, shutting down
+     your computer or losing power will wipe your entire block data and chain state for
+     your dev environment.
+  3. A random, pre-allocated developer account will be available and unlocked as
+     eth.coinbase, which can be used for testing. The random dev account is temporary,
+     stored on a ramdisk, and will be lost if your machine is restarted.
+  4. Mining is enabled by default. However, the client will only seal blocks if transactions
+     are pending in the mempool. The miner's minimum accepted gas price is 1.
+  5. Networking is disabled; there is no listen-address, the maximum number of peers is set
+     to 0, and discovery is disabled.
+`
+	if !ctx.IsSet(utils.DataDirFlag.Name) {
+		devModeBanner += fmt.Sprintf(`
+
+ Running in ephemeral mode.  The following account has been prefunded in the genesis:
+
+       Account
+       ------------------
+       0x%x (10^49 ETH)
+`, cfg.Eth.Miner.PendingFeeRecipient)
+		if cfg.Eth.Miner.PendingFeeRecipient == utils.DeveloperAddr {
+			devModeBanner += fmt.Sprintf(` 
+       Private Key
+       ------------------
+       0x%x
+`, crypto.FromECDSA(utils.DeveloperKey))
+		}
+	}
+
+	return devModeBanner
+}
+
 // makeFullNode loads geth configuration and creates the Ethereum backend.
 func makeFullNode(ctx *cli.Context) *node.Node {
 	stack, cfg := makeConfigNode(ctx)
@@ -239,6 +279,11 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 		}
 		catalyst.RegisterSimulatedBeaconAPIs(stack, simBeacon)
 		stack.RegisterLifecycle(simBeacon)
+
+		banner := constructDevModeBanner(ctx, cfg)
+		for _, line := range strings.Split(banner, "\n") {
+			log.Warn(line)
+		}
 	} else if ctx.IsSet(utils.BeaconApiFlag.Name) {
 		// Start blsync mode.
 		srv := rpc.NewServer()

cmd/geth/consolecmd_test.go

@@ -39,9 +39,8 @@ const (
 // child g gets a temporary data directory.
 func runMinimalGeth(t *testing.T, args ...string) *testgeth {
 	// --holesky to make the 'writing genesis to disk' faster (no accounts)
-	// --networkid=1337 to avoid cache bump
 	// --syncmode=full to avoid allocating fast sync bloom
-	allArgs := []string{"--holesky", "--networkid", "1337", "--authrpc.port", "0", "--syncmode=full", "--port", "0",
+	allArgs := []string{"--holesky", "--authrpc.port", "0", "--syncmode=full", "--port", "0",
 		"--nat", "none", "--nodiscover", "--maxpeers", "0", "--cache", "64",
 		"--datadir.minfreedisk", "0"}
 	return runGeth(t, append(allArgs, args...)...)
@@ -103,17 +102,17 @@ func TestAttachWelcome(t *testing.T) {
 		"--http", "--http.port", httpPort,
 		"--ws", "--ws.port", wsPort)
 	t.Run("ipc", func(t *testing.T) {
-		waitForEndpoint(t, ipc, 4*time.Second)
+		waitForEndpoint(t, ipc, 2*time.Minute)
 		testAttachWelcome(t, geth, "ipc:"+ipc, ipcAPIs)
 	})
 	t.Run("http", func(t *testing.T) {
 		endpoint := "http://127.0.0.1:" + httpPort
-		waitForEndpoint(t, endpoint, 4*time.Second)
+		waitForEndpoint(t, endpoint, 2*time.Minute)
 		testAttachWelcome(t, geth, endpoint, httpAPIs)
 	})
 	t.Run("ws", func(t *testing.T) {
 		endpoint := "ws://127.0.0.1:" + wsPort
-		waitForEndpoint(t, endpoint, 4*time.Second)
+		waitForEndpoint(t, endpoint, 2*time.Minute)
 		testAttachWelcome(t, geth, endpoint, httpAPIs)
 	})
 	geth.Kill()

cmd/geth/dbcmd.go

@@ -89,9 +89,7 @@ Remove blockchain and state databases`,
 		Action:      inspect,
 		Name:        "inspect",
 		ArgsUsage:   "<prefix> <start>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Usage:       "Inspect the storage size for each type of data in the database",
 		Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
 	}
@@ -109,16 +107,13 @@ a data corruption.`,
 		Action: dbStats,
 		Name:   "stats",
 		Usage:  "Print leveldb statistics",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:  slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 	}
 	dbCompactCmd = &cli.Command{
 		Action: dbCompact,
 		Name:   "compact",
 		Usage:  "Compact leveldb database. WARNING: May take a very long time",
 		Flags: slices.Concat([]cli.Flag{
-			utils.SyncModeFlag,
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
 		}, utils.NetworkFlags, utils.DatabaseFlags),
@@ -131,9 +126,7 @@ corruption if it is aborted during execution'!`,
 		Name:        "get",
 		Usage:       "Show the value of a database key",
 		ArgsUsage:   "<hex-encoded key>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDeleteCmd = &cli.Command{
@@ -141,9 +134,7 @@ corruption if it is aborted during execution'!`,
 		Name:      "delete",
 		Usage:     "Delete a database key (WARNING: may corrupt your database)",
 		ArgsUsage: "<hex-encoded key>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:     slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: `This command deletes the specified database key from the database.
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -152,9 +143,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:      "put",
 		Usage:     "Set the value of a database key (WARNING: may corrupt your database)",
 		ArgsUsage: "<hex-encoded key> <hex-encoded value>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:     slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: `This command sets a given database key to the given value.
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -163,9 +152,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:        "dumptrie",
 		Usage:       "Show the storage key/values of a given storage trie",
 		ArgsUsage:   "<hex-encoded state root> <hex-encoded account hash> <hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDumpFreezerIndex = &cli.Command{
@@ -173,9 +160,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:        "freezer-index",
 		Usage:       "Dump out the index of a specific freezer table",
 		ArgsUsage:   "<freezer-type> <table-type> <start (int)> <end (int)>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "This command displays information about the freezer index.",
 	}
 	dbImportCmd = &cli.Command{
@@ -183,9 +168,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:        "import",
 		Usage:       "Imports leveldb-data from an exported RLP dump.",
 		ArgsUsage:   "<dumpfile> <start (optional)",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "The import command imports the specific chain data from an RLP encoded stream.",
 	}
 	dbExportCmd = &cli.Command{
@@ -193,18 +176,14 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:        "export",
 		Usage:       "Exports the chain data into an RLP dump. If the <dumpfile> has .gz suffix, gzip compression will be used.",
 		ArgsUsage:   "<type> <dumpfile>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
 	}
 	dbMetadataCmd = &cli.Command{
 		Action:      showMetaData,
 		Name:        "metadata",
 		Usage:       "Shows metadata about the chain status.",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Shows metadata about the chain status.",
 	}
 	dbInspectHistoryCmd = &cli.Command{
@@ -213,7 +192,6 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Usage:     "Inspect the state history within block range",
 		ArgsUsage: "<address> [OPTIONAL <storage-slot>]",
 		Flags: slices.Concat([]cli.Flag{
-			utils.SyncModeFlag,
 			&cli.Uint64Flag{
 				Name:  "start",
 				Usage: "block number of the range start, zero means earliest history",

cmd/geth/main.go

@@ -91,16 +91,9 @@ var (
 		utils.LogNoHistoryFlag,
 		utils.LogExportCheckpointsFlag,
 		utils.StateHistoryFlag,
-		utils.LightServeFlag,    // deprecated
-		utils.LightIngressFlag,  // deprecated
-		utils.LightEgressFlag,   // deprecated
-		utils.LightMaxPeersFlag, // deprecated
-		utils.LightNoPruneFlag,  // deprecated
 		utils.LightKDFFlag,
-		utils.LightNoSyncServeFlag, // deprecated
 		utils.EthRequiredBlocksFlag,
 		utils.LegacyWhitelistFlag, // deprecated
-		utils.BloomFilterSizeFlag,
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
 		utils.CacheTrieFlag,
@@ -142,7 +135,6 @@ var (
 		utils.VMTraceJsonConfigFlag,
 		utils.NetworkIdFlag,
 		utils.EthStatsURLFlag,
-		utils.NoCompactionFlag,
 		utils.GpoBlocksFlag,
 		utils.GpoPercentileFlag,
 		utils.GpoMaxGasPriceFlag,
@@ -158,6 +150,7 @@ var (
 		utils.BeaconGenesisRootFlag,
 		utils.BeaconGenesisTimeFlag,
 		utils.BeaconCheckpointFlag,
+		utils.BeaconCheckpointFileFlag,
 	}, utils.NetworkFlags, utils.DatabaseFlags)
 
 	rpcFlags = []cli.Flag{
@@ -226,6 +219,8 @@ func init() {
 		removedbCommand,
 		dumpCommand,
 		dumpGenesisCommand,
+		pruneHistoryCommand,
+		downloadEraCommand,
 		// See accountcmd.go:
 		accountCommand,
 		walletCommand,
@@ -299,24 +294,6 @@ func prepare(ctx *cli.Context) {
 	case ctx.IsSet(utils.HoodiFlag.Name):
 		log.Info("Starting Geth on Hoodi testnet...")
 
-	case ctx.IsSet(utils.DeveloperFlag.Name):
-		log.Info("Starting Geth in ephemeral dev mode...")
-		log.Warn(`You are running Geth in --dev mode. Please note the following:
-
-  1. This mode is only intended for fast, iterative development without assumptions on
-     security or persistence.
-  2. The database is created in memory unless specified otherwise. Therefore, shutting down
-     your computer or losing power will wipe your entire block data and chain state for
-     your dev environment.
-  3. A random, pre-allocated developer account will be available and unlocked as
-     eth.coinbase, which can be used for testing. The random dev account is temporary,
-     stored on a ramdisk, and will be lost if your machine is restarted.
-  4. Mining is enabled by default. However, the client will only seal blocks if transactions
-     are pending in the mempool. The miner's minimum accepted gas price is 1.
-  5. Networking is disabled; there is no listen-address, the maximum number of peers is set
-     to 0, and discovery is disabled.
-`)
-
 	case !ctx.IsSet(utils.NetworkIdFlag.Name):
 		log.Info("Starting Geth on Ethereum mainnet...")
 	}

cmd/utils/cmd.go

@@ -246,8 +246,9 @@ func readList(filename string) ([]string, error) {
 }
 
 // ImportHistory imports Era1 files containing historical block information,
-// starting from genesis.
+// starting from genesis. The assumption is held that the provided chain
-func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error {
+// segment in Era1 file should all be canonical and verified.
+func ImportHistory(chain *core.BlockChain, dir string, network string) error {
 	if chain.CurrentSnapBlock().Number.BitLen() != 0 {
 		return errors.New("history import only supported when starting from genesis")
 	}
@@ -308,12 +309,8 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ
 				if err != nil {
 					return fmt.Errorf("error reading receipts %d: %w", it.Number(), err)
 				}
-				if status, err := chain.HeaderChain().InsertHeaderChain([]*types.Header{block.Header()}, start); err != nil {
+				encReceipts := types.EncodeBlockReceiptLists([]types.Receipts{receipts})
-					return fmt.Errorf("error inserting header %d: %w", it.Number(), err)
+				if _, err := chain.InsertReceiptChain([]*types.Block{block}, encReceipts, 2^64-1); err != nil {
-				} else if status != core.CanonStatTy {
-					return fmt.Errorf("error inserting header %d, not canon: %v", it.Number(), status)
-				}
-				if _, err := chain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{receipts}, 2^64-1); err != nil {
 					return fmt.Errorf("error inserting body %d: %w", it.Number(), err)
 				}
 				imported += 1

cmd/utils/export_test.go

@@ -54,8 +54,8 @@ func (iter *testIterator) Next() (byte, []byte, []byte, bool) {
 	if iter.index == 42 {
 		iter.index += 1
 	}
-	return OpBatchAdd, []byte(fmt.Sprintf("key-%04d", iter.index)),
+	return OpBatchAdd, fmt.Appendf(nil, "key-%04d", iter.index),
-		[]byte(fmt.Sprintf("value %d", iter.index)), true
+		fmt.Appendf(nil, "value %d", iter.index), true
 }
 
 func (iter *testIterator) Release() {}
@@ -72,7 +72,7 @@ func testExport(t *testing.T, f string) {
 	}
 	// verify
 	for i := 0; i < 1000; i++ {
-		v, err := db.Get([]byte(fmt.Sprintf("key-%04d", i)))
+		v, err := db.Get(fmt.Appendf(nil, "key-%04d", i))
 		if (i < 5 || i == 42) && err == nil {
 			t.Fatalf("expected no element at idx %d, got '%v'", i, string(v))
 		}
@@ -85,7 +85,7 @@ func testExport(t *testing.T, f string) {
 			}
 		}
 	}
-	v, err := db.Get([]byte(fmt.Sprintf("key-%04d", 1000)))
+	v, err := db.Get(fmt.Appendf(nil, "key-%04d", 1000))
 	if err == nil {
 		t.Fatalf("expected no element at idx %d, got '%v'", 1000, string(v))
 	}
@@ -120,7 +120,7 @@ func (iter *deletionIterator) Next() (byte, []byte, []byte, bool) {
 	if iter.index == 42 {
 		iter.index += 1
 	}
-	return OpBatchDel, []byte(fmt.Sprintf("key-%04d", iter.index)), nil, true
+	return OpBatchDel, fmt.Appendf(nil, "key-%04d", iter.index), nil, true
 }
 
 func (iter *deletionIterator) Release() {}
@@ -132,14 +132,14 @@ func testDeletion(t *testing.T, f string) {
 	}
 	db := rawdb.NewMemoryDatabase()
 	for i := 0; i < 1000; i++ {
-		db.Put([]byte(fmt.Sprintf("key-%04d", i)), []byte(fmt.Sprintf("value %d", i)))
+		db.Put(fmt.Appendf(nil, "key-%04d", i), fmt.Appendf(nil, "value %d", i))
 	}
 	err = ImportLDBData(db, f, 5, make(chan struct{}))
 	if err != nil {
 		t.Fatal(err)
 	}
 	for i := 0; i < 1000; i++ {
-		v, err := db.Get([]byte(fmt.Sprintf("key-%04d", i)))
+		v, err := db.Get(fmt.Appendf(nil, "key-%04d", i))
 		if i < 5 || i == 42 {
 			if err != nil {
 				t.Fatalf("expected element at idx %d, got '%v'", i, err)

cmd/utils/flags.go

@@ -111,6 +111,11 @@ var (
 		Usage:    "Root directory for ancient data (default = inside chaindata)",
 		Category: flags.EthCategory,
 	}
+	EraFlag = &flags.DirectoryFlag{
+		Name:     "datadir.era",
+		Usage:    "Root directory for era1 history (default = inside ancient/chain)",
+		Category: flags.EthCategory,
+	}
 	MinFreeDiskSpaceFlag = &flags.DirectoryFlag{
 		Name:     "datadir.minfreedisk",
 		Usage:    "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)",
@@ -145,7 +150,7 @@ var (
 	}
 	SepoliaFlag = &cli.BoolFlag{
 		Name:     "sepolia",
-		Usage:    "Sepolia network: pre-configured proof-of-work test network",
+		Usage:    "Sepolia network: pre-configured proof-of-stake test network",
 		Category: flags.EthCategory,
 	}
 	HoleskyFlag = &cli.BoolFlag{
@@ -342,6 +347,11 @@ var (
 		Usage:    "Beacon chain weak subjectivity checkpoint block hash",
 		Category: flags.BeaconCategory,
 	}
+	BeaconCheckpointFileFlag = &cli.StringFlag{
+		Name:     "beacon.checkpoint.file",
+		Usage:    "Beacon chain weak subjectivity checkpoint import/export file",
+		Category: flags.BeaconCategory,
+	}
 	BlsyncApiFlag = &cli.StringFlag{
 		Name:     "blsync.engine.api",
 		Usage:    "Target EL engine API URL",
@@ -972,6 +982,7 @@ var (
 	DatabaseFlags = []cli.Flag{
 		DataDirFlag,
 		AncientFlag,
+		EraFlag,
 		RemoteDBFlag,
 		DBEngineFlag,
 		StateSchemeFlag,
@@ -979,6 +990,12 @@ var (
 	}
 )
 
+// default account to prefund when running Geth in dev mode
+var (
+	DeveloperKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	DeveloperAddr   = crypto.PubkeyToAddress(DeveloperKey.PublicKey)
+)
+
 // MakeDataDir retrieves the currently requested data directory, terminating
 // if none (or the empty string) is specified. If the node is starting a testnet,
 // then a subdirectory of the specified datadir will be used.
@@ -1240,28 +1257,6 @@ func setIPC(ctx *cli.Context, cfg *node.Config) {
 	}
 }
 
-// setLes shows the deprecation warnings for LES flags.
-func setLes(ctx *cli.Context, cfg *ethconfig.Config) {
-	if ctx.IsSet(LightServeFlag.Name) {
-		log.Warn("The light server has been deprecated, please remove this flag", "flag", LightServeFlag.Name)
-	}
-	if ctx.IsSet(LightIngressFlag.Name) {
-		log.Warn("The light server has been deprecated, please remove this flag", "flag", LightIngressFlag.Name)
-	}
-	if ctx.IsSet(LightEgressFlag.Name) {
-		log.Warn("The light server has been deprecated, please remove this flag", "flag", LightEgressFlag.Name)
-	}
-	if ctx.IsSet(LightMaxPeersFlag.Name) {
-		log.Warn("The light server has been deprecated, please remove this flag", "flag", LightMaxPeersFlag.Name)
-	}
-	if ctx.IsSet(LightNoPruneFlag.Name) {
-		log.Warn("The light server has been deprecated, please remove this flag", "flag", LightNoPruneFlag.Name)
-	}
-	if ctx.IsSet(LightNoSyncServeFlag.Name) {
-		log.Warn("The light server has been deprecated, please remove this flag", "flag", LightNoSyncServeFlag.Name)
-	}
-}
-
 // MakeDatabaseHandles raises out the number of allowed file handles per process
 // for Geth and returns half of the allowance to assign to the database.
 func MakeDatabaseHandles(max int) int {
@@ -1566,8 +1561,8 @@ func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) {
 
 // SetEthConfig applies eth-related command line flags to the config.
 func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
-	// Avoid conflicting network flags
+	// Avoid conflicting network flags, don't allow network id override on preset networks
-	flags.CheckExclusive(ctx, MainnetFlag, DeveloperFlag, SepoliaFlag, HoleskyFlag, HoodiFlag)
+	flags.CheckExclusive(ctx, MainnetFlag, DeveloperFlag, SepoliaFlag, HoleskyFlag, HoodiFlag, NetworkIdFlag)
 	flags.CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
 
 	// Set configurations from CLI flags
@@ -1577,7 +1572,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	setBlobPool(ctx, &cfg.BlobPool)
 	setMiner(ctx, &cfg.Miner)
 	setRequiredBlocks(ctx, cfg)
-	setLes(ctx, cfg)
 
 	// Cap the cache allowance and tune the garbage collector
 	mem, err := gopsutil.VirtualMemory()
@@ -1625,6 +1619,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.IsSet(AncientFlag.Name) {
 		cfg.DatabaseFreezer = ctx.String(AncientFlag.Name)
 	}
+	if ctx.IsSet(EraFlag.Name) {
+		cfg.DatabaseEra = ctx.String(EraFlag.Name)
+	}
 
 	if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
 		Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
@@ -1659,13 +1656,17 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		log.Warn("The flag --txlookuplimit is deprecated and will be removed, please use --history.transactions")
 		cfg.TransactionHistory = ctx.Uint64(TxLookupLimitFlag.Name)
 	}
-	if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 {
+	if ctx.String(GCModeFlag.Name) == "archive" {
+		if cfg.TransactionHistory != 0 {
 			cfg.TransactionHistory = 0
 			log.Warn("Disabled transaction unindexing for archive node")
+		}
 
+		if cfg.StateScheme != rawdb.HashScheme {
 			cfg.StateScheme = rawdb.HashScheme
 			log.Warn("Forcing hash state-scheme for archive mode")
 		}
+	}
 	if ctx.IsSet(LogHistoryFlag.Name) {
 		cfg.LogHistory = ctx.Uint64(LogHistoryFlag.Name)
 	}
@@ -1703,7 +1704,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		}
 	}
 	if ctx.IsSet(VMEnableDebugFlag.Name) {
-		// TODO(fjl): force-enable this in --dev mode
 		cfg.EnablePreimageRecording = ctx.Bool(VMEnableDebugFlag.Name)
 	}
 
@@ -1734,34 +1734,25 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	// Override any default configs for hard coded networks.
 	switch {
 	case ctx.Bool(MainnetFlag.Name):
-		if !ctx.IsSet(NetworkIdFlag.Name) {
 		cfg.NetworkId = 1
-		}
 		cfg.Genesis = core.DefaultGenesisBlock()
 		SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash)
 	case ctx.Bool(HoleskyFlag.Name):
-		if !ctx.IsSet(NetworkIdFlag.Name) {
 		cfg.NetworkId = 17000
-		}
 		cfg.Genesis = core.DefaultHoleskyGenesisBlock()
 		SetDNSDiscoveryDefaults(cfg, params.HoleskyGenesisHash)
 	case ctx.Bool(SepoliaFlag.Name):
-		if !ctx.IsSet(NetworkIdFlag.Name) {
 		cfg.NetworkId = 11155111
-		}
 		cfg.Genesis = core.DefaultSepoliaGenesisBlock()
 		SetDNSDiscoveryDefaults(cfg, params.SepoliaGenesisHash)
 	case ctx.Bool(HoodiFlag.Name):
-		if !ctx.IsSet(NetworkIdFlag.Name) {
 		cfg.NetworkId = 560048
-		}
 		cfg.Genesis = core.DefaultHoodiGenesisBlock()
 		SetDNSDiscoveryDefaults(cfg, params.HoodiGenesisHash)
 	case ctx.Bool(DeveloperFlag.Name):
-		if !ctx.IsSet(NetworkIdFlag.Name) {
 		cfg.NetworkId = 1337
-		}
 		cfg.SyncMode = ethconfig.FullSync
+		cfg.EnablePreimageRecording = true
 		// Create new developer account or reuse existing one
 		var (
 			developer  accounts.Account
@@ -1793,9 +1784,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		} else if accs := ks.Accounts(); len(accs) > 0 {
 			developer = ks.Accounts()[0]
 		} else {
-			developer, err = ks.NewAccount(passphrase)
+			developer, err = ks.ImportECDSA(DeveloperKey, passphrase)
 			if err != nil {
-				Fatalf("Failed to create developer account: %v", err)
+				Fatalf("Failed to import developer account: %v", err)
 			}
 		}
 		// Make sure the address is configured as fee recipient, otherwise
@@ -1810,14 +1801,18 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		}
 		log.Info("Using developer account", "address", developer.Address)
 
-		// Create a new developer genesis block or reuse existing one
+		// configure default developer genesis which will be used unless a
+		// datadir is specified and a chain is preexisting at that location.
 		cfg.Genesis = core.DeveloperGenesisBlock(ctx.Uint64(DeveloperGasLimitFlag.Name), &developer.Address)
+
+		// If a datadir is specified, ensure that any preexisting chain in that location
+		// has a configuration that is compatible with dev mode: it must be merged at genesis.
 		if ctx.IsSet(DataDirFlag.Name) {
 			chaindb := tryMakeReadOnlyDatabase(ctx, stack)
 			if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) {
-				cfg.Genesis = nil // fallback to db content
+				// signal fallback to preexisting chain on disk
+				cfg.Genesis = nil
 
-				//validate genesis has PoS enabled in block 0
 				genesis, err := core.ReadGenesis(chaindb)
 				if err != nil {
 					Fatalf("Could not read genesis from database: %v", err)
@@ -1845,10 +1840,16 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.String(CryptoKZGFlag.Name) != "gokzg" && ctx.String(CryptoKZGFlag.Name) != "ckzg" {
 		Fatalf("--%s flag must be 'gokzg' or 'ckzg'", CryptoKZGFlag.Name)
 	}
+	// The initialization of the KZG library can take up to 2 seconds
+	// We start this here in parallel, so it should be available
+	// once we start executing blocks. It's threadsafe.
+	go func() {
 		log.Info("Initializing the KZG library", "backend", ctx.String(CryptoKZGFlag.Name))
 		if err := kzg4844.UseCKZG(ctx.String(CryptoKZGFlag.Name) == "ckzg"); err != nil {
 			Fatalf("Failed to set KZG library implementation to %s: %v", ctx.String(CryptoKZGFlag.Name), err)
 		}
+	}()
+
 	// VM tracing config.
 	if ctx.IsSet(VMTraceFlag.Name) {
 		if name := ctx.String(VMTraceFlag.Name); name != "" {
@@ -1886,7 +1887,7 @@ func MakeBeaconLightConfig(ctx *cli.Context) bparams.ClientConfig {
 		if !ctx.IsSet(BeaconGenesisTimeFlag.Name) {
 			Fatalf("Custom beacon chain config is specified but genesis time is missing")
 		}
-		if !ctx.IsSet(BeaconCheckpointFlag.Name) {
+		if !ctx.IsSet(BeaconCheckpointFlag.Name) && !ctx.IsSet(BeaconCheckpointFileFlag.Name) {
 			Fatalf("Custom beacon chain config is specified but checkpoint is missing")
 		}
 		config.ChainConfig = bparams.ChainConfig{
@@ -1911,13 +1912,28 @@ func MakeBeaconLightConfig(ctx *cli.Context) bparams.ClientConfig {
 		}
 	}
 	// Checkpoint is required with custom chain config and is optional with pre-defined config
-	if ctx.IsSet(BeaconCheckpointFlag.Name) {
+	// If both checkpoint block hash and checkpoint file are specified then the
-		if c, err := hexutil.Decode(ctx.String(BeaconCheckpointFlag.Name)); err == nil && len(c) <= 32 {
+	// client is initialized with the specified block hash and new checkpoints
-			copy(config.Checkpoint[:len(c)], c)
+	// are saved to the specified file.
-		} else {
+	if ctx.IsSet(BeaconCheckpointFileFlag.Name) {
-			Fatalf("Invalid hex string", "beacon.checkpoint", ctx.String(BeaconCheckpointFlag.Name), "error", err)
+		if _, err := config.SetCheckpointFile(ctx.String(BeaconCheckpointFileFlag.Name)); err != nil {
+			Fatalf("Could not load beacon checkpoint file", "beacon.checkpoint.file", ctx.String(BeaconCheckpointFileFlag.Name), "error", err)
 		}
 	}
+	if ctx.IsSet(BeaconCheckpointFlag.Name) {
+		hex := ctx.String(BeaconCheckpointFlag.Name)
+		c, err := hexutil.Decode(hex)
+		if err != nil {
+			Fatalf("Invalid hex string", "beacon.checkpoint", hex, "error", err)
+		}
+		if len(c) != 32 {
+			Fatalf("Invalid hex string length", "beacon.checkpoint", hex, "length", len(c))
+		}
+		copy(config.Checkpoint[:len(c)], c)
+	}
+	if config.Checkpoint == (common.Hash{}) {
+		Fatalf("Beacon checkpoint not specified")
+	}
 	config.Apis = ctx.StringSlice(BeaconApiFlag.Name)
 	if config.Apis == nil {
 		Fatalf("Beacon node light client API URL not specified")
@@ -2023,7 +2039,6 @@ func SetupMetrics(cfg *metrics.Config) {
 		log.Info("Enabling metrics export to InfluxDB")
 		go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap)
 	} else if enableExportV2 {
-		tagsMap := SplitTagsFlag(cfg.InfluxDBTags)
 		log.Info("Enabling metrics export to InfluxDB (v2)")
 		go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, token, bucket, organization, "geth.", tagsMap)
 	}
@@ -2076,7 +2091,15 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
 		}
 		chainDb = remotedb.New(client)
 	default:
-		chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "eth/db/chaindata/", readonly)
+		options := node.DatabaseOptions{
+			ReadOnly:          readonly,
+			Cache:             cache,
+			Handles:           handles,
+			AncientsDirectory: ctx.String(AncientFlag.Name),
+			MetricsNamespace:  "eth/db/chaindata/",
+			EraDirectory:      ctx.String(EraFlag.Name),
+		}
+		chainDb, err = stack.OpenDatabaseWithOptions("chaindata", options)
 	}
 	if err != nil {
 		Fatalf("Could not open database: %v", err)
@@ -2185,6 +2208,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
 	}
 	if !ctx.Bool(SnapshotFlag.Name) {
 		cache.SnapshotLimit = 0 // Disabled
+	} else if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheSnapshotFlag.Name) {
+		cache.SnapshotLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheSnapshotFlag.Name) / 100
 	}
 	// If we're in readonly, do not bother generating snapshot data.
 	if readonly {

cmd/utils/flags_legacy.go

@@ -39,12 +39,6 @@ var DeprecatedFlags = []cli.Flag{
 	CacheTrieRejournalFlag,
 	LegacyDiscoveryV5Flag,
 	TxLookupLimitFlag,
-	LightServeFlag,
-	LightIngressFlag,
-	LightEgressFlag,
-	LightMaxPeersFlag,
-	LightNoPruneFlag,
-	LightNoSyncServeFlag,
 	LogBacktraceAtFlag,
 	LogDebugFlag,
 	MinerNewPayloadTimeoutFlag,
@@ -88,37 +82,6 @@ var (
 		Value:    ethconfig.Defaults.TransactionHistory,
 		Category: flags.DeprecatedCategory,
 	}
-	// Light server and client settings, Deprecated November 2023
-	LightServeFlag = &cli.IntFlag{
-		Name:     "light.serve",
-		Usage:    "Maximum percentage of time allowed for serving LES requests (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
-	LightIngressFlag = &cli.IntFlag{
-		Name:     "light.ingress",
-		Usage:    "Incoming bandwidth limit for serving light clients (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
-	LightEgressFlag = &cli.IntFlag{
-		Name:     "light.egress",
-		Usage:    "Outgoing bandwidth limit for serving light clients (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
-	LightMaxPeersFlag = &cli.IntFlag{
-		Name:     "light.maxpeers",
-		Usage:    "Maximum number of light clients to serve, or light servers to attach to (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
-	LightNoPruneFlag = &cli.BoolFlag{
-		Name:     "light.nopruning",
-		Usage:    "Disable ancient light chain data pruning (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
-	LightNoSyncServeFlag = &cli.BoolFlag{
-		Name:     "light.nosyncserve",
-		Usage:    "Enables serving light clients before syncing (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
 	// Deprecated November 2023
 	LogBacktraceAtFlag = &cli.StringFlag{
 		Name:     "log.backtrace",

cmd/utils/history_test.go

@@ -158,7 +158,7 @@ func TestHistoryImportAndExport(t *testing.T) {
 	}
 
 	// Now import Era.
-	db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
+	db2, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
 	if err != nil {
 		panic(err)
 	}
@@ -171,7 +171,7 @@ func TestHistoryImportAndExport(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable to initialize chain: %v", err)
 	}
-	if err := ImportHistory(imported, db2, dir, "mainnet"); err != nil {
+	if err := ImportHistory(imported, dir, "mainnet"); err != nil {
 		t.Fatalf("failed to import chain: %v", err)
 	}
 	if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() {

cmd/workload/README.md

@@ -26,4 +26,5 @@ the following commands (in this directory) against a synced mainnet node:
 ```shell
 > go run . filtergen --queries queries/filter_queries_mainnet.json http://host:8545
 > go run . historygen --history-tests queries/history_mainnet.json http://host:8545
+> go run . tracegen --trace-tests queries/trace_mainnet.json http://host:8545
 ```

cmd/workload/client.go

@@ -0,0 +1,104 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/ethclient/gethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/urfave/cli/v2"
+)
+
+type client struct {
+	Eth  *ethclient.Client
+	Geth *gethclient.Client
+	RPC  *rpc.Client
+}
+
+func makeClient(ctx *cli.Context) *client {
+	if ctx.NArg() < 1 {
+		exit("missing RPC endpoint URL as command-line argument")
+	}
+	url := ctx.Args().First()
+	cl, err := rpc.Dial(url)
+	if err != nil {
+		exit(fmt.Errorf("could not create RPC client at %s: %v", url, err))
+	}
+	return &client{
+		RPC:  cl,
+		Eth:  ethclient.NewClient(cl),
+		Geth: gethclient.New(cl),
+	}
+}
+
+type simpleBlock struct {
+	Number hexutil.Uint64 `json:"number"`
+	Hash   common.Hash    `json:"hash"`
+}
+
+type simpleTransaction struct {
+	Hash             common.Hash    `json:"hash"`
+	TransactionIndex hexutil.Uint64 `json:"transactionIndex"`
+}
+
+func (c *client) getBlockByHash(ctx context.Context, arg common.Hash, fullTx bool) (*simpleBlock, error) {
+	var r *simpleBlock
+	err := c.RPC.CallContext(ctx, &r, "eth_getBlockByHash", arg, fullTx)
+	return r, err
+}
+
+func (c *client) getBlockByNumber(ctx context.Context, arg uint64, fullTx bool) (*simpleBlock, error) {
+	var r *simpleBlock
+	err := c.RPC.CallContext(ctx, &r, "eth_getBlockByNumber", hexutil.Uint64(arg), fullTx)
+	return r, err
+}
+
+func (c *client) getTransactionByBlockHashAndIndex(ctx context.Context, block common.Hash, index uint64) (*simpleTransaction, error) {
+	var r *simpleTransaction
+	err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockHashAndIndex", block, hexutil.Uint64(index))
+	return r, err
+}
+
+func (c *client) getTransactionByBlockNumberAndIndex(ctx context.Context, block uint64, index uint64) (*simpleTransaction, error) {
+	var r *simpleTransaction
+	err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockNumberAndIndex", hexutil.Uint64(block), hexutil.Uint64(index))
+	return r, err
+}
+
+func (c *client) getBlockTransactionCountByHash(ctx context.Context, block common.Hash) (uint64, error) {
+	var r hexutil.Uint64
+	err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByHash", block)
+	return uint64(r), err
+}
+
+func (c *client) getBlockTransactionCountByNumber(ctx context.Context, block uint64) (uint64, error) {
+	var r hexutil.Uint64
+	err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByNumber", hexutil.Uint64(block))
+	return uint64(r), err
+}
+
+func (c *client) getBlockReceipts(ctx context.Context, arg any) ([]*types.Receipt, error) {
+	var result []*types.Receipt
+	err := c.RPC.CallContext(ctx, &result, "eth_getBlockReceipts", arg)
+	return result, err
+}

cmd/workload/filtertest.go

@@ -45,11 +45,11 @@ func newFilterTestSuite(cfg testConfig) *filterTestSuite {
 	return s
 }
 
-func (s *filterTestSuite) allTests() []utesting.Test {
+func (s *filterTestSuite) allTests() []workloadTest {
-	return []utesting.Test{
+	return []workloadTest{
-		{Name: "Filter/ShortRange", Fn: s.filterShortRange},
+		newWorkLoadTest("Filter/ShortRange", s.filterShortRange),
-		{Name: "Filter/LongRange", Fn: s.filterLongRange, Slow: true},
+		newSlowWorkloadTest("Filter/LongRange", s.filterLongRange),
-		{Name: "Filter/FullRange", Fn: s.filterFullRange, Slow: true},
+		newSlowWorkloadTest("Filter/FullRange", s.filterFullRange),
 	}
 }
 
@@ -109,6 +109,9 @@ func (s *filterTestSuite) filterFullRange(t *utesting.T) {
 
 func (s *filterTestSuite) queryAndCheck(t *utesting.T, query *filterQuery) {
 	query.run(s.cfg.client, s.cfg.historyPruneBlock)
+	if query.Err == errPrunedHistory {
+		return
+	}
 	if query.Err != nil {
 		t.Errorf("Filter query failed (fromBlock: %d toBlock: %d addresses: %v topics: %v error: %v)", query.FromBlock, query.ToBlock, query.Address, query.Topics, query.Err)
 		return
@@ -126,6 +129,9 @@ func (s *filterTestSuite) fullRangeQueryAndCheck(t *utesting.T, query *filterQue
 		Topics:    query.Topics,
 	}
 	frQuery.run(s.cfg.client, s.cfg.historyPruneBlock)
+	if frQuery.Err == errPrunedHistory {
+		return
+	}
 	if frQuery.Err != nil {
 		t.Errorf("Full range filter query failed (addresses: %v topics: %v error: %v)", frQuery.Address, frQuery.Topics, frQuery.Err)
 		return
@@ -206,14 +212,11 @@ func (fq *filterQuery) run(client *client, historyPruneBlock *uint64) {
 		Addresses: fq.Address,
 		Topics:    fq.Topics,
 	})
-	if err != nil {
-		if err = validateHistoryPruneErr(fq.Err, uint64(fq.FromBlock), historyPruneBlock); err == errPrunedHistory {
-			return
-		} else if err != nil {
-			fmt.Printf("Filter query failed: fromBlock: %d toBlock: %d addresses: %v topics: %v error: %v\n",
-				fq.FromBlock, fq.ToBlock, fq.Address, fq.Topics, err)
-		}
-		fq.Err = err
-	}
 	fq.results = logs
+	fq.Err = validateHistoryPruneErr(err, uint64(fq.FromBlock), historyPruneBlock)
+}
+
+func (fq *filterQuery) printError() {
+	fmt.Printf("Filter query failed: fromBlock: %d toBlock: %d addresses: %v topics: %v error: %v\n",
+		fq.FromBlock, fq.ToBlock, fq.Address, fq.Topics, fq.Err)
 }

cmd/workload/filtertestgen.go

@@ -40,7 +40,6 @@ var (
 		Action:    filterGenCmd,
 		Flags: []cli.Flag{
 			filterQueryFileFlag,
-			filterErrorFileFlag,
 		},
 	}
 	filterQueryFileFlag = &cli.StringFlag{
@@ -72,8 +71,8 @@ func filterGenCmd(ctx *cli.Context) error {
 		query := f.newQuery()
 		query.run(f.client, nil)
 		if query.Err != nil {
-			f.errors = append(f.errors, query)
+			query.printError()
-			continue
+			exit("filter query failed")
 		}
 		if len(query.results) > 0 && len(query.results) <= maxFilterResultSize {
 			for {
@@ -90,8 +89,8 @@ func filterGenCmd(ctx *cli.Context) error {
 					)
 				}
 				if extQuery.Err != nil {
-					f.errors = append(f.errors, extQuery)
+					extQuery.printError()
-					break
+					exit("filter query failed")
 				}
 				if len(extQuery.results) > maxFilterResultSize {
 					break
@@ -101,7 +100,6 @@ func filterGenCmd(ctx *cli.Context) error {
 			f.storeQuery(query)
 			if time.Since(lastWrite) > time.Second*10 {
 				f.writeQueries()
-				f.writeErrors()
 				lastWrite = time.Now()
 			}
 		}
@@ -112,18 +110,15 @@ func filterGenCmd(ctx *cli.Context) error {
 type filterTestGen struct {
 	client    *client
 	queryFile string
-	errorFile string
 
 	finalizedBlock int64
 	queries        [filterBuckets][]*filterQuery
-	errors         []*filterQuery
 }
 
 func newFilterTestGen(ctx *cli.Context) *filterTestGen {
 	return &filterTestGen{
 		client:    makeClient(ctx),
 		queryFile: ctx.String(filterQueryFileFlag.Name),
-		errorFile: ctx.String(filterErrorFileFlag.Name),
 	}
 }
 
@@ -360,17 +355,6 @@ func (s *filterTestGen) writeQueries() {
 	file.Close()
 }
 
-// writeQueries serializes the generated errors to the error file.
-func (s *filterTestGen) writeErrors() {
-	file, err := os.Create(s.errorFile)
-	if err != nil {
-		exit(fmt.Errorf("Error creating filter error file %s: %v", s.errorFile, err))
-		return
-	}
-	defer file.Close()
-	json.NewEncoder(file).Encode(s.errors)
-}
-
 func mustGetFinalizedBlock(client *client) int64 {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
 	defer cancel()

cmd/workload/filtertestperf.go

@@ -17,8 +17,10 @@
 package main
 
 import (
+	"encoding/json"
 	"fmt"
 	"math/rand"
+	"os"
 	"slices"
 	"sort"
 	"time"
@@ -41,7 +43,7 @@ var (
 	}
 )
 
-const passCount = 1
+const passCount = 3
 
 func filterPerfCmd(ctx *cli.Context) error {
 	cfg := testConfigFromCLI(ctx)
@@ -61,7 +63,10 @@ func filterPerfCmd(ctx *cli.Context) error {
 	}
 
 	// Run test queries.
-	var failed, mismatch int
+	var (
+		failed, pruned, mismatch int
+		errors                   []*filterQuery
+	)
 	for i := 1; i <= passCount; i++ {
 		fmt.Println("Performance test pass", i, "/", passCount)
 		for len(queries) > 0 {
@@ -71,27 +76,35 @@ func filterPerfCmd(ctx *cli.Context) error {
 			queries = queries[:len(queries)-1]
 			start := time.Now()
 			qt.query.run(cfg.client, cfg.historyPruneBlock)
+			if qt.query.Err == errPrunedHistory {
+				pruned++
+				continue
+			}
 			qt.runtime = append(qt.runtime, time.Since(start))
 			slices.Sort(qt.runtime)
 			qt.medianTime = qt.runtime[len(qt.runtime)/2]
 			if qt.query.Err != nil {
+				qt.query.printError()
+				errors = append(errors, qt.query)
 				failed++
 				continue
 			}
 			if rhash := qt.query.calculateHash(); *qt.query.ResultHash != rhash {
 				fmt.Printf("Filter query result mismatch: fromBlock: %d toBlock: %d addresses: %v topics: %v expected hash: %064x calculated hash: %064x\n", qt.query.FromBlock, qt.query.ToBlock, qt.query.Address, qt.query.Topics, *qt.query.ResultHash, rhash)
+				errors = append(errors, qt.query)
+				mismatch++
 				continue
 			}
 			processed = append(processed, qt)
 			if len(processed)%50 == 0 {
-				fmt.Println(" processed:", len(processed), "remaining", len(queries), "failed:", failed, "result mismatch:", mismatch)
+				fmt.Println(" processed:", len(processed), "remaining", len(queries), "failed:", failed, "pruned:", pruned, "result mismatch:", mismatch)
 			}
 		}
 		queries, processed = processed, nil
 	}
 
 	// Show results and stats.
-	fmt.Println("Performance test finished; processed:", len(queries), "failed:", failed, "result mismatch:", mismatch)
+	fmt.Println("Performance test finished; processed:", len(queries), "failed:", failed, "pruned:", pruned, "result mismatch:", mismatch)
 	stats := make([]bucketStats, len(f.queries))
 	var wildcardStats bucketStats
 	for _, qt := range queries {
@@ -114,11 +127,14 @@ func filterPerfCmd(ctx *cli.Context) error {
 	sort.Slice(queries, func(i, j int) bool {
 		return queries[i].medianTime > queries[j].medianTime
 	})
-	for i := 0; i < 10; i++ {
+	for i, q := range queries {
-		q := queries[i]
+		if i >= 10 {
+			break
+		}
 		fmt.Printf("Most expensive query #%-2d   median runtime: %13v  max runtime: %13v  result count: %4d  fromBlock: %9d  toBlock: %9d  addresses: %v  topics: %v\n",
 			i+1, q.medianTime, q.runtime[len(q.runtime)-1], len(q.query.results), q.query.FromBlock, q.query.ToBlock, q.query.Address, q.query.Topics)
 	}
+	writeErrors(ctx.String(filterErrorFileFlag.Name), errors)
 	return nil
 }
 
@@ -135,3 +151,14 @@ func (st *bucketStats) print(name string) {
 	fmt.Printf("%-20s queries: %4d  average block length: %12.2f  average log count: %7.2f  average runtime: %13v\n",
 		name, st.count, float64(st.blocks)/float64(st.count), float64(st.logs)/float64(st.count), st.runtime/time.Duration(st.count))
 }
+
+// writeQueries serializes the generated errors to the error file.
+func writeErrors(errorFile string, errors []*filterQuery) {
+	file, err := os.Create(errorFile)
+	if err != nil {
+		exit(fmt.Errorf("Error creating filter error file %s: %v", errorFile, err))
+		return
+	}
+	defer file.Close()
+	json.NewEncoder(file).Encode(errors)
+}

cmd/workload/historytest.go

@@ -23,7 +23,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/internal/utesting"
 )
 
@@ -65,40 +64,16 @@ func (s *historyTestSuite) loadTests() error {
 	return nil
 }
 
-func (s *historyTestSuite) allTests() []utesting.Test {
+func (s *historyTestSuite) allTests() []workloadTest {
-	return []utesting.Test{
+	return []workloadTest{
-		{
+		newWorkLoadTest("History/getBlockByHash", s.testGetBlockByHash),
-			Name: "History/getBlockByHash",
+		newWorkLoadTest("History/getBlockByNumber", s.testGetBlockByNumber),
-			Fn:   s.testGetBlockByHash,
+		newWorkLoadTest("History/getBlockReceiptsByHash", s.testGetBlockReceiptsByHash),
-		},
+		newWorkLoadTest("History/getBlockReceiptsByNumber", s.testGetBlockReceiptsByNumber),
-		{
+		newWorkLoadTest("History/getBlockTransactionCountByHash", s.testGetBlockTransactionCountByHash),
-			Name: "History/getBlockByNumber",
+		newWorkLoadTest("History/getBlockTransactionCountByNumber", s.testGetBlockTransactionCountByNumber),
-			Fn:   s.testGetBlockByNumber,
+		newWorkLoadTest("History/getTransactionByBlockHashAndIndex", s.testGetTransactionByBlockHashAndIndex),
-		},
+		newWorkLoadTest("History/getTransactionByBlockNumberAndIndex", s.testGetTransactionByBlockNumberAndIndex),
-		{
-			Name: "History/getBlockReceiptsByHash",
-			Fn:   s.testGetBlockReceiptsByHash,
-		},
-		{
-			Name: "History/getBlockReceiptsByNumber",
-			Fn:   s.testGetBlockReceiptsByNumber,
-		},
-		{
-			Name: "History/getBlockTransactionCountByHash",
-			Fn:   s.testGetBlockTransactionCountByHash,
-		},
-		{
-			Name: "History/getBlockTransactionCountByNumber",
-			Fn:   s.testGetBlockTransactionCountByNumber,
-		},
-		{
-			Name: "History/getTransactionByBlockHashAndIndex",
-			Fn:   s.testGetTransactionByBlockHashAndIndex,
-		},
-		{
-			Name: "History/getTransactionByBlockNumberAndIndex",
-			Fn:   s.testGetTransactionByBlockNumberAndIndex,
-		},
 	}
 }
 
@@ -279,55 +254,3 @@ func (s *historyTestSuite) testGetTransactionByBlockNumberAndIndex(t *utesting.T
 		}
 	}
 }
-
-type simpleBlock struct {
-	Number hexutil.Uint64 `json:"number"`
-	Hash   common.Hash    `json:"hash"`
-}
-
-type simpleTransaction struct {
-	Hash             common.Hash    `json:"hash"`
-	TransactionIndex hexutil.Uint64 `json:"transactionIndex"`
-}
-
-func (c *client) getBlockByHash(ctx context.Context, arg common.Hash, fullTx bool) (*simpleBlock, error) {
-	var r *simpleBlock
-	err := c.RPC.CallContext(ctx, &r, "eth_getBlockByHash", arg, fullTx)
-	return r, err
-}
-
-func (c *client) getBlockByNumber(ctx context.Context, arg uint64, fullTx bool) (*simpleBlock, error) {
-	var r *simpleBlock
-	err := c.RPC.CallContext(ctx, &r, "eth_getBlockByNumber", hexutil.Uint64(arg), fullTx)
-	return r, err
-}
-
-func (c *client) getTransactionByBlockHashAndIndex(ctx context.Context, block common.Hash, index uint64) (*simpleTransaction, error) {
-	var r *simpleTransaction
-	err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockHashAndIndex", block, hexutil.Uint64(index))
-	return r, err
-}
-
-func (c *client) getTransactionByBlockNumberAndIndex(ctx context.Context, block uint64, index uint64) (*simpleTransaction, error) {
-	var r *simpleTransaction
-	err := c.RPC.CallContext(ctx, &r, "eth_getTransactionByBlockNumberAndIndex", hexutil.Uint64(block), hexutil.Uint64(index))
-	return r, err
-}
-
-func (c *client) getBlockTransactionCountByHash(ctx context.Context, block common.Hash) (uint64, error) {
-	var r hexutil.Uint64
-	err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByHash", block)
-	return uint64(r), err
-}
-
-func (c *client) getBlockTransactionCountByNumber(ctx context.Context, block uint64) (uint64, error) {
-	var r hexutil.Uint64
-	err := c.RPC.CallContext(ctx, &r, "eth_getBlockTransactionCountByNumber", hexutil.Uint64(block))
-	return uint64(r), err
-}
-
-func (c *client) getBlockReceipts(ctx context.Context, arg any) ([]*types.Receipt, error) {
-	var result []*types.Receipt
-	err := c.RPC.CallContext(ctx, &result, "eth_getBlockReceipts", arg)
-	return result, err
-}

cmd/workload/historytestgen.go

@@ -51,7 +51,7 @@ var (
 	}
 	historyTestEarliestFlag = &cli.IntFlag{
 		Name:     "earliest",
-		Usage:    "JSON file containing filter test queries",
+		Usage:    "The earliest block to test queries",
 		Value:    0,
 		Category: flags.TestingCategory,
 	}
@@ -139,7 +139,7 @@ func calcReceiptsHash(rcpt []*types.Receipt) common.Hash {
 func writeJSON(fileName string, value any) {
 	file, err := os.Create(fileName)
 	if err != nil {
-		exit(fmt.Errorf("Error creating %s: %v", fileName, err))
+		exit(fmt.Errorf("error creating %s: %v", fileName, err))
 		return
 	}
 	defer file.Close()

cmd/workload/main.go

@@ -20,10 +20,8 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/internal/flags"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/urfave/cli/v2"
 )
 
@@ -49,6 +47,7 @@ func init() {
 		runTestCommand,
 		historyGenerateCommand,
 		filterGenerateCommand,
+		traceGenerateCommand,
 		filterPerfCommand,
 	}
 }
@@ -57,26 +56,6 @@ func main() {
 	exit(app.Run(os.Args))
 }
 
-type client struct {
-	Eth *ethclient.Client
-	RPC *rpc.Client
-}
-
-func makeClient(ctx *cli.Context) *client {
-	if ctx.NArg() < 1 {
-		exit("missing RPC endpoint URL as command-line argument")
-	}
-	url := ctx.Args().First()
-	cl, err := rpc.Dial(url)
-	if err != nil {
-		exit(fmt.Errorf("Could not create RPC client at %s: %v", url, err))
-	}
-	return &client{
-		RPC: cl,
-		Eth: ethclient.NewClient(cl),
-	}
-}
-
 func exit(err any) {
 	if err == nil {
 		os.Exit(0)

cmd/workload/testsuite.go

@@ -21,9 +21,8 @@ import (
 	"fmt"
 	"io/fs"
 	"os"
-	"slices"
 
-	"github.com/ethereum/go-ethereum/eth/ethconfig"
+	"github.com/ethereum/go-ethereum/core/history"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/internal/utesting"
 	"github.com/ethereum/go-ethereum/log"
@@ -45,10 +44,13 @@ var (
 			testPatternFlag,
 			testTAPFlag,
 			testSlowFlag,
+			testArchiveFlag,
 			testSepoliaFlag,
 			testMainnetFlag,
 			filterQueryFileFlag,
 			historyTestFileFlag,
+			traceTestFileFlag,
+			traceTestInvalidOutputFlag,
 		},
 	}
 	testPatternFlag = &cli.StringFlag{
@@ -67,6 +69,12 @@ var (
 		Value:    false,
 		Category: flags.TestingCategory,
 	}
+	testArchiveFlag = &cli.BoolFlag{
+		Name:     "archive",
+		Usage:    "Enable archive tests",
+		Value:    false,
+		Category: flags.TestingCategory,
+	}
 	testSepoliaFlag = &cli.BoolFlag{
 		Name:     "sepolia",
 		Usage:    "Use test cases for sepolia network",
@@ -86,6 +94,7 @@ type testConfig struct {
 	filterQueryFile   string
 	historyTestFile   string
 	historyPruneBlock *uint64
+	traceTestFile     string
 }
 
 var errPrunedHistory = fmt.Errorf("attempt to access pruned history")
@@ -124,37 +133,86 @@ func testConfigFromCLI(ctx *cli.Context) (cfg testConfig) {
 		cfg.filterQueryFile = "queries/filter_queries_mainnet.json"
 		cfg.historyTestFile = "queries/history_mainnet.json"
 		cfg.historyPruneBlock = new(uint64)
-		*cfg.historyPruneBlock = ethconfig.HistoryPrunePoints[params.MainnetGenesisHash].BlockNumber
+		*cfg.historyPruneBlock = history.PrunePoints[params.MainnetGenesisHash].BlockNumber
+		cfg.traceTestFile = "queries/trace_mainnet.json"
 	case ctx.Bool(testSepoliaFlag.Name):
 		cfg.fsys = builtinTestFiles
 		cfg.filterQueryFile = "queries/filter_queries_sepolia.json"
 		cfg.historyTestFile = "queries/history_sepolia.json"
 		cfg.historyPruneBlock = new(uint64)
-		*cfg.historyPruneBlock = ethconfig.HistoryPrunePoints[params.SepoliaGenesisHash].BlockNumber
+		*cfg.historyPruneBlock = history.PrunePoints[params.SepoliaGenesisHash].BlockNumber
+		cfg.traceTestFile = "queries/trace_sepolia.json"
 	default:
 		cfg.fsys = os.DirFS(".")
 		cfg.filterQueryFile = ctx.String(filterQueryFileFlag.Name)
 		cfg.historyTestFile = ctx.String(historyTestFileFlag.Name)
+		cfg.traceTestFile = ctx.String(traceTestFileFlag.Name)
 	}
 	return cfg
 }
 
+// workloadTest represents a single test in the workload. It's a wrapper
+// of utesting.Test by adding a few additional attributes.
+type workloadTest struct {
+	utesting.Test
+
+	archive bool // Flag whether the archive node (full state history) is required for this test
+}
+
+func newWorkLoadTest(name string, fn func(t *utesting.T)) workloadTest {
+	return workloadTest{
+		Test: utesting.Test{
+			Name: name,
+			Fn:   fn,
+		},
+	}
+}
+
+func newSlowWorkloadTest(name string, fn func(t *utesting.T)) workloadTest {
+	t := newWorkLoadTest(name, fn)
+	t.Slow = true
+	return t
+}
+
+func newArchiveWorkloadTest(name string, fn func(t *utesting.T)) workloadTest {
+	t := newWorkLoadTest(name, fn)
+	t.archive = true
+	return t
+}
+
+func filterTests(tests []workloadTest, pattern string, filterFn func(t workloadTest) bool) []utesting.Test {
+	var utests []utesting.Test
+	for _, t := range tests {
+		if filterFn(t) {
+			utests = append(utests, t.Test)
+		}
+	}
+	if pattern == "" {
+		return utests
+	}
+	return utesting.MatchTests(utests, pattern)
+}
+
 func runTestCmd(ctx *cli.Context) error {
 	cfg := testConfigFromCLI(ctx)
 	filterSuite := newFilterTestSuite(cfg)
 	historySuite := newHistoryTestSuite(cfg)
+	traceSuite := newTraceTestSuite(cfg, ctx)
 
 	// Filter test cases.
 	tests := filterSuite.allTests()
 	tests = append(tests, historySuite.allTests()...)
-	if ctx.IsSet(testPatternFlag.Name) {
+	tests = append(tests, traceSuite.allTests()...)
-		tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
+
+	utests := filterTests(tests, ctx.String(testPatternFlag.Name), func(t workloadTest) bool {
+		if t.Slow && !ctx.Bool(testSlowFlag.Name) {
+			return false
 		}
-	if !ctx.Bool(testSlowFlag.Name) {
+		if t.archive && !ctx.Bool(testArchiveFlag.Name) {
-		tests = slices.DeleteFunc(tests, func(test utesting.Test) bool {
+			return false
-			return test.Slow
+		}
+		return true
 	})
-	}
 
 	// Disable logging unless explicitly enabled.
 	if !ctx.IsSet("verbosity") && !ctx.IsSet("vmodule") {
@@ -166,7 +224,7 @@ func runTestCmd(ctx *cli.Context) error {
 	if ctx.Bool(testTAPFlag.Name) {
 		run = utesting.RunTAP
 	}
-	results := run(tests, os.Stdout)
+	results := run(utests, os.Stdout)
 	if utesting.CountFailures(results) > 0 {
 		os.Exit(1)
 	}

cmd/workload/tracetest.go

@@ -0,0 +1,126 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/tracers"
+	"github.com/ethereum/go-ethereum/internal/utesting"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+)
+
+// traceTest is the content of a history test.
+type traceTest struct {
+	TxHashes     []common.Hash         `json:"txHashes"`
+	TraceConfigs []tracers.TraceConfig `json:"traceConfigs"`
+	ResultHashes []common.Hash         `json:"resultHashes"`
+}
+
+type traceTestSuite struct {
+	cfg        testConfig
+	tests      traceTest
+	invalidDir string
+}
+
+func newTraceTestSuite(cfg testConfig, ctx *cli.Context) *traceTestSuite {
+	s := &traceTestSuite{
+		cfg:        cfg,
+		invalidDir: ctx.String(traceTestInvalidOutputFlag.Name),
+	}
+	if err := s.loadTests(); err != nil {
+		exit(err)
+	}
+	return s
+}
+
+func (s *traceTestSuite) loadTests() error {
+	file, err := s.cfg.fsys.Open(s.cfg.traceTestFile)
+	if err != nil {
+		return fmt.Errorf("can't open traceTestFile: %v", err)
+	}
+	defer file.Close()
+
+	if err := json.NewDecoder(file).Decode(&s.tests); err != nil {
+		return fmt.Errorf("invalid JSON in %s: %v", s.cfg.traceTestFile, err)
+	}
+	if len(s.tests.TxHashes) == 0 {
+		return fmt.Errorf("traceTestFile %s has no test data", s.cfg.traceTestFile)
+	}
+	return nil
+}
+
+func (s *traceTestSuite) allTests() []workloadTest {
+	return []workloadTest{
+		newArchiveWorkloadTest("Trace/Transaction", s.traceTransaction),
+	}
+}
+
+// traceTransaction runs all transaction tracing tests
+func (s *traceTestSuite) traceTransaction(t *utesting.T) {
+	ctx := context.Background()
+
+	for i, hash := range s.tests.TxHashes {
+		config := s.tests.TraceConfigs[i]
+		result, err := s.cfg.client.Geth.TraceTransaction(ctx, hash, &config)
+		if err != nil {
+			t.Fatalf("Transaction %d (hash %v): error %v", i, hash, err)
+		}
+		blob, err := json.Marshal(result)
+		if err != nil {
+			t.Fatalf("Transaction %d (hash %v): error %v", i, hash, err)
+			continue
+		}
+		if crypto.Keccak256Hash(blob) != s.tests.ResultHashes[i] {
+			t.Errorf("Transaction %d (hash %v): invalid result", i, hash)
+
+			writeInvalidTraceResult(s.invalidDir, hash, result)
+		}
+	}
+}
+
+func writeInvalidTraceResult(dir string, hash common.Hash, result any) {
+	if dir == "" {
+		return
+	}
+	err := os.MkdirAll(dir, os.ModePerm)
+	if err != nil {
+		log.Info("Failed to make output directory", "err", err)
+		return
+	}
+	name := filepath.Join(dir, "invalid"+"_"+hash.String())
+	file, err := os.Create(name)
+	if err != nil {
+		exit(fmt.Errorf("error creating %s: %v", name, err))
+		return
+	}
+	defer file.Close()
+
+	data, _ := json.MarshalIndent(result, "", "    ")
+	_, err = file.Write(data)
+	if err != nil {
+		exit(fmt.Errorf("error writing %s: %v", name, err))
+		return
+	}
+}

cmd/workload/tracetestgen.go

@@ -0,0 +1,195 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>
+
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math/big"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/tracers"
+	"github.com/ethereum/go-ethereum/eth/tracers/logger"
+	"github.com/ethereum/go-ethereum/internal/flags"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	defaultBlocksToTrace = 64 // the number of states assumed to be available
+
+	traceGenerateCommand = &cli.Command{
+		Name:      "tracegen",
+		Usage:     "Generates tests for state tracing",
+		ArgsUsage: "<RPC endpoint URL>",
+		Action:    generateTraceTests,
+		Flags: []cli.Flag{
+			traceTestFileFlag,
+			traceTestResultOutputFlag,
+			traceTestBlockFlag,
+		},
+	}
+
+	traceTestFileFlag = &cli.StringFlag{
+		Name:     "trace-tests",
+		Usage:    "JSON file containing trace test queries",
+		Value:    "trace_tests.json",
+		Category: flags.TestingCategory,
+	}
+	traceTestResultOutputFlag = &cli.StringFlag{
+		Name:     "trace-output",
+		Usage:    "Folder containing the trace output files",
+		Value:    "",
+		Category: flags.TestingCategory,
+	}
+	traceTestBlockFlag = &cli.IntFlag{
+		Name:     "trace-blocks",
+		Usage:    "The number of blocks for tracing",
+		Value:    defaultBlocksToTrace,
+		Category: flags.TestingCategory,
+	}
+	traceTestInvalidOutputFlag = &cli.StringFlag{
+		Name:     "trace-invalid",
+		Usage:    "Folder containing the mismatched trace output files",
+		Value:    "",
+		Category: flags.TestingCategory,
+	}
+)
+
+func generateTraceTests(clictx *cli.Context) error {
+	var (
+		client     = makeClient(clictx)
+		outputFile = clictx.String(traceTestFileFlag.Name)
+		outputDir  = clictx.String(traceTestResultOutputFlag.Name)
+		blocks     = clictx.Int(traceTestBlockFlag.Name)
+		ctx        = context.Background()
+		test       = new(traceTest)
+	)
+	if outputDir != "" {
+		err := os.MkdirAll(outputDir, os.ModePerm)
+		if err != nil {
+			return err
+		}
+	}
+	latest, err := client.Eth.BlockNumber(ctx)
+	if err != nil {
+		exit(err)
+	}
+	if latest < uint64(blocks) {
+		exit(fmt.Errorf("node seems not synced, latest block is %d", latest))
+	}
+	// Get blocks and assign block info into the test
+	var (
+		start  = time.Now()
+		logged = time.Now()
+		failed int
+	)
+	log.Info("Trace transactions around the chain tip", "head", latest, "blocks", blocks)
+
+	for i := 0; i < blocks; i++ {
+		number := latest - uint64(i)
+		block, err := client.Eth.BlockByNumber(ctx, big.NewInt(int64(number)))
+		if err != nil {
+			exit(err)
+		}
+		for _, tx := range block.Transactions() {
+			config, configName := randomTraceOption()
+			result, err := client.Geth.TraceTransaction(ctx, tx.Hash(), config)
+			if err != nil {
+				failed += 1
+				break
+			}
+			blob, err := json.Marshal(result)
+			if err != nil {
+				failed += 1
+				break
+			}
+			test.TxHashes = append(test.TxHashes, tx.Hash())
+			test.TraceConfigs = append(test.TraceConfigs, *config)
+			test.ResultHashes = append(test.ResultHashes, crypto.Keccak256Hash(blob))
+			writeTraceResult(outputDir, tx.Hash(), result, configName)
+		}
+		if time.Since(logged) > time.Second*8 {
+			logged = time.Now()
+			log.Info("Tracing transactions", "executed", len(test.TxHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start)))
+		}
+	}
+	log.Info("Traced transactions", "executed", len(test.TxHashes), "failed", failed, "elapsed", common.PrettyDuration(time.Since(start)))
+
+	// Write output file.
+	writeJSON(outputFile, test)
+	return nil
+}
+
+func randomTraceOption() (*tracers.TraceConfig, string) {
+	x := rand.Intn(11)
+	if x == 0 {
+		// struct-logger, with all fields enabled, very heavy
+		return &tracers.TraceConfig{
+			Config: &logger.Config{
+				EnableMemory:     true,
+				EnableReturnData: true,
+			},
+		}, "structAll"
+	}
+	if x == 1 {
+		// default options for struct-logger, with stack and storage capture
+		// enabled
+		return &tracers.TraceConfig{
+			Config: &logger.Config{},
+		}, "structDefault"
+	}
+	if x == 2 || x == 3 || x == 4 {
+		// struct-logger with storage capture enabled
+		return &tracers.TraceConfig{
+			Config: &logger.Config{
+				DisableStack: true,
+			},
+		}, "structStorage"
+	}
+	// Native tracer
+	loggers := []string{"callTracer", "4byteTracer", "flatCallTracer", "muxTracer", "noopTracer", "prestateTracer"}
+	return &tracers.TraceConfig{
+		Tracer: &loggers[x-5],
+	}, loggers[x-5]
+}
+
+func writeTraceResult(dir string, hash common.Hash, result any, configName string) {
+	if dir == "" {
+		return
+	}
+	name := filepath.Join(dir, configName+"_"+hash.String())
+	file, err := os.Create(name)
+	if err != nil {
+		exit(fmt.Errorf("error creating %s: %v", name, err))
+		return
+	}
+	defer file.Close()
+
+	data, _ := json.MarshalIndent(result, "", "    ")
+	_, err = file.Write(data)
+	if err != nil {
+		exit(fmt.Errorf("error writing %s: %v", name, err))
+		return
+	}
+}

common/lru/basiclru.go

@@ -47,30 +47,37 @@ func NewBasicLRU[K comparable, V any](capacity int) BasicLRU[K, V] {
 
 // Add adds a value to the cache. Returns true if an item was evicted to store the new item.
 func (c *BasicLRU[K, V]) Add(key K, value V) (evicted bool) {
+	_, _, evicted = c.Add3(key, value)
+	return evicted
+}
+
+// Add3 adds a value to the cache. If an item was evicted to store the new one, it returns the evicted item.
+func (c *BasicLRU[K, V]) Add3(key K, value V) (ek K, ev V, evicted bool) {
 	item, ok := c.items[key]
 	if ok {
-		// Already exists in cache.
 		item.value = value
 		c.items[key] = item
 		c.list.moveToFront(item.elem)
-		return false
+		return ek, ev, false
 	}
 
 	var elem *listElem[K]
 	if c.Len() >= c.cap {
 		elem = c.list.removeLast()
-		delete(c.items, elem.v)
 		evicted = true
+		ek = elem.v
+		ev = c.items[ek].value
+		delete(c.items, ek)
 	} else {
 		elem = new(listElem[K])
 	}
 
 	// Store the new item.
-	// Note that, if another item was evicted, we re-use its list element here.
+	// Note that if another item was evicted, we re-use its list element here.
 	elem.v = key
 	c.items[key] = cacheItem[K, V]{elem, value}
 	c.list.pushElem(elem)
-	return evicted
+	return ek, ev, evicted
 }
 
 // Contains reports whether the given key exists in the cache.

common/math/big.go

@@ -72,7 +72,7 @@ func (i *HexOrDecimal256) MarshalText() ([]byte, error) {
 	if i == nil {
 		return []byte("0x0"), nil
 	}
-	return []byte(fmt.Sprintf("%#x", (*big.Int)(i))), nil
+	return fmt.Appendf(nil, "%#x", (*big.Int)(i)), nil
 }
 
 // Decimal256 unmarshals big.Int as a decimal string. When unmarshalling,

common/math/integer.go

@@ -48,7 +48,7 @@ func (i *HexOrDecimal64) UnmarshalText(input []byte) error {
 
 // MarshalText implements encoding.TextMarshaler.
 func (i HexOrDecimal64) MarshalText() ([]byte, error) {
-	return []byte(fmt.Sprintf("%#x", uint64(i))), nil
+	return fmt.Appendf(nil, "%#x", uint64(i)), nil
 }
 
 // ParseUint64 parses s as an integer in decimal or hexadecimal syntax.

consensus/beacon/consensus.go

@@ -30,7 +30,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/holiman/uint256"
 )
@@ -445,11 +444,6 @@ func (beacon *Beacon) CalcDifficulty(chain consensus.ChainHeaderReader, time uin
 	return beaconDifficulty
 }
 
-// APIs implements consensus.Engine, returning the user facing RPC APIs.
-func (beacon *Beacon) APIs(chain consensus.ChainHeaderReader) []rpc.API {
-	return beacon.ethone.APIs(chain)
-}
-
 // Close shutdowns the consensus engine
 func (beacon *Beacon) Close() error {
 	return beacon.ethone.Close()

consensus/clique/api.go

@@ -1,235 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package clique
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/consensus"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/rpc"
-)
-
-// API is a user facing RPC API to allow controlling the signer and voting
-// mechanisms of the proof-of-authority scheme.
-type API struct {
-	chain  consensus.ChainHeaderReader
-	clique *Clique
-}
-
-// GetSnapshot retrieves the state snapshot at a given block.
-func (api *API) GetSnapshot(number *rpc.BlockNumber) (*Snapshot, error) {
-	// Retrieve the requested block number (or current if none requested)
-	var header *types.Header
-	if number == nil || *number == rpc.LatestBlockNumber {
-		header = api.chain.CurrentHeader()
-	} else {
-		header = api.chain.GetHeaderByNumber(uint64(number.Int64()))
-	}
-	// Ensure we have an actually valid block and return its snapshot
-	if header == nil {
-		return nil, errUnknownBlock
-	}
-	return api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
-}
-
-// GetSnapshotAtHash retrieves the state snapshot at a given block.
-func (api *API) GetSnapshotAtHash(hash common.Hash) (*Snapshot, error) {
-	header := api.chain.GetHeaderByHash(hash)
-	if header == nil {
-		return nil, errUnknownBlock
-	}
-	return api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
-}
-
-// GetSigners retrieves the list of authorized signers at the specified block.
-func (api *API) GetSigners(number *rpc.BlockNumber) ([]common.Address, error) {
-	// Retrieve the requested block number (or current if none requested)
-	var header *types.Header
-	if number == nil || *number == rpc.LatestBlockNumber {
-		header = api.chain.CurrentHeader()
-	} else {
-		header = api.chain.GetHeaderByNumber(uint64(number.Int64()))
-	}
-	// Ensure we have an actually valid block and return the signers from its snapshot
-	if header == nil {
-		return nil, errUnknownBlock
-	}
-	snap, err := api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
-	if err != nil {
-		return nil, err
-	}
-	return snap.signers(), nil
-}
-
-// GetSignersAtHash retrieves the list of authorized signers at the specified block.
-func (api *API) GetSignersAtHash(hash common.Hash) ([]common.Address, error) {
-	header := api.chain.GetHeaderByHash(hash)
-	if header == nil {
-		return nil, errUnknownBlock
-	}
-	snap, err := api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
-	if err != nil {
-		return nil, err
-	}
-	return snap.signers(), nil
-}
-
-// Proposals returns the current proposals the node tries to uphold and vote on.
-func (api *API) Proposals() map[common.Address]bool {
-	api.clique.lock.RLock()
-	defer api.clique.lock.RUnlock()
-
-	proposals := make(map[common.Address]bool)
-	for address, auth := range api.clique.proposals {
-		proposals[address] = auth
-	}
-	return proposals
-}
-
-// Propose injects a new authorization proposal that the signer will attempt to
-// push through.
-func (api *API) Propose(address common.Address, auth bool) {
-	api.clique.lock.Lock()
-	defer api.clique.lock.Unlock()
-
-	api.clique.proposals[address] = auth
-}
-
-// Discard drops a currently running proposal, stopping the signer from casting
-// further votes (either for or against).
-func (api *API) Discard(address common.Address) {
-	api.clique.lock.Lock()
-	defer api.clique.lock.Unlock()
-
-	delete(api.clique.proposals, address)
-}
-
-type status struct {
-	InturnPercent float64                `json:"inturnPercent"`
-	SigningStatus map[common.Address]int `json:"sealerActivity"`
-	NumBlocks     uint64                 `json:"numBlocks"`
-}
-
-// Status returns the status of the last N blocks,
-// - the number of active signers,
-// - the number of signers,
-// - the percentage of in-turn blocks
-func (api *API) Status() (*status, error) {
-	var (
-		numBlocks = uint64(64)
-		header    = api.chain.CurrentHeader()
-		diff      = uint64(0)
-		optimals  = 0
-	)
-	snap, err := api.clique.snapshot(api.chain, header.Number.Uint64(), header.Hash(), nil)
-	if err != nil {
-		return nil, err
-	}
-	var (
-		signers = snap.signers()
-		end     = header.Number.Uint64()
-		start   = end - numBlocks
-	)
-	if numBlocks > end {
-		start = 1
-		numBlocks = end - start
-	}
-	signStatus := make(map[common.Address]int)
-	for _, s := range signers {
-		signStatus[s] = 0
-	}
-	for n := start; n < end; n++ {
-		h := api.chain.GetHeaderByNumber(n)
-		if h == nil {
-			return nil, fmt.Errorf("missing block %d", n)
-		}
-		if h.Difficulty.Cmp(diffInTurn) == 0 {
-			optimals++
-		}
-		diff += h.Difficulty.Uint64()
-		sealer, err := api.clique.Author(h)
-		if err != nil {
-			return nil, err
-		}
-		signStatus[sealer]++
-	}
-	return &status{
-		InturnPercent: float64(100*optimals) / float64(numBlocks),
-		SigningStatus: signStatus,
-		NumBlocks:     numBlocks,
-	}, nil
-}
-
-type blockNumberOrHashOrRLP struct {
-	*rpc.BlockNumberOrHash
-	RLP hexutil.Bytes `json:"rlp,omitempty"`
-}
-
-func (sb *blockNumberOrHashOrRLP) UnmarshalJSON(data []byte) error {
-	bnOrHash := new(rpc.BlockNumberOrHash)
-	// Try to unmarshal bNrOrHash
-	if err := bnOrHash.UnmarshalJSON(data); err == nil {
-		sb.BlockNumberOrHash = bnOrHash
-		return nil
-	}
-	// Try to unmarshal RLP
-	var input string
-	if err := json.Unmarshal(data, &input); err != nil {
-		return err
-	}
-	blob, err := hexutil.Decode(input)
-	if err != nil {
-		return err
-	}
-	sb.RLP = blob
-	return nil
-}
-
-// GetSigner returns the signer for a specific clique block.
-// Can be called with a block number, a block hash or a rlp encoded blob.
-// The RLP encoded blob can either be a block or a header.
-func (api *API) GetSigner(rlpOrBlockNr *blockNumberOrHashOrRLP) (common.Address, error) {
-	if len(rlpOrBlockNr.RLP) == 0 {
-		blockNrOrHash := rlpOrBlockNr.BlockNumberOrHash
-		var header *types.Header
-		if blockNrOrHash == nil {
-			header = api.chain.CurrentHeader()
-		} else if hash, ok := blockNrOrHash.Hash(); ok {
-			header = api.chain.GetHeaderByHash(hash)
-		} else if number, ok := blockNrOrHash.Number(); ok {
-			header = api.chain.GetHeaderByNumber(uint64(number.Int64()))
-		}
-		if header == nil {
-			return common.Address{}, fmt.Errorf("missing block %v", blockNrOrHash.String())
-		}
-		return api.clique.Author(header)
-	}
-	block := new(types.Block)
-	if err := rlp.DecodeBytes(rlpOrBlockNr.RLP, block); err == nil {
-		return api.clique.Author(block.Header())
-	}
-	header := new(types.Header)
-	if err := rlp.DecodeBytes(rlpOrBlockNr.RLP, header); err != nil {
-		return common.Address{}, err
-	}
-	return api.clique.Author(header)
-}

consensus/clique/clique.go

@@ -41,7 +41,6 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/trie"
 	"golang.org/x/crypto/sha3"
 )
@@ -641,15 +640,6 @@ func (c *Clique) Close() error {
 	return nil
 }
 
-// APIs implements consensus.Engine, returning the user facing RPC API to allow
-// controlling the signer voting.
-func (c *Clique) APIs(chain consensus.ChainHeaderReader) []rpc.API {
-	return []rpc.API{{
-		Namespace: "clique",
-		Service:   &API{chain: chain, clique: c},
-	}}
-}
-
 // SealHash returns the hash of a block prior to it being sealed.
 func SealHash(header *types.Header) (hash common.Hash) {
 	hasher := sha3.NewLegacyKeccak256()

consensus/consensus.go

@@ -25,7 +25,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rpc"
 )
 
 // ChainHeaderReader defines a small collection of methods needed to access the local
@@ -109,9 +108,6 @@ type Engine interface {
 	// that a new block should have.
 	CalcDifficulty(chain ChainHeaderReader, time uint64, parent *types.Header) *big.Int
 
-	// APIs returns the RPC APIs this consensus engine provides.
-	APIs(chain ChainHeaderReader) []rpc.API
-
 	// Close terminates any background threads maintained by the consensus engine.
 	Close() error
 }

consensus/ethash/ethash.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/rpc"
 )
 
 // Ethash is a consensus engine based on proof-of-work implementing the ethash
@@ -71,12 +70,6 @@ func (ethash *Ethash) Close() error {
 	return nil
 }
 
-// APIs implements consensus.Engine, returning no APIs as ethash is an empty
-// shell in the post-merge world.
-func (ethash *Ethash) APIs(chain consensus.ChainHeaderReader) []rpc.API {
-	return []rpc.API{}
-}
-
 // Seal generates a new sealing request for the given input block and pushes
 // the result into the given channel. For the ethash engine, this method will
 // just panic as sealing is not supported anymore.

core/block_validator_test.go

@@ -50,8 +50,11 @@ func testHeaderVerification(t *testing.T, scheme string) {
 		headers[i] = block.Header()
 	}
 	// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
-	chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
+	chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
 	defer chain.Stop()
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	for i := 0; i < len(blocks); i++ {
 		for j, valid := range []bool{true, false} {
@@ -163,8 +166,11 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) {
 		postHeaders[i] = block.Header()
 	}
 	// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
-	chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil)
+	chain, err := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, engine, vm.Config{}, nil)
 	defer chain.Stop()
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	// Verify the blocks before the merging
 	for i := 0; i < len(preBlocks); i++ {

core/blockchain.go

@@ -21,9 +21,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math"
 	"math/big"
 	"runtime"
 	"slices"
+	"sort"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -35,6 +37,7 @@ import (
 	"github.com/ethereum/go-ethereum/common/prque"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
+	"github.com/ethereum/go-ethereum/core/history"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
@@ -63,6 +66,7 @@ var (
 	headSafeBlockGauge      = metrics.NewRegisteredGauge("chain/head/safe", nil)
 
 	chainInfoGauge   = metrics.NewRegisteredGaugeInfo("chain/info", nil)
+	chainMgaspsMeter = metrics.NewRegisteredResettingTimer("chain/mgasps", nil)
 
 	accountReadTimer   = metrics.NewRegisteredResettingTimer("chain/account/reads", nil)
 	accountHashTimer   = metrics.NewRegisteredResettingTimer("chain/account/hashes", nil)
@@ -89,8 +93,10 @@ var (
 	blockReorgAddMeter  = metrics.NewRegisteredMeter("chain/reorg/add", nil)
 	blockReorgDropMeter = metrics.NewRegisteredMeter("chain/reorg/drop", nil)
 
-	blockPrefetchExecuteTimer   = metrics.NewRegisteredTimer("chain/prefetch/executes", nil)
+	blockPrefetchExecuteTimer    = metrics.NewRegisteredResettingTimer("chain/prefetch/executes", nil)
 	blockPrefetchInterruptMeter  = metrics.NewRegisteredMeter("chain/prefetch/interrupts", nil)
+	blockPrefetchTxsInvalidMeter = metrics.NewRegisteredMeter("chain/prefetch/txs/invalid", nil)
+	blockPrefetchTxsValidMeter   = metrics.NewRegisteredMeter("chain/prefetch/txs/valid", nil)
 
 	errInsertionInterrupted = errors.New("insertion is interrupted")
 	errChainStopped         = errors.New("blockchain is stopped")
@@ -98,6 +104,10 @@ var (
 	errInvalidNewChain      = errors.New("invalid new chain")
 )
 
+var (
+	forkReadyInterval = 3 * time.Minute
+)
+
 const (
 	bodyCacheLimit     = 256
 	blockCacheLimit    = 256
@@ -157,7 +167,7 @@ type CacheConfig struct {
 
 	// This defines the cutoff block for history expiry.
 	// Blocks before this number may be unavailable in the chain database.
-	HistoryPruningCutoff uint64
+	ChainHistoryMode history.HistoryMode
 }
 
 // triedbConfig derives the configures for trie database.
@@ -174,7 +184,12 @@ func (c *CacheConfig) triedbConfig(isVerkle bool) *triedb.Config {
 	if c.StateScheme == rawdb.PathScheme {
 		config.PathDB = &pathdb.Config{
 			StateHistory:   c.StateHistory,
-			CleanCacheSize:  c.TrieCleanLimit * 1024 * 1024,
+			TrieCleanSize:  c.TrieCleanLimit * 1024 * 1024,
+			StateCleanSize: c.SnapshotLimit * 1024 * 1024,
+
+			// TODO(rjl493456442): The write buffer represents the memory limit used
+			// for flushing both trie data and state data to disk. The config name
+			// should be updated to eliminate the confusion.
 			WriteBufferSize: c.TrieDirtyLimit * 1024 * 1024,
 		}
 	}
@@ -253,6 +268,7 @@ type BlockChain struct {
 	currentSnapBlock  atomic.Pointer[types.Header] // Current head of snap-sync
 	currentFinalBlock atomic.Pointer[types.Header] // Latest (consensus) finalized block
 	currentSafeBlock  atomic.Pointer[types.Header] // Latest (consensus) safe block
+	historyPrunePoint atomic.Pointer[history.PrunePoint]
 
 	bodyCache     *lru.Cache[common.Hash, *types.Body]
 	bodyRLPCache  *lru.Cache[common.Hash, rlp.RawValue]
@@ -262,7 +278,6 @@ type BlockChain struct {
 	txLookupLock  sync.RWMutex
 	txLookupCache *lru.Cache[common.Hash, txLookup]
 
-	wg            sync.WaitGroup
 	quit          chan struct{} // shutdown signal, closed in Stop.
 	stopping      atomic.Bool   // false if chain is running, true when stopped
 	procInterrupt atomic.Bool   // interrupt signaler for block processing
@@ -273,6 +288,8 @@ type BlockChain struct {
 	processor  Processor // Block transaction processor interface
 	vmConfig   vm.Config
 	logger     *tracing.Hooks
+
+	lastForkReadyAlert time.Time // Last time there was a fork readiness print out
 }
 
 // NewBlockChain returns a fully initialised block chain using information
@@ -332,10 +349,11 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	bc.prefetcher = newStatePrefetcher(chainConfig, bc.hc)
 	bc.processor = NewStateProcessor(chainConfig, bc.hc)
 
-	bc.genesisBlock = bc.GetBlockByNumber(0)
+	genesisHeader := bc.GetHeaderByNumber(0)
-	if bc.genesisBlock == nil {
+	if genesisHeader == nil {
 		return nil, ErrNoGenesis
 	}
+	bc.genesisBlock = types.NewBlockWithHeader(genesisHeader)
 
 	bc.currentBlock.Store(nil)
 	bc.currentSnapBlock.Store(nil)
@@ -367,11 +385,14 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 			// Do nothing here until the state syncer picks it up.
 			log.Info("Genesis state is missing, wait state sync")
 		} else {
-			// Head state is missing, before the state recovery, find out the
+			// Head state is missing, before the state recovery, find out the disk
-			// disk layer point of snapshot(if it's enabled). Make sure the
+			// layer point of snapshot(if it's enabled). Make sure the rewound point
-			// rewound point is lower than disk layer.
+			// is lower than disk layer.
+			//
+			// Note it's unnecessary in path mode which always keep trie data and
+			// state data consistent.
 			var diskRoot common.Hash
-			if bc.cacheConfig.SnapshotLimit > 0 {
+			if bc.cacheConfig.SnapshotLimit > 0 && bc.cacheConfig.StateScheme == rawdb.HashScheme {
 				diskRoot = rawdb.ReadSnapshotRoot(bc.db)
 			}
 			if diskRoot != (common.Hash{}) {
@@ -444,7 +465,32 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 			bc.logger.OnGenesisBlock(bc.genesisBlock, alloc)
 		}
 	}
+	bc.setupSnapshot()
 
+	// Rewind the chain in case of an incompatible config upgrade.
+	if compatErr != nil {
+		log.Warn("Rewinding chain to upgrade configuration", "err", compatErr)
+		if compatErr.RewindToTime > 0 {
+			bc.SetHeadWithTimestamp(compatErr.RewindToTime)
+		} else {
+			bc.SetHead(compatErr.RewindToBlock)
+		}
+		rawdb.WriteChainConfig(db, genesisHash, chainConfig)
+	}
+
+	// Start tx indexer if it's enabled.
+	if txLookupLimit != nil {
+		bc.txIndexer = newTxIndexer(*txLookupLimit, bc)
+	}
+	return bc, nil
+}
+
+func (bc *BlockChain) setupSnapshot() {
+	// Short circuit if the chain is established with path scheme, as the
+	// state snapshot has been integrated into path database natively.
+	if bc.cacheConfig.StateScheme == rawdb.PathScheme {
+		return
+	}
 	// Load any existing snapshot, regenerating it if loading failed
 	if bc.cacheConfig.SnapshotLimit > 0 {
 		// If the chain was rewound past the snapshot persistent layer (causing
@@ -452,7 +498,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 		// in recovery mode and in that case, don't invalidate the snapshot on a
 		// head mismatch.
 		var recover bool
-
 		head := bc.CurrentBlock()
 		if layer := rawdb.ReadSnapshotRecoveryNumber(bc.db); layer != nil && *layer >= head.Number.Uint64() {
 			log.Warn("Enabling snapshot recovery", "chainhead", head.Number, "diskbase", *layer)
@@ -469,22 +514,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 		// Re-initialize the state database with snapshot
 		bc.statedb = state.NewDatabase(bc.triedb, bc.snaps)
 	}
-
-	// Rewind the chain in case of an incompatible config upgrade.
-	if compatErr != nil {
-		log.Warn("Rewinding chain to upgrade configuration", "err", compatErr)
-		if compatErr.RewindToTime > 0 {
-			bc.SetHeadWithTimestamp(compatErr.RewindToTime)
-		} else {
-			bc.SetHead(compatErr.RewindToBlock)
-		}
-		rawdb.WriteChainConfig(db, genesisHash, chainConfig)
-	}
-	// Start tx indexer if it's enabled.
-	if txLookupLimit != nil {
-		bc.txIndexer = newTxIndexer(*txLookupLimit, bc)
-	}
-	return bc, nil
 }
 
 // empty returns an indicator whether the blockchain is empty.
@@ -511,19 +540,33 @@ func (bc *BlockChain) loadLastState() error {
 		log.Warn("Empty database, resetting chain")
 		return bc.Reset()
 	}
-	// Make sure the entire head block is available
+	headHeader := bc.GetHeaderByHash(head)
-	headBlock := bc.GetBlockByHash(head)
+	if headHeader == nil {
+		// Corrupt or empty database, init from scratch
+		log.Warn("Head header missing, resetting chain", "hash", head)
+		return bc.Reset()
+	}
+
+	var headBlock *types.Block
+	if cmp := headHeader.Number.Cmp(new(big.Int)); cmp == 1 {
+		// Make sure the entire head block is available.
+		headBlock = bc.GetBlockByHash(head)
+	} else if cmp == 0 {
+		// On a pruned node the block body might not be available. But a pruned
+		// block should never be the head block. The only exception is when, as
+		// a last resort, chain is reset to genesis.
+		headBlock = bc.genesisBlock
+	}
 	if headBlock == nil {
 		// Corrupt or empty database, init from scratch
 		log.Warn("Head block missing, resetting chain", "hash", head)
 		return bc.Reset()
 	}
 	// Everything seems to be fine, set as the head block
-	bc.currentBlock.Store(headBlock.Header())
+	bc.currentBlock.Store(headHeader)
 	headBlockGauge.Update(int64(headBlock.NumberU64()))
 
 	// Restore the last known head header
-	headHeader := headBlock.Header()
 	if head := rawdb.ReadHeadHeaderHash(bc.db); head != (common.Hash{}) {
 		if header := bc.GetHeaderByHash(head); header != nil {
 			headHeader = header
@@ -531,6 +574,12 @@ func (bc *BlockChain) loadLastState() error {
 	}
 	bc.hc.SetCurrentHeader(headHeader)
 
+	// Initialize history pruning.
+	latest := max(headBlock.NumberU64(), headHeader.Number.Uint64())
+	if err := bc.initializeHistoryPruning(latest); err != nil {
+		return err
+	}
+
 	// Restore the last known head snap block
 	bc.currentSnapBlock.Store(headBlock.Header())
 	headFastBlockGauge.Update(int64(headBlock.NumberU64()))
@@ -553,6 +602,7 @@ func (bc *BlockChain) loadLastState() error {
 			headSafeBlockGauge.Update(int64(block.NumberU64()))
 		}
 	}
+
 	// Issue a status log for the user
 	var (
 		currentSnapBlock  = bc.CurrentSnapBlock()
@@ -571,9 +621,57 @@ func (bc *BlockChain) loadLastState() error {
 	if pivot := rawdb.ReadLastPivotNumber(bc.db); pivot != nil {
 		log.Info("Loaded last snap-sync pivot marker", "number", *pivot)
 	}
+	if pruning := bc.historyPrunePoint.Load(); pruning != nil {
+		log.Info("Chain history is pruned", "earliest", pruning.BlockNumber, "hash", pruning.BlockHash)
+	}
 	return nil
 }
 
+// initializeHistoryPruning sets bc.historyPrunePoint.
+func (bc *BlockChain) initializeHistoryPruning(latest uint64) error {
+	freezerTail, _ := bc.db.Tail()
+
+	switch bc.cacheConfig.ChainHistoryMode {
+	case history.KeepAll:
+		if freezerTail == 0 {
+			return nil
+		}
+		// The database was pruned somehow, so we need to figure out if it's a known
+		// configuration or an error.
+		predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()]
+		if predefinedPoint == nil || freezerTail != predefinedPoint.BlockNumber {
+			log.Error("Chain history database is pruned with unknown configuration", "tail", freezerTail)
+			return fmt.Errorf("unexpected database tail")
+		}
+		bc.historyPrunePoint.Store(predefinedPoint)
+		return nil
+
+	case history.KeepPostMerge:
+		if freezerTail == 0 && latest != 0 {
+			// This is the case where a user is trying to run with --history.chain
+			// postmerge directly on an existing DB. We could just trigger the pruning
+			// here, but it'd be a bit dangerous since they may not have intended this
+			// action to happen. So just tell them how to do it.
+			log.Error(fmt.Sprintf("Chain history mode is configured as %q, but database is not pruned.", bc.cacheConfig.ChainHistoryMode.String()))
+			log.Error(fmt.Sprintf("Run 'geth prune-history' to prune pre-merge history."))
+			return fmt.Errorf("history pruning requested via configuration")
+		}
+		predefinedPoint := history.PrunePoints[bc.genesisBlock.Hash()]
+		if predefinedPoint == nil {
+			log.Error("Chain history pruning is not supported for this network", "genesis", bc.genesisBlock.Hash())
+			return fmt.Errorf("history pruning requested for unknown network")
+		} else if freezerTail > 0 && freezerTail != predefinedPoint.BlockNumber {
+			log.Error("Chain history database is pruned to unknown block", "tail", freezerTail)
+			return fmt.Errorf("unexpected database tail")
+		}
+		bc.historyPrunePoint.Store(predefinedPoint)
+		return nil
+
+	default:
+		return fmt.Errorf("invalid history mode: %d", bc.cacheConfig.ChainHistoryMode)
+	}
+}
+
 // SetHead rewinds the local chain to a new head. Depending on whether the node
 // was snap synced or full synced and in which state, the method will try to
 // delete minimal data from disk whilst retaining chain consistency.
@@ -584,12 +682,16 @@ func (bc *BlockChain) SetHead(head uint64) error {
 	// Send chain head event to update the transaction pool
 	header := bc.CurrentBlock()
 	if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil {
+		// In a pruned node the genesis block will not exist in the freezer.
+		// It should not happen that we set head to any other pruned block.
+		if header.Number.Uint64() > 0 {
 			// This should never happen. In practice, previously currentBlock
 			// contained the entire block whereas now only a "marker", so there
 			// is an ever so slight chance for a race we should handle.
 			log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash())
 			return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4])
 		}
+	}
 	bc.chainHeadFeed.Send(ChainHeadEvent{Header: header})
 	return nil
 }
@@ -605,12 +707,16 @@ func (bc *BlockChain) SetHeadWithTimestamp(timestamp uint64) error {
 	// Send chain head event to update the transaction pool
 	header := bc.CurrentBlock()
 	if block := bc.GetBlock(header.Hash(), header.Number.Uint64()); block == nil {
+		// In a pruned node the genesis block will not exist in the freezer.
+		// It should not happen that we set head to any other pruned block.
+		if header.Number.Uint64() > 0 {
 			// This should never happen. In practice, previously currentBlock
 			// contained the entire block whereas now only a "marker", so there
 			// is an ever so slight chance for a race we should handle.
 			log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash())
 			return fmt.Errorf("current block missing: #%d [%x..]", header.Number, header.Hash().Bytes()[:4])
 		}
+	}
 	bc.chainHeadFeed.Send(ChainHeadEvent{Header: header})
 	return nil
 }
@@ -892,21 +998,20 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 		// Ignore the error here since light client won't hit this path
 		frozen, _ := bc.db.Ancients()
 		if num+1 <= frozen {
-			// Truncate all relative data(header, total difficulty, body, receipt
+			// The chain segment, such as the block header, canonical hash,
-			// and canonical hash) from ancient store.
+			// body, and receipt, will be removed from the ancient store
-			if _, err := bc.db.TruncateHead(num); err != nil {
+			// in one go.
-				log.Crit("Failed to truncate ancient data", "number", num, "err", err)
+			//
-			}
+			// The hash-to-number mapping in the key-value store will be
-			// Remove the hash <-> number mapping from the active store.
+			// removed by the hc.SetHead function.
-			rawdb.DeleteHeaderNumber(db, hash)
 		} else {
-			// Remove relative body and receipts from the active store.
+			// Remove the associated body and receipts from the key-value store.
-			// The header, total difficulty and canonical hash will be
+			// The header, hash-to-number mapping, and canonical hash will be
-			// removed in the hc.SetHead function.
+			// removed by the hc.SetHead function.
 			rawdb.DeleteBody(db, hash, num)
 			rawdb.DeleteReceipts(db, hash, num)
 		}
-		// Todo(rjl493456442) txlookup, bloombits, etc
+		// Todo(rjl493456442) txlookup, log index, etc
 	}
 	// If SetHead was only called as a chain reparation method, try to skip
 	// touching the header chain altogether, unless the freezer is broken
@@ -1012,7 +1117,9 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
 	bc.hc.SetCurrentHeader(bc.genesisBlock.Header())
 	bc.currentSnapBlock.Store(bc.genesisBlock.Header())
 	headFastBlockGauge.Update(int64(bc.genesisBlock.NumberU64()))
-	return nil
+
+	// Reset history pruning status.
+	return bc.initializeHistoryPruning(0)
 }
 
 // Export writes the active chain to the given writer.
@@ -1109,7 +1216,6 @@ func (bc *BlockChain) stopWithoutSaving() {
 	// the mutex should become available quickly. It cannot be taken again after Close has
 	// returned.
 	bc.chainmu.Close()
-	bc.wg.Wait()
 }
 
 // Stop stops the blockchain service. If any imports are currently in progress
@@ -1196,81 +1302,65 @@ const (
 	SideStatTy
 )
 
-// InsertReceiptChain attempts to complete an already existing header chain with
+// InsertReceiptChain inserts a batch of blocks along with their receipts into
-// transaction and receipt data.
+// the database. Unlike InsertChain, this function does not verify the state root
-func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts, ancientLimit uint64) (int, error) {
+// in the blocks. It is used exclusively for snap sync. All the inserted blocks
-	// We don't require the chainMu here since we want to maximize the
+// will be regarded as canonical, chain reorg is not supported.
-	// concurrency of header insertion and receipt insertion.
+//
-	bc.wg.Add(1)
+// The optional ancientLimit can also be specified and chain segment before that
-	defer bc.wg.Done()
+// will be directly stored in the ancient, getting rid of the chain migration.
-
+func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []rlp.RawValue, ancientLimit uint64) (int, error) {
-	var (
+	// Verify the supplied headers before insertion without lock
-		ancientBlocks, liveBlocks     types.Blocks
+	var headers []*types.Header
-		ancientReceipts, liveReceipts []types.Receipts
+	for _, block := range blockChain {
-	)
+		headers = append(headers, block.Header())
-	// Do a sanity check that the provided chain is actually ordered and linked
+		// Here we also validate that blob transactions in the block do not
-	for i, block := range blockChain {
+		// contain a sidecar. While the sidecar does not affect the block hash
-		if i != 0 {
+		// or tx hash, sending blobs within a block is not allowed.
-			prev := blockChain[i-1]
-			if block.NumberU64() != prev.NumberU64()+1 || block.ParentHash() != prev.Hash() {
-				log.Error("Non contiguous receipt insert",
-					"number", block.Number(), "hash", block.Hash(), "parent", block.ParentHash(),
-					"prevnumber", prev.Number(), "prevhash", prev.Hash())
-				return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])",
-					i-1, prev.NumberU64(), prev.Hash().Bytes()[:4],
-					i, block.NumberU64(), block.Hash().Bytes()[:4], block.ParentHash().Bytes()[:4])
-			}
-		}
-		if block.NumberU64() <= ancientLimit {
-			ancientBlocks, ancientReceipts = append(ancientBlocks, block), append(ancientReceipts, receiptChain[i])
-		} else {
-			liveBlocks, liveReceipts = append(liveBlocks, block), append(liveReceipts, receiptChain[i])
-		}
-
-		// Here we also validate that blob transactions in the block do not contain a sidecar.
-		// While the sidecar does not affect the block hash / tx hash, sending blobs within a block is not allowed.
 		for txIndex, tx := range block.Transactions() {
 			if tx.Type() == types.BlobTxType && tx.BlobTxSidecar() != nil {
 				return 0, fmt.Errorf("block #%d contains unexpected blob sidecar in tx at index %d", block.NumberU64(), txIndex)
 			}
 		}
 	}
+	if n, err := bc.hc.ValidateHeaderChain(headers); err != nil {
+		return n, err
+	}
+	// Hold the mutation lock
+	if !bc.chainmu.TryLock() {
+		return 0, errChainStopped
+	}
+	defer bc.chainmu.Unlock()
 
 	var (
 		stats = struct{ processed, ignored int32 }{}
 		start = time.Now()
 		size  = int64(0)
 	)
-
+	// updateHead updates the head header and head snap block flags.
-	// updateHead updates the head snap sync block if the inserted blocks are better
+	updateHead := func(header *types.Header) error {
-	// and returns an indicator whether the inserted blocks are canonical.
+		batch := bc.db.NewBatch()
-	updateHead := func(head *types.Block) bool {
+		hash := header.Hash()
-		if !bc.chainmu.TryLock() {
+		rawdb.WriteHeadHeaderHash(batch, hash)
-			return false
+		rawdb.WriteHeadFastBlockHash(batch, hash)
+		if err := batch.Write(); err != nil {
+			return err
 		}
-		defer bc.chainmu.Unlock()
+		bc.hc.currentHeader.Store(header)
-
+		bc.currentSnapBlock.Store(header)
-		// Rewind may have occurred, skip in that case.
+		headHeaderGauge.Update(header.Number.Int64())
-		if bc.CurrentHeader().Number.Cmp(head.Number()) >= 0 {
+		headFastBlockGauge.Update(header.Number.Int64())
-			rawdb.WriteHeadFastBlockHash(bc.db, head.Hash())
+		return nil
-			bc.currentSnapBlock.Store(head.Header())
-			headFastBlockGauge.Update(int64(head.NumberU64()))
-			return true
-		}
-		return false
 	}
 	// writeAncient writes blockchain and corresponding receipt chain into ancient store.
 	//
 	// this function only accepts canonical chain data. All side chain will be reverted
 	// eventually.
-	writeAncient := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) {
+	writeAncient := func(blockChain types.Blocks, receiptChain []rlp.RawValue) (int, error) {
-		first := blockChain[0]
+		// Ensure genesis is in the ancient store
-		last := blockChain[len(blockChain)-1]
+		if blockChain[0].NumberU64() == 1 {
-
-		// Ensure genesis is in ancients.
-		if first.NumberU64() == 1 {
 			if frozen, _ := bc.db.Ancients(); frozen == 0 {
-				writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil})
+				writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []rlp.RawValue{rlp.EmptyList})
 				if err != nil {
 					log.Error("Error writing genesis to ancients", "err", err)
 					return 0, err
@@ -1279,12 +1369,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 				log.Info("Wrote genesis to ancients")
 			}
 		}
-		// Before writing the blocks to the ancients, we need to ensure that
-		// they correspond to the what the headerchain 'expects'.
-		// We only check the last block/header, since it's a contiguous chain.
-		if !bc.HasHeader(last.Hash(), last.NumberU64()) {
-			return 0, fmt.Errorf("containing header #%d [%x..] unknown", last.Number(), last.Hash().Bytes()[:4])
-		}
 		// Write all chain data to ancients.
 		writeSize, err := rawdb.WriteAncientBlocks(bc.db, blockChain, receiptChain)
 		if err != nil {
@@ -1294,48 +1378,32 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 		size += writeSize
 
 		// Sync the ancient store explicitly to ensure all data has been flushed to disk.
-		if err := bc.db.Sync(); err != nil {
+		if err := bc.db.SyncAncient(); err != nil {
+			return 0, err
+		}
+		// Write hash to number mappings
+		batch := bc.db.NewBatch()
+		for _, block := range blockChain {
+			rawdb.WriteHeaderNumber(batch, block.Hash(), block.NumberU64())
+		}
+		if err := batch.Write(); err != nil {
 			return 0, err
 		}
 		// Update the current snap block because all block data is now present in DB.
-		previousSnapBlock := bc.CurrentSnapBlock().Number.Uint64()
+		if err := updateHead(blockChain[len(blockChain)-1].Header()); err != nil {
-		if !updateHead(blockChain[len(blockChain)-1]) {
-			// We end up here if the header chain has reorg'ed, and the blocks/receipts
-			// don't match the canonical chain.
-			if _, err := bc.db.TruncateHead(previousSnapBlock + 1); err != nil {
-				log.Error("Can't truncate ancient store after failed insert", "err", err)
-			}
-			return 0, errSideChainReceipts
-		}
-
-		// Delete block data from the main database.
-		var (
-			batch       = bc.db.NewBatch()
-			canonHashes = make(map[common.Hash]struct{}, len(blockChain))
-		)
-		for _, block := range blockChain {
-			canonHashes[block.Hash()] = struct{}{}
-			if block.NumberU64() == 0 {
-				continue
-			}
-			rawdb.DeleteCanonicalHash(batch, block.NumberU64())
-			rawdb.DeleteBlockWithoutNumber(batch, block.Hash(), block.NumberU64())
-		}
-		// Delete side chain hash-to-number mappings.
-		for _, nh := range rawdb.ReadAllHashesInRange(bc.db, first.NumberU64(), last.NumberU64()) {
-			if _, canon := canonHashes[nh.Hash]; !canon {
-				rawdb.DeleteHeader(batch, nh.Hash, nh.Number)
-			}
-		}
-		if err := batch.Write(); err != nil {
 			return 0, err
 		}
 		stats.processed += int32(len(blockChain))
 		return 0, nil
 	}
 
-	// writeLive writes blockchain and corresponding receipt chain into active store.
+	// writeLive writes the blockchain and corresponding receipt chain to the active store.
-	writeLive := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) {
+	//
+	// Notably, in different snap sync cycles, the supplied chain may partially reorganize
+	// existing local chain segments (reorg around the chain tip). The reorganized part
+	// will be included in the provided chain segment, and stale canonical markers will be
+	// silently rewritten. Therefore, no explicit reorg logic is needed.
+	writeLive := func(blockChain types.Blocks, receiptChain []rlp.RawValue) (int, error) {
 		var (
 			skipPresenceCheck = false
 			batch             = bc.db.NewBatch()
@@ -1345,10 +1413,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 			if bc.insertStopped() {
 				return 0, errInsertionInterrupted
 			}
-			// Short circuit if the owner header is unknown
-			if !bc.HasHeader(block.Hash(), block.NumberU64()) {
-				return i, fmt.Errorf("containing header #%d [%x..] unknown", block.Number(), block.Hash().Bytes()[:4])
-			}
 			if !skipPresenceCheck {
 				// Ignore if the entire data is already known
 				if bc.HasBlock(block.Hash(), block.NumberU64()) {
@@ -1362,8 +1426,9 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 				}
 			}
 			// Write all the data out into the database
-			rawdb.WriteBody(batch, block.Hash(), block.NumberU64(), block.Body())
+			rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64())
-			rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i])
+			rawdb.WriteBlock(batch, block)
+			rawdb.WriteRawReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i])
 
 			// Write everything belongs to the blocks into the database. So that
 			// we can ensure all components of body is completed(body, receipts)
@@ -1386,21 +1451,27 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 				return 0, err
 			}
 		}
-		updateHead(blockChain[len(blockChain)-1])
+		if err := updateHead(blockChain[len(blockChain)-1].Header()); err != nil {
+			return 0, err
+		}
 		return 0, nil
 	}
 
-	// Write downloaded chain data and corresponding receipt chain data
+	// Split the supplied blocks into two groups, according to the
-	if len(ancientBlocks) > 0 {
+	// given ancient limit.
-		if n, err := writeAncient(ancientBlocks, ancientReceipts); err != nil {
+	index := sort.Search(len(blockChain), func(i int) bool {
+		return blockChain[i].NumberU64() >= ancientLimit
+	})
+	if index > 0 {
+		if n, err := writeAncient(blockChain[:index], receiptChain[:index]); err != nil {
 			if err == errInsertionInterrupted {
 				return 0, nil
 			}
 			return n, err
 		}
 	}
-	if len(liveBlocks) > 0 {
+	if index != len(blockChain) {
-		if n, err := writeLive(liveBlocks, liveReceipts); err != nil {
+		if n, err := writeLive(blockChain[index:], receiptChain[index:]); err != nil {
 			if err == errInsertionInterrupted {
 				return 0, nil
 			}
@@ -1419,7 +1490,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 		context = append(context, []interface{}{"ignored", stats.ignored}...)
 	}
 	log.Debug("Imported new block receipts", context...)
-
 	return 0, nil
 }
 
@@ -1706,18 +1776,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness
 		bc.reportBlock(block, nil, err)
 		return nil, it.index, err
 	}
-	// No validation errors for the first block (or chain prefix skipped)
-	var activeState *state.StateDB
-	defer func() {
-		// The chain importer is starting and stopping trie prefetchers. If a bad
-		// block or other error is hit however, an early return may not properly
-		// terminate the background threads. This defer ensures that we clean up
-		// and dangling prefetcher, without deferring each and holding on live refs.
-		if activeState != nil {
-			activeState.StopPrefetcher()
-		}
-	}()
-
 	// Track the singleton witness from this chain insertion (if any)
 	var witness *stateless.Witness
 
@@ -1773,63 +1831,20 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness
 			continue
 		}
 		// Retrieve the parent block and it's state to execute on top
-		start := time.Now()
 		parent := it.previous()
 		if parent == nil {
 			parent = bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
 		}
-		statedb, err := state.New(parent.Root, bc.statedb)
-		if err != nil {
-			return nil, it.index, err
-		}
-
-		// If we are past Byzantium, enable prefetching to pull in trie node paths
-		// while processing transactions. Before Byzantium the prefetcher is mostly
-		// useless due to the intermediate root hashing after each transaction.
-		if bc.chainConfig.IsByzantium(block.Number()) {
-			// Generate witnesses either if we're self-testing, or if it's the
-			// only block being inserted. A bit crude, but witnesses are huge,
-			// so we refuse to make an entire chain of them.
-			if bc.vmConfig.StatelessSelfValidation || (makeWitness && len(chain) == 1) {
-				witness, err = stateless.NewWitness(block.Header(), bc)
-				if err != nil {
-					return nil, it.index, err
-				}
-			}
-			statedb.StartPrefetcher("chain", witness)
-		}
-		activeState = statedb
-
-		// If we have a followup block, run that against the current state to pre-cache
-		// transactions and probabilistically some of the account/storage trie nodes.
-		var followupInterrupt atomic.Bool
-		if !bc.cacheConfig.TrieCleanNoPrefetch {
-			if followup, err := it.peek(); followup != nil && err == nil {
-				throwaway, _ := state.New(parent.Root, bc.statedb)
-
-				go func(start time.Time, followup *types.Block, throwaway *state.StateDB) {
-					// Disable tracing for prefetcher executions.
-					vmCfg := bc.vmConfig
-					vmCfg.Tracer = nil
-					bc.prefetcher.Prefetch(followup, throwaway, vmCfg, &followupInterrupt)
-
-					blockPrefetchExecuteTimer.Update(time.Since(start))
-					if followupInterrupt.Load() {
-						blockPrefetchInterruptMeter.Mark(1)
-					}
-				}(time.Now(), followup, throwaway)
-			}
-		}
-
 		// The traced section of block import.
-		res, err := bc.processBlock(block, statedb, start, setHead)
+		start := time.Now()
-		followupInterrupt.Store(true)
+		res, err := bc.processBlock(parent.Root, block, setHead, makeWitness && len(chain) == 1)
 		if err != nil {
 			return nil, it.index, err
 		}
 		// Report the import stats before returning the various results
 		stats.processed++
 		stats.usedGas += res.usedGas
+		witness = res.witness
 
 		var snapDiffItems, snapBufItems common.StorageSize
 		if bc.snaps != nil {
@@ -1838,6 +1853,9 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness
 		trieDiffNodes, trieBufNodes, _ := bc.triedb.Size()
 		stats.report(chain, it.index, snapDiffItems, snapBufItems, trieDiffNodes, trieBufNodes, setHead)
 
+		// Print confirmation that a future fork is scheduled, but not yet active.
+		bc.logForkReadiness(block)
+
 		if !setHead {
 			// After merge we expect few side chains. Simply count
 			// all blocks the CL gives us for GC processing time
@@ -1871,6 +1889,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness
 				"root", block.Root())
 		}
 	}
+
 	stats.ignored += it.remaining()
 	return witness, it.index, err
 }
@@ -1881,11 +1900,74 @@ type blockProcessingResult struct {
 	usedGas  uint64
 	procTime time.Duration
 	status   WriteStatus
+	witness  *stateless.Witness
 }
 
 // processBlock executes and validates the given block. If there was no error
 // it writes the block and associated state to database.
-func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, start time.Time, setHead bool) (_ *blockProcessingResult, blockEndErr error) {
+func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, setHead bool, makeWitness bool) (_ *blockProcessingResult, blockEndErr error) {
+	var (
+		err       error
+		startTime = time.Now()
+		statedb   *state.StateDB
+		interrupt atomic.Bool
+	)
+	defer interrupt.Store(true) // terminate the prefetch at the end
+
+	if bc.cacheConfig.TrieCleanNoPrefetch {
+		statedb, err = state.New(parentRoot, bc.statedb)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		// If prefetching is enabled, run that against the current state to pre-cache
+		// transactions and probabilistically some of the account/storage trie nodes.
+		//
+		// Note: the main processor and prefetcher share the same reader with a local
+		// cache for mitigating the overhead of state access.
+		reader, err := bc.statedb.ReaderWithCache(parentRoot)
+		if err != nil {
+			return nil, err
+		}
+		throwaway, err := state.NewWithReader(parentRoot, bc.statedb, reader)
+		if err != nil {
+			return nil, err
+		}
+		statedb, err = state.NewWithReader(parentRoot, bc.statedb, reader)
+		if err != nil {
+			return nil, err
+		}
+		go func(start time.Time, throwaway *state.StateDB, block *types.Block) {
+			// Disable tracing for prefetcher executions.
+			vmCfg := bc.vmConfig
+			vmCfg.Tracer = nil
+			bc.prefetcher.Prefetch(block, throwaway, vmCfg, &interrupt)
+
+			blockPrefetchExecuteTimer.Update(time.Since(start))
+			if interrupt.Load() {
+				blockPrefetchInterruptMeter.Mark(1)
+			}
+		}(time.Now(), throwaway, block)
+	}
+
+	// If we are past Byzantium, enable prefetching to pull in trie node paths
+	// while processing transactions. Before Byzantium the prefetcher is mostly
+	// useless due to the intermediate root hashing after each transaction.
+	var witness *stateless.Witness
+	if bc.chainConfig.IsByzantium(block.Number()) {
+		// Generate witnesses either if we're self-testing, or if it's the
+		// only block being inserted. A bit crude, but witnesses are huge,
+		// so we refuse to make an entire chain of them.
+		if bc.vmConfig.StatelessSelfValidation || makeWitness {
+			witness, err = stateless.NewWitness(block.Header(), bc)
+			if err != nil {
+				return nil, err
+			}
+		}
+		statedb.StartPrefetcher("chain", witness)
+		defer statedb.StopPrefetcher()
+	}
+
 	if bc.logger != nil && bc.logger.OnBlockStart != nil {
 		bc.logger.OnBlockStart(tracing.BlockEvent{
 			Block:     block,
@@ -1944,7 +2026,7 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s
 		}
 	}
 	xvtime := time.Since(xvstart)
-	proctime := time.Since(start) // processing + validation + cross validation
+	proctime := time.Since(startTime) // processing + validation + cross validation
 
 	// Update the metrics touched during block processing and validation
 	accountReadTimer.Update(statedb.AccountReads) // Account reads are complete(in processing)
@@ -1985,9 +2067,19 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s
 	triedbCommitTimer.Update(statedb.TrieDBCommits)     // Trie database commits are complete, we can mark them
 
 	blockWriteTimer.Update(time.Since(wstart) - max(statedb.AccountCommits, statedb.StorageCommits) /* concurrent */ - statedb.SnapshotCommits - statedb.TrieDBCommits)
-	blockInsertTimer.UpdateSince(start)
+	elapsed := time.Since(startTime) + 1 // prevent zero division
+	blockInsertTimer.Update(elapsed)
 
-	return &blockProcessingResult{usedGas: res.GasUsed, procTime: proctime, status: status}, nil
+	// TODO(rjl493456442) generalize the ResettingTimer
+	mgasps := float64(res.GasUsed) * 1000 / float64(elapsed)
+	chainMgaspsMeter.Update(time.Duration(mgasps))
+
+	return &blockProcessingResult{
+		usedGas:  res.GasUsed,
+		procTime: proctime,
+		status:   status,
+		witness:  witness,
+	}, nil
 }
 
 // insertSideChain is called when an import batch hits upon a pruned ancestor
@@ -2468,6 +2560,31 @@ func (bc *BlockChain) reportBlock(block *types.Block, res *ProcessResult, err er
 	log.Error(summarizeBadBlock(block, receipts, bc.Config(), err))
 }
 
+// logForkReadiness will write a log when a future fork is scheduled, but not
+// active. This is useful so operators know their client is ready for the fork.
+func (bc *BlockChain) logForkReadiness(block *types.Block) {
+	config := bc.Config()
+	current, last := config.LatestFork(block.Time()), config.LatestFork(math.MaxUint64)
+
+	// Short circuit if the timestamp of the last fork is undefined,
+	// or if the network has already passed the last configured fork.
+	t := config.Timestamp(last)
+	if t == nil || current >= last {
+		return
+	}
+	at := time.Unix(int64(*t), 0)
+
+	// Only log if:
+	// - Current time is before the fork activation time
+	// - Enough time has passed since last alert
+	now := time.Now()
+	if now.Before(at) && now.After(bc.lastForkReadyAlert.Add(forkReadyInterval)) {
+		log.Info("Ready for fork activation", "fork", last, "date", at.Format(time.RFC822),
+			"remaining", time.Until(at).Round(time.Second), "timestamp", at.Unix())
+		bc.lastForkReadyAlert = time.Now()
+	}
+}
+
 // summarizeBadBlock returns a string summarizing the bad block and other
 // relevant information.
 func summarizeBadBlock(block *types.Block, receipts []*types.Receipt, config *params.ChainConfig, err error) string {
@@ -2504,15 +2621,84 @@ func (bc *BlockChain) InsertHeaderChain(chain []*types.Header) (int, error) {
 	if i, err := bc.hc.ValidateHeaderChain(chain); err != nil {
 		return i, err
 	}
-
 	if !bc.chainmu.TryLock() {
 		return 0, errChainStopped
 	}
 	defer bc.chainmu.Unlock()
+
 	_, err := bc.hc.InsertHeaderChain(chain, start)
 	return 0, err
 }
 
+// InsertHeadersBeforeCutoff inserts the given headers into the ancient store
+// as they are claimed older than the configured chain cutoff point. All the
+// inserted headers are regarded as canonical and chain reorg is not supported.
+func (bc *BlockChain) InsertHeadersBeforeCutoff(headers []*types.Header) (int, error) {
+	if len(headers) == 0 {
+		return 0, nil
+	}
+	// TODO(rjl493456442): Headers before the configured cutoff have already
+	// been verified by the hash of cutoff header. Theoretically, header validation
+	// could be skipped here.
+	if n, err := bc.hc.ValidateHeaderChain(headers); err != nil {
+		return n, err
+	}
+	if !bc.chainmu.TryLock() {
+		return 0, errChainStopped
+	}
+	defer bc.chainmu.Unlock()
+
+	// Initialize the ancient store with genesis block if it's empty.
+	var (
+		frozen, _ = bc.db.Ancients()
+		first     = headers[0].Number.Uint64()
+	)
+	if first == 1 && frozen == 0 {
+		_, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []rlp.RawValue{rlp.EmptyList})
+		if err != nil {
+			log.Error("Error writing genesis to ancients", "err", err)
+			return 0, err
+		}
+		log.Info("Wrote genesis to ancient store")
+	} else if frozen != first {
+		return 0, fmt.Errorf("headers are gapped with the ancient store, first: %d, ancient: %d", first, frozen)
+	}
+
+	// Write headers to the ancient store, with block bodies and receipts set to nil
+	// to ensure consistency across tables in the freezer.
+	_, err := rawdb.WriteAncientHeaderChain(bc.db, headers)
+	if err != nil {
+		return 0, err
+	}
+	// Sync the ancient store explicitly to ensure all data has been flushed to disk.
+	if err := bc.db.SyncAncient(); err != nil {
+		return 0, err
+	}
+	// Write hash to number mappings
+	batch := bc.db.NewBatch()
+	for _, header := range headers {
+		rawdb.WriteHeaderNumber(batch, header.Hash(), header.Number.Uint64())
+	}
+	// Write head header and head snap block flags
+	last := headers[len(headers)-1]
+	rawdb.WriteHeadHeaderHash(batch, last.Hash())
+	rawdb.WriteHeadFastBlockHash(batch, last.Hash())
+	if err := batch.Write(); err != nil {
+		return 0, err
+	}
+	// Truncate the useless chain segment (zero bodies and receipts) in the
+	// ancient store.
+	if _, err := bc.db.TruncateTail(last.Number.Uint64() + 1); err != nil {
+		return 0, err
+	}
+	// Last step update all in-memory markers
+	bc.hc.currentHeader.Store(last)
+	bc.currentSnapBlock.Store(last)
+	headHeaderGauge.Update(last.Number.Int64())
+	headFastBlockGauge.Update(last.Number.Int64())
+	return 0, nil
+}
+
 // SetBlockValidatorAndProcessorForTesting sets the current validator and processor.
 // This method can be used to force an invalid blockchain to be verified for tests.
 // This method is unsafe and should only be used before block import starts.
@@ -2532,9 +2718,3 @@ func (bc *BlockChain) SetTrieFlushInterval(interval time.Duration) {
 func (bc *BlockChain) GetTrieFlushInterval() time.Duration {
 	return time.Duration(bc.flushInterval.Load())
 }
-
-// HistoryPruningCutoff returns the configured history pruning point.
-// Blocks before this might not be available in the database.
-func (bc *BlockChain) HistoryPruningCutoff() uint64 {
-	return bc.cacheConfig.HistoryPruningCutoff
-}

core/blockchain_insert.go

@@ -27,7 +27,7 @@ import (
 
 // insertStats tracks and reports on block insertion.
 type insertStats struct {
-	queued, processed, ignored int
+	processed, ignored int
 	usedGas            uint64
 	lastIndex          int
 	startTime          mclock.AbsTime
@@ -43,7 +43,8 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn
 	// Fetch the timings for the batch
 	var (
 		now     = mclock.Now()
-		elapsed = now.Sub(st.startTime)
+		elapsed = now.Sub(st.startTime) + 1 // prevent zero division
+		mgasps  = float64(st.usedGas) * 1000 / float64(elapsed)
 	)
 	// If we're at the last block of the batch or report period reached, log
 	if index == len(chain)-1 || elapsed >= statsReportLimit {
@@ -58,7 +59,7 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn
 		context := []interface{}{
 			"number", end.Number(), "hash", end.Hash(),
 			"blocks", st.processed, "txs", txs, "mgas", float64(st.usedGas) / 1000000,
-			"elapsed", common.PrettyDuration(elapsed), "mgasps", float64(st.usedGas) * 1000 / float64(elapsed),
+			"elapsed", common.PrettyDuration(elapsed), "mgasps", mgasps,
 		}
 		if timestamp := time.Unix(int64(end.Time()), 0); time.Since(timestamp) > time.Minute {
 			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
@@ -74,9 +75,6 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn
 		}
 		context = append(context, []interface{}{"triedirty", triebufNodes}...)
 
-		if st.queued > 0 {
-			context = append(context, []interface{}{"queued", st.queued}...)
-		}
 		if st.ignored > 0 {
 			context = append(context, []interface{}{"ignored", st.ignored}...)
 		}
@@ -138,6 +136,7 @@ func (it *insertIterator) next() (*types.Block, error) {
 //
 // Both header and body validation errors (nil too) is cached into the iterator
 // to avoid duplicating work on the following next() call.
+// nolint:unused
 func (it *insertIterator) peek() (*types.Block, error) {
 	// If we reached the end of the chain, abort
 	if it.index+1 >= len(it.chain) {

core/blockchain_reader.go

@@ -86,6 +86,11 @@ func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header {
 	return bc.hc.GetHeaderByNumber(number)
 }
 
+// GetBlockNumber retrieves the block number associated with a block hash.
+func (bc *BlockChain) GetBlockNumber(hash common.Hash) *uint64 {
+	return bc.hc.GetBlockNumber(hash)
+}
+
 // GetHeadersFrom returns a contiguous segment of headers, in rlp-form, going
 // backwards from the given number.
 func (bc *BlockChain) GetHeadersFrom(number, count uint64) []rlp.RawValue {
@@ -229,6 +234,24 @@ func (bc *BlockChain) GetReceiptsByHash(hash common.Hash) types.Receipts {
 	return receipts
 }
 
+// GetRawReceipts retrieves the receipts for all transactions in a given block
+// without deriving the internal fields and the Bloom.
+func (bc *BlockChain) GetRawReceipts(hash common.Hash, number uint64) types.Receipts {
+	if receipts, ok := bc.receiptsCache.Get(hash); ok {
+		return receipts
+	}
+	return rawdb.ReadRawReceipts(bc.db, hash, number)
+}
+
+// GetReceiptsRLP retrieves the receipts of a block.
+func (bc *BlockChain) GetReceiptsRLP(hash common.Hash) rlp.RawValue {
+	number := rawdb.ReadHeaderNumber(bc.db, hash)
+	if number == nil {
+		return nil
+	}
+	return rawdb.ReadReceiptsRLP(bc.db, hash, *number)
+}
+
 // GetUnclesInChain retrieves all the uncles from a given block backwards until
 // a specific distance is reached.
 func (bc *BlockChain) GetUnclesInChain(block *types.Block, length int) []*types.Header {
@@ -257,42 +280,20 @@ func (bc *BlockChain) GetAncestor(hash common.Hash, number, ancestor uint64, max
 // GetTransactionLookup retrieves the lookup along with the transaction
 // itself associate with the given transaction hash.
 //
-// An error will be returned if the transaction is not found, and background
+// A null will be returned if the transaction is not found. This can be due to
-// indexing for transactions is still in progress. The transaction might be
+// the transaction indexer not being finished. The caller must explicitly check
-// reachable shortly once it's indexed.
+// the indexer progress.
-//
+func (bc *BlockChain) GetTransactionLookup(hash common.Hash) (*rawdb.LegacyTxLookupEntry, *types.Transaction) {
-// A null will be returned in the transaction is not found and background
-// transaction indexing is already finished. The transaction is not existent
-// from the node's perspective.
-func (bc *BlockChain) GetTransactionLookup(hash common.Hash) (*rawdb.LegacyTxLookupEntry, *types.Transaction, error) {
 	bc.txLookupLock.RLock()
 	defer bc.txLookupLock.RUnlock()
 
 	// Short circuit if the txlookup already in the cache, retrieve otherwise
 	if item, exist := bc.txLookupCache.Get(hash); exist {
-		return item.lookup, item.transaction, nil
+		return item.lookup, item.transaction
 	}
 	tx, blockHash, blockNumber, txIndex := rawdb.ReadTransaction(bc.db, hash)
 	if tx == nil {
-		progress, err := bc.TxIndexProgress()
+		return nil, nil
-		if err != nil {
-			// No error is returned if the transaction indexing progress is unreachable
-			// due to unexpected internal errors. In such cases, it is impossible to
-			// determine whether the transaction does not exist or has simply not been
-			// indexed yet without a progress marker.
-			//
-			// In such scenarios, the transaction is treated as unreachable, though
-			// this is clearly an unintended and unexpected situation.
-			return nil, nil, nil
-		}
-		// The transaction indexing is not finished yet, returning an
-		// error to explicitly indicate it.
-		if !progress.Done() {
-			return nil, nil, errors.New("transaction indexing still in progress")
-		}
-		// The transaction is already indexed, the transaction is either
-		// not existent or not in the range of index, returning null.
-		return nil, nil, nil
 	}
 	lookup := &rawdb.LegacyTxLookupEntry{
 		BlockHash:  blockHash,
@@ -303,7 +304,23 @@ func (bc *BlockChain) GetTransactionLookup(hash common.Hash) (*rawdb.LegacyTxLoo
 		lookup:      lookup,
 		transaction: tx,
 	})
-	return lookup, tx, nil
+	return lookup, tx
+}
+
+// TxIndexDone returns true if the transaction indexer has finished indexing.
+func (bc *BlockChain) TxIndexDone() bool {
+	progress, err := bc.TxIndexProgress()
+	if err != nil {
+		// No error is returned if the transaction indexing progress is unreachable
+		// due to unexpected internal errors. In such cases, it is impossible to
+		// determine whether the transaction does not exist or has simply not been
+		// indexed yet without a progress marker.
+		//
+		// In such scenarios, the transaction is treated as unreachable, though
+		// this is clearly an unintended and unexpected situation.
+		return true
+	}
+	return progress.Done()
 }
 
 // HasState checks if state trie is fully present in the database or not.
@@ -399,7 +416,17 @@ func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) {
 	if bc.txIndexer == nil {
 		return TxIndexProgress{}, errors.New("tx indexer is not enabled")
 	}
-	return bc.txIndexer.txIndexProgress()
+	return bc.txIndexer.txIndexProgress(), nil
+}
+
+// HistoryPruningCutoff returns the configured history pruning point.
+// Blocks before this might not be available in the database.
+func (bc *BlockChain) HistoryPruningCutoff() (uint64, common.Hash) {
+	pt := bc.historyPrunePoint.Load()
+	if pt == nil {
+		return 0, bc.genesisBlock.Hash()
+	}
+	return pt.BlockNumber, pt.BlockHash
 }
 
 // TrieDB retrieves the low level trie database used for data storage.

core/blockchain_repair_test.go

@@ -1769,7 +1769,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
 	if err != nil {
 		t.Fatalf("Failed to create persistent key-value database: %v", err)
 	}
-	db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
+	db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
 	if err != nil {
 		t.Fatalf("Failed to create persistent freezer database: %v", err)
 	}
@@ -1791,7 +1791,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
 		}
 	)
 	defer engine.Close()
-	if snapshots {
+	if snapshots && scheme == rawdb.HashScheme {
 		config.SnapshotLimit = 256
 		config.SnapshotWait = true
 	}
@@ -1820,7 +1820,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
 		if err := chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false); err != nil {
 			t.Fatalf("Failed to flush trie state: %v", err)
 		}
-		if snapshots {
+		if snapshots && scheme == rawdb.HashScheme {
 			if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
 				t.Fatalf("Failed to flatten snapshots: %v", err)
 			}
@@ -1854,7 +1854,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
 	if err != nil {
 		t.Fatalf("Failed to reopen persistent key-value database: %v", err)
 	}
-	db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
+	db, err = rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
 	if err != nil {
 		t.Fatalf("Failed to reopen persistent freezer database: %v", err)
 	}
@@ -1919,7 +1919,7 @@ func testIssue23496(t *testing.T, scheme string) {
 	if err != nil {
 		t.Fatalf("Failed to create persistent key-value database: %v", err)
 	}
-	db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
+	db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
 	if err != nil {
 		t.Fatalf("Failed to create persistent freezer database: %v", err)
 	}
@@ -1952,9 +1952,11 @@ func testIssue23496(t *testing.T, scheme string) {
 	if _, err := chain.InsertChain(blocks[1:2]); err != nil {
 		t.Fatalf("Failed to import canonical chain start: %v", err)
 	}
+	if scheme == rawdb.HashScheme {
 		if err := chain.snaps.Cap(blocks[1].Root(), 0); err != nil {
 			t.Fatalf("Failed to flatten snapshots: %v", err)
 		}
+	}
 
 	// Insert block B3 and commit the state into disk
 	if _, err := chain.InsertChain(blocks[2:3]); err != nil {
@@ -1977,7 +1979,7 @@ func testIssue23496(t *testing.T, scheme string) {
 	if err != nil {
 		t.Fatalf("Failed to reopen persistent key-value database: %v", err)
 	}
-	db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
+	db, err = rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
 	if err != nil {
 		t.Fatalf("Failed to reopen persistent freezer database: %v", err)
 	}
@@ -1997,16 +1999,24 @@ func testIssue23496(t *testing.T, scheme string) {
 	}
 	expHead := uint64(1)
 	if scheme == rawdb.PathScheme {
-		expHead = uint64(2)
+		// The pathdb database makes sure that snapshot and trie are consistent,
+		// so only the last block is reverted in case of a crash.
+		expHead = uint64(3)
 	}
 	if head := chain.CurrentBlock(); head.Number.Uint64() != expHead {
 		t.Errorf("Head block mismatch: have %d, want %d", head.Number, expHead)
 	}
-
+	if scheme == rawdb.PathScheme {
+		// Reinsert B4
+		if _, err := chain.InsertChain(blocks[3:]); err != nil {
+			t.Fatalf("Failed to import canonical chain tail: %v", err)
+		}
+	} else {
 		// Reinsert B2-B4
 		if _, err := chain.InsertChain(blocks[1:]); err != nil {
 			t.Fatalf("Failed to import canonical chain tail: %v", err)
 		}
+	}
 	if head := chain.CurrentHeader(); head.Number.Uint64() != uint64(4) {
 		t.Errorf("Head header mismatch: have %d, want %d", head.Number, 4)
 	}
@@ -2016,7 +2026,9 @@ func testIssue23496(t *testing.T, scheme string) {
 	if head := chain.CurrentBlock(); head.Number.Uint64() != uint64(4) {
 		t.Errorf("Head block mismatch: have %d, want %d", head.Number, uint64(4))
 	}
+	if scheme == rawdb.HashScheme {
 		if layer := chain.Snapshots().Snapshot(blocks[2].Root()); layer == nil {
 			t.Error("Failed to regenerate the snapshot of known state")
 		}
 	}
+}

core/blockchain_sethead_test.go

@@ -1973,7 +1973,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
 	if err != nil {
 		t.Fatalf("Failed to create persistent key-value database: %v", err)
 	}
-	db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
+	db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
 	if err != nil {
 		t.Fatalf("Failed to create persistent freezer database: %v", err)
 	}
@@ -2023,7 +2023,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
 	}
 	if tt.commitBlock > 0 {
 		chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false)
-		if snapshots {
+		if snapshots && scheme == rawdb.HashScheme {
 			if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
 				t.Fatalf("Failed to flatten snapshots: %v", err)
 			}

core/blockchain_snapshot_test.go

@@ -70,7 +70,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 	if err != nil {
 		t.Fatalf("Failed to create persistent key-value database: %v", err)
 	}
-	db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
+	db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
 	if err != nil {
 		t.Fatalf("Failed to create persistent freezer database: %v", err)
 	}
@@ -105,7 +105,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 		if basic.commitBlock > 0 && basic.commitBlock == point {
 			chain.TrieDB().Commit(blocks[point-1].Root(), false)
 		}
-		if basic.snapshotBlock > 0 && basic.snapshotBlock == point {
+		if basic.snapshotBlock > 0 && basic.snapshotBlock == point && basic.scheme == rawdb.HashScheme {
 			// Flushing the entire snap tree into the disk, the
 			// relevant (a) snapshot root and (b) snapshot generator
 			// will be persisted atomically.
@@ -149,15 +149,19 @@ func (basic *snapshotTestBasic) verify(t *testing.T, chain *BlockChain, blocks [
 	block := chain.GetBlockByNumber(basic.expSnapshotBottom)
 	if block == nil {
 		t.Errorf("The corresponding block[%d] of snapshot disk layer is missing", basic.expSnapshotBottom)
-	} else if !bytes.Equal(chain.snaps.DiskRoot().Bytes(), block.Root().Bytes()) {
+	} else if basic.scheme == rawdb.HashScheme {
+		if !bytes.Equal(chain.snaps.DiskRoot().Bytes(), block.Root().Bytes()) {
 			t.Errorf("The snapshot disk layer root is incorrect, want %x, get %x", block.Root(), chain.snaps.DiskRoot())
 		}
+	}
 
 	// Check the snapshot, ensure it's integrated
+	if basic.scheme == rawdb.HashScheme {
 		if err := chain.snaps.Verify(block.Root()); err != nil {
 			t.Errorf("The disk layer is not integrated %v", err)
 		}
 	}
+}
 
 //nolint:unused
 func (basic *snapshotTestBasic) dump() string {
@@ -261,7 +265,7 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create persistent key-value database: %v", err)
 	}
-	newdb, err := rawdb.NewDatabaseWithFreezer(pdb, snaptest.ancient, "", false)
+	newdb, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: snaptest.ancient})
 	if err != nil {
 		t.Fatalf("Failed to create persistent freezer database: %v", err)
 	}
@@ -565,12 +569,14 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
 	//
 	// Expected head header    : C8
 	// Expected head fast block: C8
-	// Expected head block     : G
+	// Expected head block     : G (Hash mode), C6 (Hash mode)
-	// Expected snapshot disk  : C4
+	// Expected snapshot disk  : C4 (Hash mode)
 	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
 		expHead := uint64(0)
 		if scheme == rawdb.PathScheme {
-			expHead = uint64(4)
+			// The pathdb database makes sure that snapshot and trie are consistent,
+			// so only the last two blocks are reverted in case of a crash.
+			expHead = uint64(6)
 		}
 		test := &crashSnapshotTest{
 			snapshotTestBasic{

core/blockchain_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/beacon"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core/history"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -733,18 +734,11 @@ func testFastVsFullChains(t *testing.T, scheme string) {
 	fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
 	defer fast.Stop()
 
-	headers := make([]*types.Header, len(blocks))
+	if n, err := fast.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0); err != nil {
-	for i, block := range blocks {
-		headers[i] = block.Header()
-	}
-	if n, err := fast.InsertHeaderChain(headers); err != nil {
-		t.Fatalf("failed to insert header %d: %v", n, err)
-	}
-	if n, err := fast.InsertReceiptChain(blocks, receipts, 0); err != nil {
 		t.Fatalf("failed to insert receipt %d: %v", n, err)
 	}
 	// Freezer style fast import the chain.
-	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
+	ancientDb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -753,10 +747,7 @@ func testFastVsFullChains(t *testing.T, scheme string) {
 	ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
 	defer ancient.Stop()
 
-	if n, err := ancient.InsertHeaderChain(headers); err != nil {
+	if n, err := ancient.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), uint64(len(blocks)/2)); err != nil {
-		t.Fatalf("failed to insert header %d: %v", n, err)
-	}
-	if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(len(blocks)/2)); err != nil {
 		t.Fatalf("failed to insert receipt %d: %v", n, err)
 	}
 
@@ -833,7 +824,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
 
 	// makeDb creates a db instance for testing.
 	makeDb := func() ethdb.Database {
-		db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
+		db, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
 		if err != nil {
 			t.Fatalf("failed to create temp freezer db: %v", err)
 		}
@@ -880,14 +871,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
 	fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
 	defer fast.Stop()
 
-	headers := make([]*types.Header, len(blocks))
+	if n, err := fast.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0); err != nil {
-	for i, block := range blocks {
-		headers[i] = block.Header()
-	}
-	if n, err := fast.InsertHeaderChain(headers); err != nil {
-		t.Fatalf("failed to insert header %d: %v", n, err)
-	}
-	if n, err := fast.InsertReceiptChain(blocks, receipts, 0); err != nil {
 		t.Fatalf("failed to insert receipt %d: %v", n, err)
 	}
 	assert(t, "fast", fast, height, height, 0)
@@ -900,10 +884,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
 	ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
 	defer ancient.Stop()
 
-	if n, err := ancient.InsertHeaderChain(headers); err != nil {
+	if n, err := ancient.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), uint64(3*len(blocks)/4)); err != nil {
-		t.Fatalf("failed to insert header %d: %v", n, err)
-	}
-	if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err != nil {
 		t.Fatalf("failed to insert receipt %d: %v", n, err)
 	}
 	assert(t, "ancient", ancient, height, height, 0)
@@ -916,6 +897,11 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
 	// Import the chain as a light node and ensure all pointers are updated
 	lightDb := makeDb()
 	defer lightDb.Close()
+
+	headers := make([]*types.Header, len(blocks))
+	for i, block := range blocks {
+		headers[i] = block.Header()
+	}
 	light, _ := NewBlockChain(lightDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
 	if n, err := light.InsertHeaderChain(headers); err != nil {
 		t.Fatalf("failed to insert header %d: %v", n, err)
@@ -1637,7 +1623,7 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) {
 	competitor, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*state.TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) })
 
 	// Import the shared chain and the original canonical one
-	db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
+	db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
 	defer db.Close()
 
 	chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil)
@@ -1703,21 +1689,14 @@ func testBlockchainRecovery(t *testing.T, scheme string) {
 	_, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(height), nil)
 
 	// Import the chain as a ancient-first node and ensure all pointers are updated
-	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false)
+	ancientDb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{Ancient: t.TempDir()})
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
 	defer ancientDb.Close()
 	ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
 
-	headers := make([]*types.Header, len(blocks))
+	if n, err := ancient.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), uint64(3*len(blocks)/4)); err != nil {
-	for i, block := range blocks {
-		headers[i] = block.Header()
-	}
-	if n, err := ancient.InsertHeaderChain(headers); err != nil {
-		t.Fatalf("failed to insert header %d: %v", n, err)
-	}
-	if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err != nil {
 		t.Fatalf("failed to insert receipt %d: %v", n, err)
 	}
 	rawdb.WriteLastPivotNumber(ancientDb, blocks[len(blocks)-1].NumberU64()) // Force fast sync behavior
@@ -1741,82 +1720,6 @@ func testBlockchainRecovery(t *testing.T, scheme string) {
 	}
 }
 
-// This test checks that InsertReceiptChain will roll back correctly when attempting to insert a side chain.
-func TestInsertReceiptChainRollback(t *testing.T) {
-	testInsertReceiptChainRollback(t, rawdb.HashScheme)
-	testInsertReceiptChainRollback(t, rawdb.PathScheme)
-}
-
-func testInsertReceiptChainRollback(t *testing.T, scheme string) {
-	// Generate forked chain. The returned BlockChain object is used to process the side chain blocks.
-	tmpChain, sideblocks, canonblocks, gspec, err := getLongAndShortChains(scheme)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer tmpChain.Stop()
-	// Get the side chain receipts.
-	if _, err := tmpChain.InsertChain(sideblocks); err != nil {
-		t.Fatal("processing side chain failed:", err)
-	}
-	t.Log("sidechain head:", tmpChain.CurrentBlock().Number, tmpChain.CurrentBlock().Hash())
-	sidechainReceipts := make([]types.Receipts, len(sideblocks))
-	for i, block := range sideblocks {
-		sidechainReceipts[i] = tmpChain.GetReceiptsByHash(block.Hash())
-	}
-	// Get the canon chain receipts.
-	if _, err := tmpChain.InsertChain(canonblocks); err != nil {
-		t.Fatal("processing canon chain failed:", err)
-	}
-	t.Log("canon head:", tmpChain.CurrentBlock().Number, tmpChain.CurrentBlock().Hash())
-	canonReceipts := make([]types.Receipts, len(canonblocks))
-	for i, block := range canonblocks {
-		canonReceipts[i] = tmpChain.GetReceiptsByHash(block.Hash())
-	}
-
-	// Set up a BlockChain that uses the ancient store.
-	ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
-	if err != nil {
-		t.Fatalf("failed to create temp freezer db: %v", err)
-	}
-	defer ancientDb.Close()
-
-	ancientChain, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
-	defer ancientChain.Stop()
-
-	// Import the canonical header chain.
-	canonHeaders := make([]*types.Header, len(canonblocks))
-	for i, block := range canonblocks {
-		canonHeaders[i] = block.Header()
-	}
-	if _, err = ancientChain.InsertHeaderChain(canonHeaders); err != nil {
-		t.Fatal("can't import canon headers:", err)
-	}
-
-	// Try to insert blocks/receipts of the side chain.
-	_, err = ancientChain.InsertReceiptChain(sideblocks, sidechainReceipts, uint64(len(sideblocks)))
-	if err == nil {
-		t.Fatal("expected error from InsertReceiptChain.")
-	}
-	if ancientChain.CurrentSnapBlock().Number.Uint64() != 0 {
-		t.Fatalf("failed to rollback ancient data, want %d, have %d", 0, ancientChain.CurrentSnapBlock().Number)
-	}
-	if frozen, err := ancientChain.db.Ancients(); err != nil || frozen != 1 {
-		t.Fatalf("failed to truncate ancient data, frozen index is %d", frozen)
-	}
-
-	// Insert blocks/receipts of the canonical chain.
-	_, err = ancientChain.InsertReceiptChain(canonblocks, canonReceipts, uint64(len(canonblocks)))
-	if err != nil {
-		t.Fatalf("can't import canon chain receipts: %v", err)
-	}
-	if ancientChain.CurrentSnapBlock().Number.Uint64() != canonblocks[len(canonblocks)-1].NumberU64() {
-		t.Fatalf("failed to insert ancient recept chain after rollback")
-	}
-	if frozen, _ := ancientChain.db.Ancients(); frozen != uint64(len(canonblocks))+1 {
-		t.Fatalf("wrong ancients count %d", frozen)
-	}
-}
-
 // Tests that importing a very large side fork, which is larger than the canon chain,
 // but where the difficulty per block is kept low: this means that it will not
 // overtake the 'canon' chain until after it's passed canon by about 200 blocks.
@@ -1844,7 +1747,7 @@ func testLowDiffLongChain(t *testing.T, scheme string) {
 	})
 
 	// Import the canonical chain
-	diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
+	diskdb, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
 	defer diskdb.Close()
 
 	chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil)
@@ -2056,7 +1959,7 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) {
 		b.OffsetTime(-9) // A higher difficulty
 	})
 	// Import the shared chain and the original canonical one
-	chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
+	chaindb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -2088,15 +1991,7 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) {
 		}
 	} else if typ == "receipts" {
 		inserter = func(blocks []*types.Block, receipts []types.Receipts) error {
-			headers := make([]*types.Header, 0, len(blocks))
+			_, err = chain.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0)
-			for _, block := range blocks {
-				headers = append(headers, block.Header())
-			}
-			_, err := chain.InsertHeaderChain(headers)
-			if err != nil {
-				return err
-			}
-			_, err = chain.InsertReceiptChain(blocks, receipts, 0)
 			return err
 		}
 		asserter = func(t *testing.T, block *types.Block) {
@@ -2227,7 +2122,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i
 		}
 	})
 	// Import the shared chain and the original canonical one
-	chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
+	chaindb, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
 	if err != nil {
 		t.Fatalf("failed to create temp freezer db: %v", err)
 	}
@@ -2262,15 +2157,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i
 		}
 	} else if typ == "receipts" {
 		inserter = func(blocks []*types.Block, receipts []types.Receipts) error {
-			headers := make([]*types.Header, 0, len(blocks))
+			_, err = chain.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), 0)
-			for _, block := range blocks {
-				headers = append(headers, block.Header())
-			}
-			i, err := chain.InsertHeaderChain(headers)
-			if err != nil {
-				return fmt.Errorf("index %d: %w", i, err)
-			}
-			_, err = chain.InsertReceiptChain(blocks, receipts, 0)
 			return err
 		}
 		asserter = func(t *testing.T, block *types.Block) {
@@ -2609,7 +2496,7 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) {
 	if err != nil {
 		t.Fatalf("Failed to create persistent key-value database: %v", err)
 	}
-	db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
+	db, err := rawdb.Open(pdb, rawdb.OpenOptions{Ancient: ancient})
 	if err != nil {
 		t.Fatalf("Failed to create persistent freezer database: %v", err)
 	}
@@ -3516,7 +3403,7 @@ func testSetCanonical(t *testing.T, scheme string) {
 		}
 		gen.AddTx(tx)
 	})
-	diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
+	diskdb, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
 	defer diskdb.Close()
 
 	chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil)
@@ -4265,3 +4152,241 @@ func TestEIP7702(t *testing.T) {
 		t.Fatalf("addr2 storage wrong: expected %d, got %d", fortyTwo, actual)
 	}
 }
+
+// Tests the scenario that the synchronization target in snap sync has been changed
+// with a chain reorg at the tip. In this case the reorg'd segment should be unmarked
+// with canonical flags.
+func TestChainReorgSnapSync(t *testing.T) {
+	testChainReorgSnapSync(t, 0)
+	testChainReorgSnapSync(t, 32)
+	testChainReorgSnapSync(t, gomath.MaxUint64)
+}
+
+func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) {
+	// log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
+
+	// Configure and generate a sample block chain
+	var (
+		key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		address = crypto.PubkeyToAddress(key.PublicKey)
+		funds   = big.NewInt(1000000000000000)
+		gspec   = &Genesis{
+			Config:  params.TestChainConfig,
+			Alloc:   types.GenesisAlloc{address: {Balance: funds}},
+			BaseFee: big.NewInt(params.InitialBaseFee),
+		}
+		signer = types.LatestSigner(gspec.Config)
+		engine = beacon.New(ethash.NewFaker())
+	)
+	genDb, blocks, receipts := GenerateChainWithGenesis(gspec, engine, 32, func(i int, block *BlockGen) {
+		block.SetCoinbase(common.Address{0x00})
+
+		// If the block number is multiple of 3, send a few bonus transactions to the miner
+		if i%3 == 2 {
+			for j := 0; j < i%4+1; j++ {
+				tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key)
+				if err != nil {
+					panic(err)
+				}
+				block.AddTx(tx)
+			}
+		}
+	})
+	chainA, receiptsA := GenerateChain(gspec.Config, blocks[len(blocks)-1], engine, genDb, 16, func(i int, gen *BlockGen) {
+		gen.SetCoinbase(common.Address{0: byte(0xa), 19: byte(i)})
+	})
+	chainB, receiptsB := GenerateChain(gspec.Config, blocks[len(blocks)-1], engine, genDb, 20, func(i int, gen *BlockGen) {
+		gen.SetCoinbase(common.Address{0: byte(0xb), 19: byte(i)})
+	})
+
+	db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
+	defer db.Close()
+
+	chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
+	defer chain.Stop()
+
+	if n, err := chain.InsertReceiptChain(blocks, types.EncodeBlockReceiptLists(receipts), ancientLimit); err != nil {
+		t.Fatalf("failed to insert receipt %d: %v", n, err)
+	}
+	if n, err := chain.InsertReceiptChain(chainA, types.EncodeBlockReceiptLists(receiptsA), ancientLimit); err != nil {
+		t.Fatalf("failed to insert receipt %d: %v", n, err)
+	}
+	// If the common ancestor is below the ancient limit, rewind the chain head.
+	// It's aligned with the behavior in the snap sync
+	ancestor := blocks[len(blocks)-1].NumberU64()
+	if ancestor < ancientLimit {
+		rawdb.WriteLastPivotNumber(db, ancestor)
+		chain.SetHead(ancestor)
+	}
+	if n, err := chain.InsertReceiptChain(chainB, types.EncodeBlockReceiptLists(receiptsB), ancientLimit); err != nil {
+		t.Fatalf("failed to insert receipt %d: %v", n, err)
+	}
+	head := chain.CurrentSnapBlock()
+	if head.Hash() != chainB[len(chainB)-1].Hash() {
+		t.Errorf("head snap block #%d: header mismatch: want: %v, got: %v", head.Number, chainB[len(chainB)-1].Hash(), head.Hash())
+	}
+
+	// Iterate over all chain data components, and cross reference
+	for i := 0; i < len(blocks); i++ {
+		num, hash := blocks[i].NumberU64(), blocks[i].Hash()
+		header := chain.GetHeaderByNumber(num)
+		if header.Hash() != hash {
+			t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
+		}
+	}
+	for i := 0; i < len(chainA); i++ {
+		num, hash := chainA[i].NumberU64(), chainA[i].Hash()
+		header := chain.GetHeaderByNumber(num)
+		if header == nil {
+			continue
+		}
+		if header.Hash() == hash {
+			t.Errorf("block #%d: unexpected canonical header: %v", num, hash)
+		}
+	}
+	for i := 0; i < len(chainB); i++ {
+		num, hash := chainB[i].NumberU64(), chainB[i].Hash()
+		header := chain.GetHeaderByNumber(num)
+		if header.Hash() != hash {
+			t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
+		}
+	}
+}
+
+// Tests the scenario that all the inserted chain segment are with the configured
+// chain cutoff point. In this case the chain segment before the cutoff should
+// be persisted without the receipts and bodies; chain after should be persisted
+// normally.
+func TestInsertChainWithCutoff(t *testing.T) {
+	const chainLength = 64
+
+	// Configure and generate a sample block chain
+	var (
+		key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		address = crypto.PubkeyToAddress(key.PublicKey)
+		funds   = big.NewInt(1000000000000000)
+		gspec   = &Genesis{
+			Config:  params.TestChainConfig,
+			Alloc:   types.GenesisAlloc{address: {Balance: funds}},
+			BaseFee: big.NewInt(params.InitialBaseFee),
+		}
+		signer = types.LatestSigner(gspec.Config)
+		engine = beacon.New(ethash.NewFaker())
+	)
+	_, blocks, receipts := GenerateChainWithGenesis(gspec, engine, chainLength, func(i int, block *BlockGen) {
+		block.SetCoinbase(common.Address{0x00})
+		tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key)
+		if err != nil {
+			panic(err)
+		}
+		block.AddTx(tx)
+	})
+
+	// Run the actual tests.
+	t.Run("cutoff-32/ancientLimit-32", func(t *testing.T) {
+		// cutoff = 32, ancientLimit = 32
+		testInsertChainWithCutoff(t, 32, 32, gspec, blocks, receipts)
+	})
+	t.Run("cutoff-32/ancientLimit-64", func(t *testing.T) {
+		// cutoff = 32, ancientLimit = 64 (entire chain in ancient)
+		testInsertChainWithCutoff(t, 32, 64, gspec, blocks, receipts)
+	})
+	t.Run("cutoff-32/ancientLimit-64", func(t *testing.T) {
+		// cutoff = 32, ancientLimit = 65 (64 blocks in ancient, 1 block in live)
+		testInsertChainWithCutoff(t, 32, 65, gspec, blocks, receipts)
+	})
+}
+
+func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64, genesis *Genesis, blocks []*types.Block, receipts []types.Receipts) {
+	// log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
+
+	// Add a known pruning point for the duration of the test.
+	ghash := genesis.ToBlock().Hash()
+	cutoffBlock := blocks[cutoff-1]
+	history.PrunePoints[ghash] = &history.PrunePoint{
+		BlockNumber: cutoffBlock.NumberU64(),
+		BlockHash:   cutoffBlock.Hash(),
+	}
+	defer func() {
+		delete(history.PrunePoints, ghash)
+	}()
+
+	// Enable pruning in cache config.
+	config := DefaultCacheConfigWithScheme(rawdb.PathScheme)
+	config.ChainHistoryMode = history.KeepPostMerge
+
+	db, _ := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
+	defer db.Close()
+	chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), genesis, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
+	defer chain.Stop()
+
+	var (
+		headersBefore []*types.Header
+		blocksAfter   []*types.Block
+		receiptsAfter []types.Receipts
+	)
+	for i, b := range blocks {
+		if b.NumberU64() < cutoffBlock.NumberU64() {
+			headersBefore = append(headersBefore, b.Header())
+		} else {
+			blocksAfter = append(blocksAfter, b)
+			receiptsAfter = append(receiptsAfter, receipts[i])
+		}
+	}
+	if n, err := chain.InsertHeadersBeforeCutoff(headersBefore); err != nil {
+		t.Fatalf("failed to insert headers before cutoff %d: %v", n, err)
+	}
+	if n, err := chain.InsertReceiptChain(blocksAfter, types.EncodeBlockReceiptLists(receiptsAfter), ancientLimit); err != nil {
+		t.Fatalf("failed to insert receipt %d: %v", n, err)
+	}
+	headSnap := chain.CurrentSnapBlock()
+	if headSnap.Hash() != blocks[len(blocks)-1].Hash() {
+		t.Errorf("head snap block #%d: header mismatch: want: %v, got: %v", headSnap.Number, blocks[len(blocks)-1].Hash(), headSnap.Hash())
+	}
+	headHeader := chain.CurrentHeader()
+	if headHeader.Hash() != blocks[len(blocks)-1].Hash() {
+		t.Errorf("head header #%d: header mismatch: want: %v, got: %v", headHeader.Number, blocks[len(blocks)-1].Hash(), headHeader.Hash())
+	}
+	headBlock := chain.CurrentBlock()
+	if headBlock.Hash() != ghash {
+		t.Errorf("head block #%d: header mismatch: want: %v, got: %v", headBlock.Number, ghash, headBlock.Hash())
+	}
+
+	// Iterate over all chain data components, and cross reference
+	for i := 0; i < len(blocks); i++ {
+		num, hash := blocks[i].NumberU64(), blocks[i].Hash()
+
+		// Canonical headers should be visible regardless of cutoff
+		header := chain.GetHeaderByNumber(num)
+		if header.Hash() != hash {
+			t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
+		}
+		tail, err := db.Tail()
+		if err != nil {
+			t.Fatalf("Failed to get chain tail, %v", err)
+		}
+		if tail != cutoffBlock.NumberU64() {
+			t.Fatalf("Unexpected chain tail, want: %d, got: %d", cutoffBlock.NumberU64(), tail)
+		}
+		// Block bodies and receipts before the cutoff should be non-existent
+		if num < cutoffBlock.NumberU64() {
+			body := chain.GetBody(hash)
+			if body != nil {
+				t.Fatalf("Unexpected block body: %d, cutoff: %d", num, cutoffBlock.NumberU64())
+			}
+			receipts := chain.GetReceiptsByHash(hash)
+			if receipts != nil {
+				t.Fatalf("Unexpected block receipts: %d, cutoff: %d", num, cutoffBlock.NumberU64())
+			}
+		} else {
+			body := chain.GetBody(hash)
+			if body == nil || len(body.Transactions) != 1 {
+				t.Fatalf("Missed block body: %d, cutoff: %d", num, cutoffBlock.NumberU64())
+			}
+			receipts := chain.GetReceiptsByHash(hash)
+			if receipts == nil || len(receipts) != 1 {
+				t.Fatalf("Missed block receipts: %d, cutoff: %d", num, cutoffBlock.NumberU64())
+			}
+		}
+	}
+}

core/bloom_indexer.go

@@ -1,92 +0,0 @@
-// Copyright 2021 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package core
-
-import (
-	"context"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/bitutil"
-	"github.com/ethereum/go-ethereum/core/bloombits"
-	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethdb"
-)
-
-const (
-	// bloomThrottling is the time to wait between processing two consecutive index
-	// sections. It's useful during chain upgrades to prevent disk overload.
-	bloomThrottling = 100 * time.Millisecond
-)
-
-// BloomIndexer implements a core.ChainIndexer, building up a rotated bloom bits index
-// for the Ethereum header bloom filters, permitting blazing fast filtering.
-type BloomIndexer struct {
-	size    uint64               // section size to generate bloombits for
-	db      ethdb.Database       // database instance to write index data and metadata into
-	gen     *bloombits.Generator // generator to rotate the bloom bits crating the bloom index
-	section uint64               // Section is the section number being processed currently
-	head    common.Hash          // Head is the hash of the last header processed
-}
-
-// NewBloomIndexer returns a chain indexer that generates bloom bits data for the
-// canonical chain for fast logs filtering.
-func NewBloomIndexer(db ethdb.Database, size, confirms uint64) *ChainIndexer {
-	backend := &BloomIndexer{
-		db:   db,
-		size: size,
-	}
-	table := rawdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix))
-
-	return NewChainIndexer(db, table, backend, size, confirms, bloomThrottling, "bloombits")
-}
-
-// Reset implements core.ChainIndexerBackend, starting a new bloombits index
-// section.
-func (b *BloomIndexer) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error {
-	gen, err := bloombits.NewGenerator(uint(b.size))
-	b.gen, b.section, b.head = gen, section, common.Hash{}
-	return err
-}
-
-// Process implements core.ChainIndexerBackend, adding a new header's bloom into
-// the index.
-func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error {
-	b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom)
-	b.head = header.Hash()
-	return nil
-}
-
-// Commit implements core.ChainIndexerBackend, finalizing the bloom section and
-// writing it out into the database.
-func (b *BloomIndexer) Commit() error {
-	batch := b.db.NewBatchWithSize((int(b.size) / 8) * types.BloomBitLength)
-	for i := 0; i < types.BloomBitLength; i++ {
-		bits, err := b.gen.Bitset(uint(i))
-		if err != nil {
-			return err
-		}
-		rawdb.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits))
-	}
-	return batch.Write()
-}
-
-// Prune returns an empty error since we don't support pruning here.
-func (b *BloomIndexer) Prune(threshold uint64) error {
-	return nil
-}

core/bloombits/doc.go

@@ -1,18 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package bloombits implements bloom filtering on batches of data.
-package bloombits

core/bloombits/generator.go

@@ -1,98 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"errors"
-
-	"github.com/ethereum/go-ethereum/core/types"
-)
-
-var (
-	// errSectionOutOfBounds is returned if the user tried to add more bloom filters
-	// to the batch than available space, or if tries to retrieve above the capacity.
-	errSectionOutOfBounds = errors.New("section out of bounds")
-
-	// errBloomBitOutOfBounds is returned if the user tried to retrieve specified
-	// bit bloom above the capacity.
-	errBloomBitOutOfBounds = errors.New("bloom bit out of bounds")
-)
-
-// Generator takes a number of bloom filters and generates the rotated bloom bits
-// to be used for batched filtering.
-type Generator struct {
-	blooms   [types.BloomBitLength][]byte // Rotated blooms for per-bit matching
-	sections uint                         // Number of sections to batch together
-	nextSec  uint                         // Next section to set when adding a bloom
-}
-
-// NewGenerator creates a rotated bloom generator that can iteratively fill a
-// batched bloom filter's bits.
-func NewGenerator(sections uint) (*Generator, error) {
-	if sections%8 != 0 {
-		return nil, errors.New("section count not multiple of 8")
-	}
-	b := &Generator{sections: sections}
-	for i := 0; i < types.BloomBitLength; i++ {
-		b.blooms[i] = make([]byte, sections/8)
-	}
-	return b, nil
-}
-
-// AddBloom takes a single bloom filter and sets the corresponding bit column
-// in memory accordingly.
-func (b *Generator) AddBloom(index uint, bloom types.Bloom) error {
-	// Make sure we're not adding more bloom filters than our capacity
-	if b.nextSec >= b.sections {
-		return errSectionOutOfBounds
-	}
-	if b.nextSec != index {
-		return errors.New("bloom filter with unexpected index")
-	}
-	// Rotate the bloom and insert into our collection
-	byteIndex := b.nextSec / 8
-	bitIndex := byte(7 - b.nextSec%8)
-	for byt := 0; byt < types.BloomByteLength; byt++ {
-		bloomByte := bloom[types.BloomByteLength-1-byt]
-		if bloomByte == 0 {
-			continue
-		}
-		base := 8 * byt
-		b.blooms[base+7][byteIndex] |= ((bloomByte >> 7) & 1) << bitIndex
-		b.blooms[base+6][byteIndex] |= ((bloomByte >> 6) & 1) << bitIndex
-		b.blooms[base+5][byteIndex] |= ((bloomByte >> 5) & 1) << bitIndex
-		b.blooms[base+4][byteIndex] |= ((bloomByte >> 4) & 1) << bitIndex
-		b.blooms[base+3][byteIndex] |= ((bloomByte >> 3) & 1) << bitIndex
-		b.blooms[base+2][byteIndex] |= ((bloomByte >> 2) & 1) << bitIndex
-		b.blooms[base+1][byteIndex] |= ((bloomByte >> 1) & 1) << bitIndex
-		b.blooms[base][byteIndex] |= (bloomByte & 1) << bitIndex
-	}
-	b.nextSec++
-	return nil
-}
-
-// Bitset returns the bit vector belonging to the given bit index after all
-// blooms have been added.
-func (b *Generator) Bitset(idx uint) ([]byte, error) {
-	if b.nextSec != b.sections {
-		return nil, errors.New("bloom not fully generated yet")
-	}
-	if idx >= types.BloomBitLength {
-		return nil, errBloomBitOutOfBounds
-	}
-	return b.blooms[idx], nil
-}

core/bloombits/generator_test.go

@@ -1,100 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"bytes"
-	crand "crypto/rand"
-	"math/rand"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/core/types"
-)
-
-// Tests that batched bloom bits are correctly rotated from the input bloom
-// filters.
-func TestGenerator(t *testing.T) {
-	// Generate the input and the rotated output
-	var input, output [types.BloomBitLength][types.BloomByteLength]byte
-
-	for i := 0; i < types.BloomBitLength; i++ {
-		for j := 0; j < types.BloomBitLength; j++ {
-			bit := byte(rand.Int() % 2)
-
-			input[i][j/8] |= bit << byte(7-j%8)
-			output[types.BloomBitLength-1-j][i/8] |= bit << byte(7-i%8)
-		}
-	}
-	// Crunch the input through the generator and verify the result
-	gen, err := NewGenerator(types.BloomBitLength)
-	if err != nil {
-		t.Fatalf("failed to create bloombit generator: %v", err)
-	}
-	for i, bloom := range input {
-		if err := gen.AddBloom(uint(i), bloom); err != nil {
-			t.Fatalf("bloom %d: failed to add: %v", i, err)
-		}
-	}
-	for i, want := range output {
-		have, err := gen.Bitset(uint(i))
-		if err != nil {
-			t.Fatalf("output %d: failed to retrieve bits: %v", i, err)
-		}
-		if !bytes.Equal(have, want[:]) {
-			t.Errorf("output %d: bit vector mismatch have %x, want %x", i, have, want)
-		}
-	}
-}
-
-func BenchmarkGenerator(b *testing.B) {
-	var input [types.BloomBitLength][types.BloomByteLength]byte
-	b.Run("empty", func(b *testing.B) {
-		b.ReportAllocs()
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			// Crunch the input through the generator and verify the result
-			gen, err := NewGenerator(types.BloomBitLength)
-			if err != nil {
-				b.Fatalf("failed to create bloombit generator: %v", err)
-			}
-			for j, bloom := range &input {
-				if err := gen.AddBloom(uint(j), bloom); err != nil {
-					b.Fatalf("bloom %d: failed to add: %v", i, err)
-				}
-			}
-		}
-	})
-	for i := 0; i < types.BloomBitLength; i++ {
-		crand.Read(input[i][:])
-	}
-	b.Run("random", func(b *testing.B) {
-		b.ReportAllocs()
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			// Crunch the input through the generator and verify the result
-			gen, err := NewGenerator(types.BloomBitLength)
-			if err != nil {
-				b.Fatalf("failed to create bloombit generator: %v", err)
-			}
-			for j, bloom := range &input {
-				if err := gen.AddBloom(uint(j), bloom); err != nil {
-					b.Fatalf("bloom %d: failed to add: %v", i, err)
-				}
-			}
-		}
-	})
-}

core/bloombits/matcher.go

@@ -1,649 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"math"
-	"sort"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common/bitutil"
-	"github.com/ethereum/go-ethereum/crypto"
-)
-
-// bloomIndexes represents the bit indexes inside the bloom filter that belong
-// to some key.
-type bloomIndexes [3]uint
-
-// calcBloomIndexes returns the bloom filter bit indexes belonging to the given key.
-func calcBloomIndexes(b []byte) bloomIndexes {
-	b = crypto.Keccak256(b)
-
-	var idxs bloomIndexes
-	for i := 0; i < len(idxs); i++ {
-		idxs[i] = (uint(b[2*i])<<8)&2047 + uint(b[2*i+1])
-	}
-	return idxs
-}
-
-// partialMatches with a non-nil vector represents a section in which some sub-
-// matchers have already found potential matches. Subsequent sub-matchers will
-// binary AND their matches with this vector. If vector is nil, it represents a
-// section to be processed by the first sub-matcher.
-type partialMatches struct {
-	section uint64
-	bitset  []byte
-}
-
-// Retrieval represents a request for retrieval task assignments for a given
-// bit with the given number of fetch elements, or a response for such a request.
-// It can also have the actual results set to be used as a delivery data struct.
-//
-// The context and error fields are used by the light client to terminate matching
-// early if an error is encountered on some path of the pipeline.
-type Retrieval struct {
-	Bit      uint
-	Sections []uint64
-	Bitsets  [][]byte
-
-	Context context.Context
-	Error   error
-}
-
-// Matcher is a pipelined system of schedulers and logic matchers which perform
-// binary AND/OR operations on the bit-streams, creating a stream of potential
-// blocks to inspect for data content.
-type Matcher struct {
-	sectionSize uint64 // Size of the data batches to filter on
-
-	filters    [][]bloomIndexes    // Filter the system is matching for
-	schedulers map[uint]*scheduler // Retrieval schedulers for loading bloom bits
-
-	retrievers chan chan uint       // Retriever processes waiting for bit allocations
-	counters   chan chan uint       // Retriever processes waiting for task count reports
-	retrievals chan chan *Retrieval // Retriever processes waiting for task allocations
-	deliveries chan *Retrieval      // Retriever processes waiting for task response deliveries
-
-	running atomic.Bool // Atomic flag whether a session is live or not
-}
-
-// NewMatcher creates a new pipeline for retrieving bloom bit streams and doing
-// address and topic filtering on them. Setting a filter component to `nil` is
-// allowed and will result in that filter rule being skipped (OR 0x11...1).
-func NewMatcher(sectionSize uint64, filters [][][]byte) *Matcher {
-	// Create the matcher instance
-	m := &Matcher{
-		sectionSize: sectionSize,
-		schedulers:  make(map[uint]*scheduler),
-		retrievers:  make(chan chan uint),
-		counters:    make(chan chan uint),
-		retrievals:  make(chan chan *Retrieval),
-		deliveries:  make(chan *Retrieval),
-	}
-	// Calculate the bloom bit indexes for the groups we're interested in
-	m.filters = nil
-
-	for _, filter := range filters {
-		// Gather the bit indexes of the filter rule, special casing the nil filter
-		if len(filter) == 0 {
-			continue
-		}
-		bloomBits := make([]bloomIndexes, len(filter))
-		for i, clause := range filter {
-			if clause == nil {
-				bloomBits = nil
-				break
-			}
-			bloomBits[i] = calcBloomIndexes(clause)
-		}
-		// Accumulate the filter rules if no nil rule was within
-		if bloomBits != nil {
-			m.filters = append(m.filters, bloomBits)
-		}
-	}
-	// For every bit, create a scheduler to load/download the bit vectors
-	for _, bloomIndexLists := range m.filters {
-		for _, bloomIndexList := range bloomIndexLists {
-			for _, bloomIndex := range bloomIndexList {
-				m.addScheduler(bloomIndex)
-			}
-		}
-	}
-	return m
-}
-
-// addScheduler adds a bit stream retrieval scheduler for the given bit index if
-// it has not existed before. If the bit is already selected for filtering, the
-// existing scheduler can be used.
-func (m *Matcher) addScheduler(idx uint) {
-	if _, ok := m.schedulers[idx]; ok {
-		return
-	}
-	m.schedulers[idx] = newScheduler(idx)
-}
-
-// Start starts the matching process and returns a stream of bloom matches in
-// a given range of blocks. If there are no more matches in the range, the result
-// channel is closed.
-func (m *Matcher) Start(ctx context.Context, begin, end uint64, results chan uint64) (*MatcherSession, error) {
-	// Make sure we're not creating concurrent sessions
-	if m.running.Swap(true) {
-		return nil, errors.New("matcher already running")
-	}
-	defer m.running.Store(false)
-
-	// Initiate a new matching round
-	session := &MatcherSession{
-		matcher: m,
-		quit:    make(chan struct{}),
-		ctx:     ctx,
-	}
-	for _, scheduler := range m.schedulers {
-		scheduler.reset()
-	}
-	sink := m.run(begin, end, cap(results), session)
-
-	// Read the output from the result sink and deliver to the user
-	session.pend.Add(1)
-	go func() {
-		defer session.pend.Done()
-		defer close(results)
-
-		for {
-			select {
-			case <-session.quit:
-				return
-
-			case res, ok := <-sink:
-				// New match result found
-				if !ok {
-					return
-				}
-				// Calculate the first and last blocks of the section
-				sectionStart := res.section * m.sectionSize
-
-				first := sectionStart
-				if begin > first {
-					first = begin
-				}
-				last := sectionStart + m.sectionSize - 1
-				if end < last {
-					last = end
-				}
-				// Iterate over all the blocks in the section and return the matching ones
-				for i := first; i <= last; i++ {
-					// Skip the entire byte if no matches are found inside (and we're processing an entire byte!)
-					next := res.bitset[(i-sectionStart)/8]
-					if next == 0 {
-						if i%8 == 0 {
-							i += 7
-						}
-						continue
-					}
-					// Some bit it set, do the actual submatching
-					if bit := 7 - i%8; next&(1<<bit) != 0 {
-						select {
-						case <-session.quit:
-							return
-						case results <- i:
-						}
-					}
-				}
-			}
-		}
-	}()
-	return session, nil
-}
-
-// run creates a daisy-chain of sub-matchers, one for the address set and one
-// for each topic set, each sub-matcher receiving a section only if the previous
-// ones have all found a potential match in one of the blocks of the section,
-// then binary AND-ing its own matches and forwarding the result to the next one.
-//
-// The method starts feeding the section indexes into the first sub-matcher on a
-// new goroutine and returns a sink channel receiving the results.
-func (m *Matcher) run(begin, end uint64, buffer int, session *MatcherSession) chan *partialMatches {
-	// Create the source channel and feed section indexes into
-	source := make(chan *partialMatches, buffer)
-
-	session.pend.Add(1)
-	go func() {
-		defer session.pend.Done()
-		defer close(source)
-
-		for i := begin / m.sectionSize; i <= end/m.sectionSize; i++ {
-			select {
-			case <-session.quit:
-				return
-			case source <- &partialMatches{i, bytes.Repeat([]byte{0xff}, int(m.sectionSize/8))}:
-			}
-		}
-	}()
-	// Assemble the daisy-chained filtering pipeline
-	next := source
-	dist := make(chan *request, buffer)
-
-	for _, bloom := range m.filters {
-		next = m.subMatch(next, dist, bloom, session)
-	}
-	// Start the request distribution
-	session.pend.Add(1)
-	go m.distributor(dist, session)
-
-	return next
-}
-
-// subMatch creates a sub-matcher that filters for a set of addresses or topics, binary OR-s those matches, then
-// binary AND-s the result to the daisy-chain input (source) and forwards it to the daisy-chain output.
-// The matches of each address/topic are calculated by fetching the given sections of the three bloom bit indexes belonging to
-// that address/topic, and binary AND-ing those vectors together.
-func (m *Matcher) subMatch(source chan *partialMatches, dist chan *request, bloom []bloomIndexes, session *MatcherSession) chan *partialMatches {
-	// Start the concurrent schedulers for each bit required by the bloom filter
-	sectionSources := make([][3]chan uint64, len(bloom))
-	sectionSinks := make([][3]chan []byte, len(bloom))
-	for i, bits := range bloom {
-		for j, bit := range bits {
-			sectionSources[i][j] = make(chan uint64, cap(source))
-			sectionSinks[i][j] = make(chan []byte, cap(source))
-
-			m.schedulers[bit].run(sectionSources[i][j], dist, sectionSinks[i][j], session.quit, &session.pend)
-		}
-	}
-
-	process := make(chan *partialMatches, cap(source)) // entries from source are forwarded here after fetches have been initiated
-	results := make(chan *partialMatches, cap(source))
-
-	session.pend.Add(2)
-	go func() {
-		// Tear down the goroutine and terminate all source channels
-		defer session.pend.Done()
-		defer close(process)
-
-		defer func() {
-			for _, bloomSources := range sectionSources {
-				for _, bitSource := range bloomSources {
-					close(bitSource)
-				}
-			}
-		}()
-		// Read sections from the source channel and multiplex into all bit-schedulers
-		for {
-			select {
-			case <-session.quit:
-				return
-
-			case subres, ok := <-source:
-				// New subresult from previous link
-				if !ok {
-					return
-				}
-				// Multiplex the section index to all bit-schedulers
-				for _, bloomSources := range sectionSources {
-					for _, bitSource := range bloomSources {
-						select {
-						case <-session.quit:
-							return
-						case bitSource <- subres.section:
-						}
-					}
-				}
-				// Notify the processor that this section will become available
-				select {
-				case <-session.quit:
-					return
-				case process <- subres:
-				}
-			}
-		}
-	}()
-
-	go func() {
-		// Tear down the goroutine and terminate the final sink channel
-		defer session.pend.Done()
-		defer close(results)
-
-		// Read the source notifications and collect the delivered results
-		for {
-			select {
-			case <-session.quit:
-				return
-
-			case subres, ok := <-process:
-				// Notified of a section being retrieved
-				if !ok {
-					return
-				}
-				// Gather all the sub-results and merge them together
-				var orVector []byte
-				for _, bloomSinks := range sectionSinks {
-					var andVector []byte
-					for _, bitSink := range bloomSinks {
-						var data []byte
-						select {
-						case <-session.quit:
-							return
-						case data = <-bitSink:
-						}
-						if andVector == nil {
-							andVector = make([]byte, int(m.sectionSize/8))
-							copy(andVector, data)
-						} else {
-							bitutil.ANDBytes(andVector, andVector, data)
-						}
-					}
-					if orVector == nil {
-						orVector = andVector
-					} else {
-						bitutil.ORBytes(orVector, orVector, andVector)
-					}
-				}
-
-				if orVector == nil {
-					orVector = make([]byte, int(m.sectionSize/8))
-				}
-				if subres.bitset != nil {
-					bitutil.ANDBytes(orVector, orVector, subres.bitset)
-				}
-				if bitutil.TestBytes(orVector) {
-					select {
-					case <-session.quit:
-						return
-					case results <- &partialMatches{subres.section, orVector}:
-					}
-				}
-			}
-		}
-	}()
-	return results
-}
-
-// distributor receives requests from the schedulers and queues them into a set
-// of pending requests, which are assigned to retrievers wanting to fulfil them.
-func (m *Matcher) distributor(dist chan *request, session *MatcherSession) {
-	defer session.pend.Done()
-
-	var (
-		requests   = make(map[uint][]uint64) // Per-bit list of section requests, ordered by section number
-		unallocs   = make(map[uint]struct{}) // Bits with pending requests but not allocated to any retriever
-		retrievers chan chan uint            // Waiting retrievers (toggled to nil if unallocs is empty)
-		allocs     int                       // Number of active allocations to handle graceful shutdown requests
-		shutdown   = session.quit            // Shutdown request channel, will gracefully wait for pending requests
-	)
-
-	// assign is a helper method to try to assign a pending bit an actively
-	// listening servicer, or schedule it up for later when one arrives.
-	assign := func(bit uint) {
-		select {
-		case fetcher := <-m.retrievers:
-			allocs++
-			fetcher <- bit
-		default:
-			// No retrievers active, start listening for new ones
-			retrievers = m.retrievers
-			unallocs[bit] = struct{}{}
-		}
-	}
-
-	for {
-		select {
-		case <-shutdown:
-			// Shutdown requested. No more retrievers can be allocated,
-			// but we still need to wait until all pending requests have returned.
-			shutdown = nil
-			if allocs == 0 {
-				return
-			}
-
-		case req := <-dist:
-			// New retrieval request arrived to be distributed to some fetcher process
-			queue := requests[req.bit]
-			index := sort.Search(len(queue), func(i int) bool { return queue[i] >= req.section })
-			requests[req.bit] = append(queue[:index], append([]uint64{req.section}, queue[index:]...)...)
-
-			// If it's a new bit and we have waiting fetchers, allocate to them
-			if len(queue) == 0 {
-				assign(req.bit)
-			}
-
-		case fetcher := <-retrievers:
-			// New retriever arrived, find the lowest section-ed bit to assign
-			bit, best := uint(0), uint64(math.MaxUint64)
-			for idx := range unallocs {
-				if requests[idx][0] < best {
-					bit, best = idx, requests[idx][0]
-				}
-			}
-			// Stop tracking this bit (and alloc notifications if no more work is available)
-			delete(unallocs, bit)
-			if len(unallocs) == 0 {
-				retrievers = nil
-			}
-			allocs++
-			fetcher <- bit
-
-		case fetcher := <-m.counters:
-			// New task count request arrives, return number of items
-			fetcher <- uint(len(requests[<-fetcher]))
-
-		case fetcher := <-m.retrievals:
-			// New fetcher waiting for tasks to retrieve, assign
-			task := <-fetcher
-			if want := len(task.Sections); want >= len(requests[task.Bit]) {
-				task.Sections = requests[task.Bit]
-				delete(requests, task.Bit)
-			} else {
-				task.Sections = append(task.Sections[:0], requests[task.Bit][:want]...)
-				requests[task.Bit] = append(requests[task.Bit][:0], requests[task.Bit][want:]...)
-			}
-			fetcher <- task
-
-			// If anything was left unallocated, try to assign to someone else
-			if len(requests[task.Bit]) > 0 {
-				assign(task.Bit)
-			}
-
-		case result := <-m.deliveries:
-			// New retrieval task response from fetcher, split out missing sections and
-			// deliver complete ones
-			var (
-				sections = make([]uint64, 0, len(result.Sections))
-				bitsets  = make([][]byte, 0, len(result.Bitsets))
-				missing  = make([]uint64, 0, len(result.Sections))
-			)
-			for i, bitset := range result.Bitsets {
-				if len(bitset) == 0 {
-					missing = append(missing, result.Sections[i])
-					continue
-				}
-				sections = append(sections, result.Sections[i])
-				bitsets = append(bitsets, bitset)
-			}
-			m.schedulers[result.Bit].deliver(sections, bitsets)
-			allocs--
-
-			// Reschedule missing sections and allocate bit if newly available
-			if len(missing) > 0 {
-				queue := requests[result.Bit]
-				for _, section := range missing {
-					index := sort.Search(len(queue), func(i int) bool { return queue[i] >= section })
-					queue = append(queue[:index], append([]uint64{section}, queue[index:]...)...)
-				}
-				requests[result.Bit] = queue
-
-				if len(queue) == len(missing) {
-					assign(result.Bit)
-				}
-			}
-
-			// End the session when all pending deliveries have arrived.
-			if shutdown == nil && allocs == 0 {
-				return
-			}
-		}
-	}
-}
-
-// MatcherSession is returned by a started matcher to be used as a terminator
-// for the actively running matching operation.
-type MatcherSession struct {
-	matcher *Matcher
-
-	closer sync.Once     // Sync object to ensure we only ever close once
-	quit   chan struct{} // Quit channel to request pipeline termination
-
-	ctx     context.Context // Context used by the light client to abort filtering
-	err     error           // Global error to track retrieval failures deep in the chain
-	errLock sync.Mutex
-
-	pend sync.WaitGroup
-}
-
-// Close stops the matching process and waits for all subprocesses to terminate
-// before returning. The timeout may be used for graceful shutdown, allowing the
-// currently running retrievals to complete before this time.
-func (s *MatcherSession) Close() {
-	s.closer.Do(func() {
-		// Signal termination and wait for all goroutines to tear down
-		close(s.quit)
-		s.pend.Wait()
-	})
-}
-
-// Error returns any failure encountered during the matching session.
-func (s *MatcherSession) Error() error {
-	s.errLock.Lock()
-	defer s.errLock.Unlock()
-
-	return s.err
-}
-
-// allocateRetrieval assigns a bloom bit index to a client process that can either
-// immediately request and fetch the section contents assigned to this bit or wait
-// a little while for more sections to be requested.
-func (s *MatcherSession) allocateRetrieval() (uint, bool) {
-	fetcher := make(chan uint)
-
-	select {
-	case <-s.quit:
-		return 0, false
-	case s.matcher.retrievers <- fetcher:
-		bit, ok := <-fetcher
-		return bit, ok
-	}
-}
-
-// pendingSections returns the number of pending section retrievals belonging to
-// the given bloom bit index.
-func (s *MatcherSession) pendingSections(bit uint) int {
-	fetcher := make(chan uint)
-
-	select {
-	case <-s.quit:
-		return 0
-	case s.matcher.counters <- fetcher:
-		fetcher <- bit
-		return int(<-fetcher)
-	}
-}
-
-// allocateSections assigns all or part of an already allocated bit-task queue
-// to the requesting process.
-func (s *MatcherSession) allocateSections(bit uint, count int) []uint64 {
-	fetcher := make(chan *Retrieval)
-
-	select {
-	case <-s.quit:
-		return nil
-	case s.matcher.retrievals <- fetcher:
-		task := &Retrieval{
-			Bit:      bit,
-			Sections: make([]uint64, count),
-		}
-		fetcher <- task
-		return (<-fetcher).Sections
-	}
-}
-
-// deliverSections delivers a batch of section bit-vectors for a specific bloom
-// bit index to be injected into the processing pipeline.
-func (s *MatcherSession) deliverSections(bit uint, sections []uint64, bitsets [][]byte) {
-	s.matcher.deliveries <- &Retrieval{Bit: bit, Sections: sections, Bitsets: bitsets}
-}
-
-// Multiplex polls the matcher session for retrieval tasks and multiplexes it into
-// the requested retrieval queue to be serviced together with other sessions.
-//
-// This method will block for the lifetime of the session. Even after termination
-// of the session, any request in-flight need to be responded to! Empty responses
-// are fine though in that case.
-func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan *Retrieval) {
-	waitTimer := time.NewTimer(wait)
-	defer waitTimer.Stop()
-
-	for {
-		// Allocate a new bloom bit index to retrieve data for, stopping when done
-		bit, ok := s.allocateRetrieval()
-		if !ok {
-			return
-		}
-		// Bit allocated, throttle a bit if we're below our batch limit
-		if s.pendingSections(bit) < batch {
-			waitTimer.Reset(wait)
-			select {
-			case <-s.quit:
-				// Session terminating, we can't meaningfully service, abort
-				s.allocateSections(bit, 0)
-				s.deliverSections(bit, []uint64{}, [][]byte{})
-				return
-
-			case <-waitTimer.C:
-				// Throttling up, fetch whatever is available
-			}
-		}
-		// Allocate as much as we can handle and request servicing
-		sections := s.allocateSections(bit, batch)
-		request := make(chan *Retrieval)
-
-		select {
-		case <-s.quit:
-			// Session terminating, we can't meaningfully service, abort
-			s.deliverSections(bit, sections, make([][]byte, len(sections)))
-			return
-
-		case mux <- request:
-			// Retrieval accepted, something must arrive before we're aborting
-			request <- &Retrieval{Bit: bit, Sections: sections, Context: s.ctx}
-
-			result := <-request
-
-			// Deliver a result before s.Close() to avoid a deadlock
-			s.deliverSections(result.Bit, result.Sections, result.Bitsets)
-
-			if result.Error != nil {
-				s.errLock.Lock()
-				s.err = result.Error
-				s.errLock.Unlock()
-				s.Close()
-			}
-		}
-	}
-}

core/bloombits/matcher_test.go

@@ -1,292 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"context"
-	"math/rand"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-)
-
-const testSectionSize = 4096
-
-// Tests that wildcard filter rules (nil) can be specified and are handled well.
-func TestMatcherWildcards(t *testing.T) {
-	t.Parallel()
-	matcher := NewMatcher(testSectionSize, [][][]byte{
-		{common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard
-		{common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()},       // Default hash is not a wildcard
-		{common.Hash{0x01}.Bytes()},                              // Plain rule, sanity check
-		{common.Hash{0x01}.Bytes(), nil},                         // Wildcard suffix, drop rule
-		{nil, common.Hash{0x01}.Bytes()},                         // Wildcard prefix, drop rule
-		{nil, nil},                                               // Wildcard combo, drop rule
-		{},                                                       // Inited wildcard rule, drop rule
-		nil,                                                      // Proper wildcard rule, drop rule
-	})
-	if len(matcher.filters) != 3 {
-		t.Fatalf("filter system size mismatch: have %d, want %d", len(matcher.filters), 3)
-	}
-	if len(matcher.filters[0]) != 2 {
-		t.Fatalf("address clause size mismatch: have %d, want %d", len(matcher.filters[0]), 2)
-	}
-	if len(matcher.filters[1]) != 2 {
-		t.Fatalf("combo topic clause size mismatch: have %d, want %d", len(matcher.filters[1]), 2)
-	}
-	if len(matcher.filters[2]) != 1 {
-		t.Fatalf("singletone topic clause size mismatch: have %d, want %d", len(matcher.filters[2]), 1)
-	}
-}
-
-// Tests the matcher pipeline on a single continuous workflow without interrupts.
-func TestMatcherContinuous(t *testing.T) {
-	t.Parallel()
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, false, 75)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, false, 81)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, false, 36)
-}
-
-// Tests the matcher pipeline on a constantly interrupted and resumed work pattern
-// with the aim of ensuring data items are requested only once.
-func TestMatcherIntermittent(t *testing.T) {
-	t.Parallel()
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, true, 75)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, true, 81)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, true, 36)
-}
-
-// Tests the matcher pipeline on random input to hopefully catch anomalies.
-func TestMatcherRandom(t *testing.T) {
-	t.Parallel()
-	for i := 0; i < 10; i++ {
-		testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 0, 10000, 0)
-	}
-}
-
-// Tests that the matcher can properly find matches if the starting block is
-// shifted from a multiple of 8. This is needed to cover an optimisation with
-// bitset matching https://github.com/ethereum/go-ethereum/issues/15309.
-func TestMatcherShifted(t *testing.T) {
-	t.Parallel()
-	// Block 0 always matches in the tests, skip ahead of first 8 blocks with the
-	// start to get a potential zero byte in the matcher bitset.
-
-	// To keep the second bitset byte zero, the filter must only match for the first
-	// time in block 16, so doing an all-16 bit filter should suffice.
-
-	// To keep the starting block non divisible by 8, block number 9 is the first
-	// that would introduce a shift and not match block 0.
-	testMatcherBothModes(t, [][]bloomIndexes{{{16, 16, 16}}}, 9, 64, 0)
-}
-
-// Tests that matching on everything doesn't crash (special case internally).
-func TestWildcardMatcher(t *testing.T) {
-	t.Parallel()
-	testMatcherBothModes(t, nil, 0, 10000, 0)
-}
-
-// makeRandomIndexes generates a random filter system, composed of multiple filter
-// criteria, each having one bloom list component for the address and arbitrarily
-// many topic bloom list components.
-func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {
-	res := make([][]bloomIndexes, len(lengths))
-	for i, topics := range lengths {
-		res[i] = make([]bloomIndexes, topics)
-		for j := 0; j < topics; j++ {
-			for k := 0; k < len(res[i][j]); k++ {
-				res[i][j][k] = uint(rand.Intn(max-1) + 2)
-			}
-		}
-	}
-	return res
-}
-
-// testMatcherDiffBatches runs the given matches test in single-delivery and also
-// in batches delivery mode, verifying that all kinds of deliveries are handled
-// correctly within.
-func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) {
-	singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1)
-	batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16)
-
-	if singleton != batched {
-		t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in singleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched)
-	}
-}
-
-// testMatcherBothModes runs the given matcher test in both continuous as well as
-// in intermittent mode, verifying that the request counts match each other.
-func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, retrievals uint32) {
-	continuous := testMatcher(t, filter, start, blocks, false, retrievals, 16)
-	intermittent := testMatcher(t, filter, start, blocks, true, retrievals, 16)
-
-	if continuous != intermittent {
-		t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent)
-	}
-}
-
-// testMatcher is a generic tester to run the given matcher test and return the
-// number of requests made for cross validation between different modes.
-func testMatcher(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 {
-	// Create a new matcher an simulate our explicit random bitsets
-	matcher := NewMatcher(testSectionSize, nil)
-	matcher.filters = filter
-
-	for _, rule := range filter {
-		for _, topic := range rule {
-			for _, bit := range topic {
-				matcher.addScheduler(bit)
-			}
-		}
-	}
-	// Track the number of retrieval requests made
-	var requested atomic.Uint32
-
-	// Start the matching session for the filter and the retriever goroutines
-	quit := make(chan struct{})
-	matches := make(chan uint64, 16)
-
-	session, err := matcher.Start(context.Background(), start, blocks-1, matches)
-	if err != nil {
-		t.Fatalf("failed to stat matcher session: %v", err)
-	}
-	startRetrievers(session, quit, &requested, maxReqCount)
-
-	// Iterate over all the blocks and verify that the pipeline produces the correct matches
-	for i := start; i < blocks; i++ {
-		if expMatch3(filter, i) {
-			match, ok := <-matches
-			if !ok {
-				t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i)
-				return 0
-			}
-			if match != i {
-				t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match)
-			}
-			// If we're testing intermittent mode, abort and restart the pipeline
-			if intermittent {
-				session.Close()
-				close(quit)
-
-				quit = make(chan struct{})
-				matches = make(chan uint64, 16)
-
-				session, err = matcher.Start(context.Background(), i+1, blocks-1, matches)
-				if err != nil {
-					t.Fatalf("failed to stat matcher session: %v", err)
-				}
-				startRetrievers(session, quit, &requested, maxReqCount)
-			}
-		}
-	}
-	// Ensure the result channel is torn down after the last block
-	match, ok := <-matches
-	if ok {
-		t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match)
-	}
-	// Clean up the session and ensure we match the expected retrieval count
-	session.Close()
-	close(quit)
-
-	if retrievals != 0 && requested.Load() != retrievals {
-		t.Errorf("filter = %v  blocks = %v  intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested.Load(), retrievals)
-	}
-	return requested.Load()
-}
-
-// startRetrievers starts a batch of goroutines listening for section requests
-// and serving them.
-func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *atomic.Uint32, batch int) {
-	requests := make(chan chan *Retrieval)
-
-	for i := 0; i < 10; i++ {
-		// Start a multiplexer to test multiple threaded execution
-		go session.Multiplex(batch, 100*time.Microsecond, requests)
-
-		// Start a services to match the above multiplexer
-		go func() {
-			for {
-				// Wait for a service request or a shutdown
-				select {
-				case <-quit:
-					return
-
-				case request := <-requests:
-					task := <-request
-
-					task.Bitsets = make([][]byte, len(task.Sections))
-					for i, section := range task.Sections {
-						if rand.Int()%4 != 0 { // Handle occasional missing deliveries
-							task.Bitsets[i] = generateBitset(task.Bit, section)
-							retrievals.Add(1)
-						}
-					}
-					request <- task
-				}
-			}
-		}()
-	}
-}
-
-// generateBitset generates the rotated bitset for the given bloom bit and section
-// numbers.
-func generateBitset(bit uint, section uint64) []byte {
-	bitset := make([]byte, testSectionSize/8)
-	for i := 0; i < len(bitset); i++ {
-		for b := 0; b < 8; b++ {
-			blockIdx := section*testSectionSize + uint64(i*8+b)
-			bitset[i] += bitset[i]
-			if (blockIdx % uint64(bit)) == 0 {
-				bitset[i]++
-			}
-		}
-	}
-	return bitset
-}
-
-func expMatch1(filter bloomIndexes, i uint64) bool {
-	for _, ii := range filter {
-		if (i % uint64(ii)) != 0 {
-			return false
-		}
-	}
-	return true
-}
-
-func expMatch2(filter []bloomIndexes, i uint64) bool {
-	for _, ii := range filter {
-		if expMatch1(ii, i) {
-			return true
-		}
-	}
-	return false
-}
-
-func expMatch3(filter [][]bloomIndexes, i uint64) bool {
-	for _, ii := range filter {
-		if !expMatch2(ii, i) {
-			return false
-		}
-	}
-	return true
-}

core/bloombits/scheduler.go

@@ -1,181 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"sync"
-)
-
-// request represents a bloom retrieval task to prioritize and pull from the local
-// database or remotely from the network.
-type request struct {
-	section uint64 // Section index to retrieve the bit-vector from
-	bit     uint   // Bit index within the section to retrieve the vector of
-}
-
-// response represents the state of a requested bit-vector through a scheduler.
-type response struct {
-	cached []byte        // Cached bits to dedup multiple requests
-	done   chan struct{} // Channel to allow waiting for completion
-}
-
-// scheduler handles the scheduling of bloom-filter retrieval operations for
-// entire section-batches belonging to a single bloom bit. Beside scheduling the
-// retrieval operations, this struct also deduplicates the requests and caches
-// the results to minimize network/database overhead even in complex filtering
-// scenarios.
-type scheduler struct {
-	bit       uint                 // Index of the bit in the bloom filter this scheduler is responsible for
-	responses map[uint64]*response // Currently pending retrieval requests or already cached responses
-	lock      sync.Mutex           // Lock protecting the responses from concurrent access
-}
-
-// newScheduler creates a new bloom-filter retrieval scheduler for a specific
-// bit index.
-func newScheduler(idx uint) *scheduler {
-	return &scheduler{
-		bit:       idx,
-		responses: make(map[uint64]*response),
-	}
-}
-
-// run creates a retrieval pipeline, receiving section indexes from sections and
-// returning the results in the same order through the done channel. Concurrent
-// runs of the same scheduler are allowed, leading to retrieval task deduplication.
-func (s *scheduler) run(sections chan uint64, dist chan *request, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) {
-	// Create a forwarder channel between requests and responses of the same size as
-	// the distribution channel (since that will block the pipeline anyway).
-	pend := make(chan uint64, cap(dist))
-
-	// Start the pipeline schedulers to forward between user -> distributor -> user
-	wg.Add(2)
-	go s.scheduleRequests(sections, dist, pend, quit, wg)
-	go s.scheduleDeliveries(pend, done, quit, wg)
-}
-
-// reset cleans up any leftovers from previous runs. This is required before a
-// restart to ensure the no previously requested but never delivered state will
-// cause a lockup.
-func (s *scheduler) reset() {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	for section, res := range s.responses {
-		if res.cached == nil {
-			delete(s.responses, section)
-		}
-	}
-}
-
-// scheduleRequests reads section retrieval requests from the input channel,
-// deduplicates the stream and pushes unique retrieval tasks into the distribution
-// channel for a database or network layer to honour.
-func (s *scheduler) scheduleRequests(reqs chan uint64, dist chan *request, pend chan uint64, quit chan struct{}, wg *sync.WaitGroup) {
-	// Clean up the goroutine and pipeline when done
-	defer wg.Done()
-	defer close(pend)
-
-	// Keep reading and scheduling section requests
-	for {
-		select {
-		case <-quit:
-			return
-
-		case section, ok := <-reqs:
-			// New section retrieval requested
-			if !ok {
-				return
-			}
-			// Deduplicate retrieval requests
-			unique := false
-
-			s.lock.Lock()
-			if s.responses[section] == nil {
-				s.responses[section] = &response{
-					done: make(chan struct{}),
-				}
-				unique = true
-			}
-			s.lock.Unlock()
-
-			// Schedule the section for retrieval and notify the deliverer to expect this section
-			if unique {
-				select {
-				case <-quit:
-					return
-				case dist <- &request{bit: s.bit, section: section}:
-				}
-			}
-			select {
-			case <-quit:
-				return
-			case pend <- section:
-			}
-		}
-	}
-}
-
-// scheduleDeliveries reads section acceptance notifications and waits for them
-// to be delivered, pushing them into the output data buffer.
-func (s *scheduler) scheduleDeliveries(pend chan uint64, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) {
-	// Clean up the goroutine and pipeline when done
-	defer wg.Done()
-	defer close(done)
-
-	// Keep reading notifications and scheduling deliveries
-	for {
-		select {
-		case <-quit:
-			return
-
-		case idx, ok := <-pend:
-			// New section retrieval pending
-			if !ok {
-				return
-			}
-			// Wait until the request is honoured
-			s.lock.Lock()
-			res := s.responses[idx]
-			s.lock.Unlock()
-
-			select {
-			case <-quit:
-				return
-			case <-res.done:
-			}
-			// Deliver the result
-			select {
-			case <-quit:
-				return
-			case done <- res.cached:
-			}
-		}
-	}
-}
-
-// deliver is called by the request distributor when a reply to a request arrives.
-func (s *scheduler) deliver(sections []uint64, data [][]byte) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	for i, section := range sections {
-		if res := s.responses[section]; res != nil && res.cached == nil { // Avoid non-requests and double deliveries
-			res.cached = data[i]
-			close(res.done)
-		}
-	}
-}

core/bloombits/scheduler_test.go

@@ -1,103 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"bytes"
-	"math/big"
-	"sync"
-	"sync/atomic"
-	"testing"
-)
-
-// Tests that the scheduler can deduplicate and forward retrieval requests to
-// underlying fetchers and serve responses back, irrelevant of the concurrency
-// of the requesting clients or serving data fetchers.
-func TestSchedulerSingleClientSingleFetcher(t *testing.T) { testScheduler(t, 1, 1, 5000) }
-func TestSchedulerSingleClientMultiFetcher(t *testing.T)  { testScheduler(t, 1, 10, 5000) }
-func TestSchedulerMultiClientSingleFetcher(t *testing.T)  { testScheduler(t, 10, 1, 5000) }
-func TestSchedulerMultiClientMultiFetcher(t *testing.T)   { testScheduler(t, 10, 10, 5000) }
-
-func testScheduler(t *testing.T, clients int, fetchers int, requests int) {
-	t.Parallel()
-	f := newScheduler(0)
-
-	// Create a batch of handler goroutines that respond to bloom bit requests and
-	// deliver them to the scheduler.
-	var fetchPend sync.WaitGroup
-	fetchPend.Add(fetchers)
-	defer fetchPend.Wait()
-
-	fetch := make(chan *request, 16)
-	defer close(fetch)
-
-	var delivered atomic.Uint32
-	for i := 0; i < fetchers; i++ {
-		go func() {
-			defer fetchPend.Done()
-
-			for req := range fetch {
-				delivered.Add(1)
-
-				f.deliver([]uint64{
-					req.section + uint64(requests), // Non-requested data (ensure it doesn't go out of bounds)
-					req.section,                    // Requested data
-					req.section,                    // Duplicated data (ensure it doesn't double close anything)
-				}, [][]byte{
-					{},
-					new(big.Int).SetUint64(req.section).Bytes(),
-					new(big.Int).SetUint64(req.section).Bytes(),
-				})
-			}
-		}()
-	}
-	// Start a batch of goroutines to concurrently run scheduling tasks
-	quit := make(chan struct{})
-
-	var pend sync.WaitGroup
-	pend.Add(clients)
-
-	for i := 0; i < clients; i++ {
-		go func() {
-			defer pend.Done()
-
-			in := make(chan uint64, 16)
-			out := make(chan []byte, 16)
-
-			f.run(in, fetch, out, quit, &pend)
-
-			go func() {
-				for j := 0; j < requests; j++ {
-					in <- uint64(j)
-				}
-				close(in)
-			}()
-			b := new(big.Int)
-			for j := 0; j < requests; j++ {
-				bits := <-out
-				if want := b.SetUint64(uint64(j)).Bytes(); !bytes.Equal(bits, want) {
-					t.Errorf("vector %d: delivered content mismatch: have %x, want %x", j, bits, want)
-				}
-			}
-		}()
-	}
-	pend.Wait()
-
-	if have := delivered.Load(); int(have) != requests {
-		t.Errorf("request count mismatch: have %v, want %v", have, requests)
-	}
-}

core/chain_indexer.go

@@ -1,522 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package core
-
-import (
-	"context"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/event"
-	"github.com/ethereum/go-ethereum/log"
-)
-
-// ChainIndexerBackend defines the methods needed to process chain segments in
-// the background and write the segment results into the database. These can be
-// used to create filter blooms or CHTs.
-type ChainIndexerBackend interface {
-	// Reset initiates the processing of a new chain segment, potentially terminating
-	// any partially completed operations (in case of a reorg).
-	Reset(ctx context.Context, section uint64, prevHead common.Hash) error
-
-	// Process crunches through the next header in the chain segment. The caller
-	// will ensure a sequential order of headers.
-	Process(ctx context.Context, header *types.Header) error
-
-	// Commit finalizes the section metadata and stores it into the database.
-	Commit() error
-
-	// Prune deletes the chain index older than the given threshold.
-	Prune(threshold uint64) error
-}
-
-// ChainIndexerChain interface is used for connecting the indexer to a blockchain
-type ChainIndexerChain interface {
-	// CurrentHeader retrieves the latest locally known header.
-	CurrentHeader() *types.Header
-
-	// SubscribeChainHeadEvent subscribes to new head header notifications.
-	SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription
-}
-
-// ChainIndexer does a post-processing job for equally sized sections of the
-// canonical chain (like BlooomBits and CHT structures). A ChainIndexer is
-// connected to the blockchain through the event system by starting a
-// ChainHeadEventLoop in a goroutine.
-//
-// Further child ChainIndexers can be added which use the output of the parent
-// section indexer. These child indexers receive new head notifications only
-// after an entire section has been finished or in case of rollbacks that might
-// affect already finished sections.
-type ChainIndexer struct {
-	chainDb  ethdb.Database      // Chain database to index the data from
-	indexDb  ethdb.Database      // Prefixed table-view of the db to write index metadata into
-	backend  ChainIndexerBackend // Background processor generating the index data content
-	children []*ChainIndexer     // Child indexers to cascade chain updates to
-
-	active    atomic.Bool     // Flag whether the event loop was started
-	update    chan struct{}   // Notification channel that headers should be processed
-	quit      chan chan error // Quit channel to tear down running goroutines
-	ctx       context.Context
-	ctxCancel func()
-
-	sectionSize uint64 // Number of blocks in a single chain segment to process
-	confirmsReq uint64 // Number of confirmations before processing a completed segment
-
-	storedSections uint64 // Number of sections successfully indexed into the database
-	knownSections  uint64 // Number of sections known to be complete (block wise)
-	cascadedHead   uint64 // Block number of the last completed section cascaded to subindexers
-
-	checkpointSections uint64      // Number of sections covered by the checkpoint
-	checkpointHead     common.Hash // Section head belonging to the checkpoint
-
-	throttling time.Duration // Disk throttling to prevent a heavy upgrade from hogging resources
-
-	log  log.Logger
-	lock sync.Mutex
-}
-
-// NewChainIndexer creates a new chain indexer to do background processing on
-// chain segments of a given size after certain number of confirmations passed.
-// The throttling parameter might be used to prevent database thrashing.
-func NewChainIndexer(chainDb ethdb.Database, indexDb ethdb.Database, backend ChainIndexerBackend, section, confirm uint64, throttling time.Duration, kind string) *ChainIndexer {
-	c := &ChainIndexer{
-		chainDb:     chainDb,
-		indexDb:     indexDb,
-		backend:     backend,
-		update:      make(chan struct{}, 1),
-		quit:        make(chan chan error),
-		sectionSize: section,
-		confirmsReq: confirm,
-		throttling:  throttling,
-		log:         log.New("type", kind),
-	}
-	// Initialize database dependent fields and start the updater
-	c.loadValidSections()
-	c.ctx, c.ctxCancel = context.WithCancel(context.Background())
-
-	go c.updateLoop()
-
-	return c
-}
-
-// AddCheckpoint adds a checkpoint. Sections are never processed and the chain
-// is not expected to be available before this point. The indexer assumes that
-// the backend has sufficient information available to process subsequent sections.
-//
-// Note: knownSections == 0 and storedSections == checkpointSections until
-// syncing reaches the checkpoint
-func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	// Short circuit if the given checkpoint is below than local's.
-	if c.checkpointSections >= section+1 || section < c.storedSections {
-		return
-	}
-	c.checkpointSections = section + 1
-	c.checkpointHead = shead
-
-	c.setSectionHead(section, shead)
-	c.setValidSections(section + 1)
-}
-
-// Start creates a goroutine to feed chain head events into the indexer for
-// cascading background processing. Children do not need to be started, they
-// are notified about new events by their parents.
-func (c *ChainIndexer) Start(chain ChainIndexerChain) {
-	events := make(chan ChainHeadEvent, 10)
-	sub := chain.SubscribeChainHeadEvent(events)
-
-	go c.eventLoop(chain.CurrentHeader(), events, sub)
-}
-
-// Close tears down all goroutines belonging to the indexer and returns any error
-// that might have occurred internally.
-func (c *ChainIndexer) Close() error {
-	var errs []error
-
-	c.ctxCancel()
-
-	// Tear down the primary update loop
-	errc := make(chan error)
-	c.quit <- errc
-	if err := <-errc; err != nil {
-		errs = append(errs, err)
-	}
-	// If needed, tear down the secondary event loop
-	if c.active.Load() {
-		c.quit <- errc
-		if err := <-errc; err != nil {
-			errs = append(errs, err)
-		}
-	}
-	// Close all children
-	for _, child := range c.children {
-		if err := child.Close(); err != nil {
-			errs = append(errs, err)
-		}
-	}
-	// Return any failures
-	switch {
-	case len(errs) == 0:
-		return nil
-
-	case len(errs) == 1:
-		return errs[0]
-
-	default:
-		return fmt.Errorf("%v", errs)
-	}
-}
-
-// eventLoop is a secondary - optional - event loop of the indexer which is only
-// started for the outermost indexer to push chain head events into a processing
-// queue.
-func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainHeadEvent, sub event.Subscription) {
-	// Mark the chain indexer as active, requiring an additional teardown
-	c.active.Store(true)
-
-	defer sub.Unsubscribe()
-
-	// Fire the initial new head event to start any outstanding processing
-	c.newHead(currentHeader.Number.Uint64(), false)
-
-	var (
-		prevHeader = currentHeader
-		prevHash   = currentHeader.Hash()
-	)
-	for {
-		select {
-		case errc := <-c.quit:
-			// Chain indexer terminating, report no failure and abort
-			errc <- nil
-			return
-
-		case ev, ok := <-events:
-			// Received a new event, ensure it's not nil (closing) and update
-			if !ok {
-				errc := <-c.quit
-				errc <- nil
-				return
-			}
-			if ev.Header.ParentHash != prevHash {
-				// Reorg to the common ancestor if needed (might not exist in light sync mode, skip reorg then)
-				// TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly?
-
-				if rawdb.ReadCanonicalHash(c.chainDb, prevHeader.Number.Uint64()) != prevHash {
-					if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, ev.Header); h != nil {
-						c.newHead(h.Number.Uint64(), true)
-					}
-				}
-			}
-			c.newHead(ev.Header.Number.Uint64(), false)
-
-			prevHeader, prevHash = ev.Header, ev.Header.Hash()
-		}
-	}
-}
-
-// newHead notifies the indexer about new chain heads and/or reorgs.
-func (c *ChainIndexer) newHead(head uint64, reorg bool) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	// If a reorg happened, invalidate all sections until that point
-	if reorg {
-		// Revert the known section number to the reorg point
-		known := (head + 1) / c.sectionSize
-		stored := known
-		if known < c.checkpointSections {
-			known = 0
-		}
-		if stored < c.checkpointSections {
-			stored = c.checkpointSections
-		}
-		if known < c.knownSections {
-			c.knownSections = known
-		}
-		// Revert the stored sections from the database to the reorg point
-		if stored < c.storedSections {
-			c.setValidSections(stored)
-		}
-		// Update the new head number to the finalized section end and notify children
-		head = known * c.sectionSize
-
-		if head < c.cascadedHead {
-			c.cascadedHead = head
-			for _, child := range c.children {
-				child.newHead(c.cascadedHead, true)
-			}
-		}
-		return
-	}
-	// No reorg, calculate the number of newly known sections and update if high enough
-	var sections uint64
-	if head >= c.confirmsReq {
-		sections = (head + 1 - c.confirmsReq) / c.sectionSize
-		if sections < c.checkpointSections {
-			sections = 0
-		}
-		if sections > c.knownSections {
-			if c.knownSections < c.checkpointSections {
-				// syncing reached the checkpoint, verify section head
-				syncedHead := rawdb.ReadCanonicalHash(c.chainDb, c.checkpointSections*c.sectionSize-1)
-				if syncedHead != c.checkpointHead {
-					c.log.Error("Synced chain does not match checkpoint", "number", c.checkpointSections*c.sectionSize-1, "expected", c.checkpointHead, "synced", syncedHead)
-					return
-				}
-			}
-			c.knownSections = sections
-
-			select {
-			case c.update <- struct{}{}:
-			default:
-			}
-		}
-	}
-}
-
-// updateLoop is the main event loop of the indexer which pushes chain segments
-// down into the processing backend.
-func (c *ChainIndexer) updateLoop() {
-	var (
-		updating bool
-		updated  time.Time
-	)
-
-	for {
-		select {
-		case errc := <-c.quit:
-			// Chain indexer terminating, report no failure and abort
-			errc <- nil
-			return
-
-		case <-c.update:
-			// Section headers completed (or rolled back), update the index
-			c.lock.Lock()
-			if c.knownSections > c.storedSections {
-				// Periodically print an upgrade log message to the user
-				if time.Since(updated) > 8*time.Second {
-					if c.knownSections > c.storedSections+1 {
-						updating = true
-						c.log.Info("Upgrading chain index", "percentage", c.storedSections*100/c.knownSections)
-					}
-					updated = time.Now()
-				}
-				// Cache the current section count and head to allow unlocking the mutex
-				c.verifyLastHead()
-				section := c.storedSections
-				var oldHead common.Hash
-				if section > 0 {
-					oldHead = c.SectionHead(section - 1)
-				}
-				// Process the newly defined section in the background
-				c.lock.Unlock()
-				newHead, err := c.processSection(section, oldHead)
-				if err != nil {
-					select {
-					case <-c.ctx.Done():
-						<-c.quit <- nil
-						return
-					default:
-					}
-					c.log.Error("Section processing failed", "error", err)
-				}
-				c.lock.Lock()
-
-				// If processing succeeded and no reorgs occurred, mark the section completed
-				if err == nil && (section == 0 || oldHead == c.SectionHead(section-1)) {
-					c.setSectionHead(section, newHead)
-					c.setValidSections(section + 1)
-					if c.storedSections == c.knownSections && updating {
-						updating = false
-						c.log.Info("Finished upgrading chain index")
-					}
-					c.cascadedHead = c.storedSections*c.sectionSize - 1
-					for _, child := range c.children {
-						c.log.Trace("Cascading chain index update", "head", c.cascadedHead)
-						child.newHead(c.cascadedHead, false)
-					}
-				} else {
-					// If processing failed, don't retry until further notification
-					c.log.Debug("Chain index processing failed", "section", section, "err", err)
-					c.verifyLastHead()
-					c.knownSections = c.storedSections
-				}
-			}
-			// If there are still further sections to process, reschedule
-			if c.knownSections > c.storedSections {
-				time.AfterFunc(c.throttling, func() {
-					select {
-					case c.update <- struct{}{}:
-					default:
-					}
-				})
-			}
-			c.lock.Unlock()
-		}
-	}
-}
-
-// processSection processes an entire section by calling backend functions while
-// ensuring the continuity of the passed headers. Since the chain mutex is not
-// held while processing, the continuity can be broken by a long reorg, in which
-// case the function returns with an error.
-func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (common.Hash, error) {
-	c.log.Trace("Processing new chain section", "section", section)
-
-	// Reset and partial processing
-	if err := c.backend.Reset(c.ctx, section, lastHead); err != nil {
-		c.setValidSections(0)
-		return common.Hash{}, err
-	}
-
-	for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ {
-		hash := rawdb.ReadCanonicalHash(c.chainDb, number)
-		if hash == (common.Hash{}) {
-			return common.Hash{}, fmt.Errorf("canonical block #%d unknown", number)
-		}
-		header := rawdb.ReadHeader(c.chainDb, hash, number)
-		if header == nil {
-			return common.Hash{}, fmt.Errorf("block #%d [%x..] not found", number, hash[:4])
-		} else if header.ParentHash != lastHead {
-			return common.Hash{}, errors.New("chain reorged during section processing")
-		}
-		if err := c.backend.Process(c.ctx, header); err != nil {
-			return common.Hash{}, err
-		}
-		lastHead = header.Hash()
-	}
-	if err := c.backend.Commit(); err != nil {
-		return common.Hash{}, err
-	}
-	return lastHead, nil
-}
-
-// verifyLastHead compares last stored section head with the corresponding block hash in the
-// actual canonical chain and rolls back reorged sections if necessary to ensure that stored
-// sections are all valid
-func (c *ChainIndexer) verifyLastHead() {
-	for c.storedSections > 0 && c.storedSections > c.checkpointSections {
-		if c.SectionHead(c.storedSections-1) == rawdb.ReadCanonicalHash(c.chainDb, c.storedSections*c.sectionSize-1) {
-			return
-		}
-		c.setValidSections(c.storedSections - 1)
-	}
-}
-
-// Sections returns the number of processed sections maintained by the indexer
-// and also the information about the last header indexed for potential canonical
-// verifications.
-func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	c.verifyLastHead()
-	return c.storedSections, c.storedSections*c.sectionSize - 1, c.SectionHead(c.storedSections - 1)
-}
-
-// AddChildIndexer adds a child ChainIndexer that can use the output of this one
-func (c *ChainIndexer) AddChildIndexer(indexer *ChainIndexer) {
-	if indexer == c {
-		panic("can't add indexer as a child of itself")
-	}
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	c.children = append(c.children, indexer)
-
-	// Cascade any pending updates to new children too
-	sections := c.storedSections
-	if c.knownSections < sections {
-		// if a section is "stored" but not "known" then it is a checkpoint without
-		// available chain data so we should not cascade it yet
-		sections = c.knownSections
-	}
-	if sections > 0 {
-		indexer.newHead(sections*c.sectionSize-1, false)
-	}
-}
-
-// Prune deletes all chain data older than given threshold.
-func (c *ChainIndexer) Prune(threshold uint64) error {
-	return c.backend.Prune(threshold)
-}
-
-// loadValidSections reads the number of valid sections from the index database
-// and caches is into the local state.
-func (c *ChainIndexer) loadValidSections() {
-	data, _ := c.indexDb.Get([]byte("count"))
-	if len(data) == 8 {
-		c.storedSections = binary.BigEndian.Uint64(data)
-	}
-}
-
-// setValidSections writes the number of valid sections to the index database
-func (c *ChainIndexer) setValidSections(sections uint64) {
-	// Set the current number of valid sections in the database
-	var data [8]byte
-	binary.BigEndian.PutUint64(data[:], sections)
-	c.indexDb.Put([]byte("count"), data[:])
-
-	// Remove any reorged sections, caching the valids in the mean time
-	for c.storedSections > sections {
-		c.storedSections--
-		c.removeSectionHead(c.storedSections)
-	}
-	c.storedSections = sections // needed if new > old
-}
-
-// SectionHead retrieves the last block hash of a processed section from the
-// index database.
-func (c *ChainIndexer) SectionHead(section uint64) common.Hash {
-	var data [8]byte
-	binary.BigEndian.PutUint64(data[:], section)
-
-	hash, _ := c.indexDb.Get(append([]byte("shead"), data[:]...))
-	if len(hash) == len(common.Hash{}) {
-		return common.BytesToHash(hash)
-	}
-	return common.Hash{}
-}
-
-// setSectionHead writes the last block hash of a processed section to the index
-// database.
-func (c *ChainIndexer) setSectionHead(section uint64, hash common.Hash) {
-	var data [8]byte
-	binary.BigEndian.PutUint64(data[:], section)
-
-	c.indexDb.Put(append([]byte("shead"), data[:]...), hash.Bytes())
-}
-
-// removeSectionHead removes the reference to a processed section from the index
-// database.
-func (c *ChainIndexer) removeSectionHead(section uint64) {
-	var data [8]byte
-	binary.BigEndian.PutUint64(data[:], section)
-
-	c.indexDb.Delete(append([]byte("shead"), data[:]...))
-}

core/chain_indexer_test.go

@@ -1,246 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package core
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"math/big"
-	"math/rand"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/core/types"
-)
-
-// Runs multiple tests with randomized parameters.
-func TestChainIndexerSingle(t *testing.T) {
-	for i := 0; i < 10; i++ {
-		testChainIndexer(t, 1)
-	}
-}
-
-// Runs multiple tests with randomized parameters and different number of
-// chain backends.
-func TestChainIndexerWithChildren(t *testing.T) {
-	for i := 2; i < 8; i++ {
-		testChainIndexer(t, i)
-	}
-}
-
-// testChainIndexer runs a test with either a single chain indexer or a chain of
-// multiple backends. The section size and required confirmation count parameters
-// are randomized.
-func testChainIndexer(t *testing.T, count int) {
-	db := rawdb.NewMemoryDatabase()
-	defer db.Close()
-
-	// Create a chain of indexers and ensure they all report empty
-	backends := make([]*testChainIndexBackend, count)
-	for i := 0; i < count; i++ {
-		var (
-			sectionSize = uint64(rand.Intn(100) + 1)
-			confirmsReq = uint64(rand.Intn(10))
-		)
-		backends[i] = &testChainIndexBackend{t: t, processCh: make(chan uint64)}
-		backends[i].indexer = NewChainIndexer(db, rawdb.NewTable(db, string([]byte{byte(i)})), backends[i], sectionSize, confirmsReq, 0, fmt.Sprintf("indexer-%d", i))
-
-		if sections, _, _ := backends[i].indexer.Sections(); sections != 0 {
-			t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, 0)
-		}
-		if i > 0 {
-			backends[i-1].indexer.AddChildIndexer(backends[i].indexer)
-		}
-	}
-	defer backends[0].indexer.Close() // parent indexer shuts down children
-	// notify pings the root indexer about a new head or reorg, then expect
-	// processed blocks if a section is processable
-	notify := func(headNum, failNum uint64, reorg bool) {
-		backends[0].indexer.newHead(headNum, reorg)
-		if reorg {
-			for _, backend := range backends {
-				headNum = backend.reorg(headNum)
-				backend.assertSections()
-			}
-			return
-		}
-		var cascade bool
-		for _, backend := range backends {
-			headNum, cascade = backend.assertBlocks(headNum, failNum)
-			if !cascade {
-				break
-			}
-			backend.assertSections()
-		}
-	}
-	// inject inserts a new random canonical header into the database directly
-	inject := func(number uint64) {
-		header := &types.Header{Number: big.NewInt(int64(number)), Extra: big.NewInt(rand.Int63()).Bytes()}
-		if number > 0 {
-			header.ParentHash = rawdb.ReadCanonicalHash(db, number-1)
-		}
-		rawdb.WriteHeader(db, header)
-		rawdb.WriteCanonicalHash(db, header.Hash(), number)
-	}
-	// Start indexer with an already existing chain
-	for i := uint64(0); i <= 100; i++ {
-		inject(i)
-	}
-	notify(100, 100, false)
-
-	// Add new blocks one by one
-	for i := uint64(101); i <= 1000; i++ {
-		inject(i)
-		notify(i, i, false)
-	}
-	// Do a reorg
-	notify(500, 500, true)
-
-	// Create new fork
-	for i := uint64(501); i <= 1000; i++ {
-		inject(i)
-		notify(i, i, false)
-	}
-	for i := uint64(1001); i <= 1500; i++ {
-		inject(i)
-	}
-	// Failed processing scenario where less blocks are available than notified
-	notify(2000, 1500, false)
-
-	// Notify about a reorg (which could have caused the missing blocks if happened during processing)
-	notify(1500, 1500, true)
-
-	// Create new fork
-	for i := uint64(1501); i <= 2000; i++ {
-		inject(i)
-		notify(i, i, false)
-	}
-}
-
-// testChainIndexBackend implements ChainIndexerBackend
-type testChainIndexBackend struct {
-	t                          *testing.T
-	indexer                    *ChainIndexer
-	section, headerCnt, stored uint64
-	processCh                  chan uint64
-}
-
-// assertSections verifies if a chain indexer has the correct number of section.
-func (b *testChainIndexBackend) assertSections() {
-	// Keep trying for 3 seconds if it does not match
-	var sections uint64
-	for i := 0; i < 300; i++ {
-		sections, _, _ = b.indexer.Sections()
-		if sections == b.stored {
-			return
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	b.t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, b.stored)
-}
-
-// assertBlocks expects processing calls after new blocks have arrived. If the
-// failNum < headNum then we are simulating a scenario where a reorg has happened
-// after the processing has started and the processing of a section fails.
-func (b *testChainIndexBackend) assertBlocks(headNum, failNum uint64) (uint64, bool) {
-	var sections uint64
-	if headNum >= b.indexer.confirmsReq {
-		sections = (headNum + 1 - b.indexer.confirmsReq) / b.indexer.sectionSize
-		if sections > b.stored {
-			// expect processed blocks
-			for expectd := b.stored * b.indexer.sectionSize; expectd < sections*b.indexer.sectionSize; expectd++ {
-				if expectd > failNum {
-					// rolled back after processing started, no more process calls expected
-					// wait until updating is done to make sure that processing actually fails
-					var updating bool
-					for i := 0; i < 300; i++ {
-						b.indexer.lock.Lock()
-						updating = b.indexer.knownSections > b.indexer.storedSections
-						b.indexer.lock.Unlock()
-						if !updating {
-							break
-						}
-						time.Sleep(10 * time.Millisecond)
-					}
-					if updating {
-						b.t.Fatalf("update did not finish")
-					}
-					sections = expectd / b.indexer.sectionSize
-					break
-				}
-				select {
-				case <-time.After(10 * time.Second):
-					b.t.Fatalf("Expected processed block #%d, got nothing", expectd)
-				case processed := <-b.processCh:
-					if processed != expectd {
-						b.t.Errorf("Expected processed block #%d, got #%d", expectd, processed)
-					}
-				}
-			}
-			b.stored = sections
-		}
-	}
-	if b.stored == 0 {
-		return 0, false
-	}
-	return b.stored*b.indexer.sectionSize - 1, true
-}
-
-func (b *testChainIndexBackend) reorg(headNum uint64) uint64 {
-	firstChanged := (headNum + 1) / b.indexer.sectionSize
-	if firstChanged < b.stored {
-		b.stored = firstChanged
-	}
-	return b.stored * b.indexer.sectionSize
-}
-
-func (b *testChainIndexBackend) Reset(ctx context.Context, section uint64, prevHead common.Hash) error {
-	b.section = section
-	b.headerCnt = 0
-	return nil
-}
-
-func (b *testChainIndexBackend) Process(ctx context.Context, header *types.Header) error {
-	b.headerCnt++
-	if b.headerCnt > b.indexer.sectionSize {
-		b.t.Error("Processing too many headers")
-	}
-	//t.processCh <- header.Number.Uint64()
-	select {
-	case <-time.After(10 * time.Second):
-		b.t.Error("Unexpected call to Process")
-		// Can't use Fatal since this is not the test's goroutine.
-		// Returning error stops the chainIndexer's updateLoop
-		return errors.New("unexpected call to Process")
-	case b.processCh <- header.Number.Uint64():
-	}
-	return nil
-}
-
-func (b *testChainIndexBackend) Commit() error {
-	if b.headerCnt != b.indexer.sectionSize {
-		b.t.Error("Not enough headers processed")
-	}
-	return nil
-}
-
-func (b *testChainIndexBackend) Prune(threshold uint64) error {
-	return nil
-}

core/chain_makers.go

@@ -328,9 +328,13 @@ func (b *BlockGen) collectRequests(readonly bool) (requests [][]byte) {
 		blockContext := NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase)
 		evm := vm.NewEVM(blockContext, statedb, b.cm.config, vm.Config{})
 		// EIP-7002
-		ProcessWithdrawalQueue(&requests, evm)
+		if err := ProcessWithdrawalQueue(&requests, evm); err != nil {
+			panic(fmt.Sprintf("could not process withdrawal requests: %v", err))
+		}
 		// EIP-7251
-		ProcessConsolidationQueue(&requests, evm)
+		if err := ProcessConsolidationQueue(&requests, evm); err != nil {
+			panic(fmt.Sprintf("could not process consolidation requests: %v", err))
+		}
 	}
 	return requests
 }
@@ -504,6 +508,13 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
 		if gen != nil {
 			gen(i, b)
 		}
+
+		requests := b.collectRequests(false)
+		if requests != nil {
+			reqHash := types.CalcRequestsHash(requests)
+			b.header.RequestsHash = &reqHash
+		}
+
 		body := &types.Body{
 			Transactions: b.txs,
 			Uncles:       b.uncles,

core/error.go

@@ -28,8 +28,6 @@ var (
 
 	// ErrNoGenesis is returned when there is no Genesis Block.
 	ErrNoGenesis = errors.New("genesis not found in chain")
-
-	errSideChainReceipts = errors.New("side blocks can't be accepted as ancient chain data")
 )
 
 // List of evm-call-message pre-checking errors. All state transition messages will

core/filtermaps/chain_view.go

@@ -17,6 +17,8 @@
 package filtermaps
 
 import (
+	"sync"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
@@ -27,6 +29,7 @@ type blockchain interface {
 	GetHeader(hash common.Hash, number uint64) *types.Header
 	GetCanonicalHash(number uint64) common.Hash
 	GetReceiptsByHash(hash common.Hash) types.Receipts
+	GetRawReceipts(hash common.Hash, number uint64) types.Receipts
 }
 
 // ChainView represents an immutable view of a chain with a block id and a set
@@ -39,6 +42,7 @@ type blockchain interface {
 // of the underlying blockchain, it should only possess the block headers
 // and receipts up until the expected chain view head.
 type ChainView struct {
+	lock       sync.Mutex
 	chain      blockchain
 	headNumber uint64
 	hashes     []common.Hash // block hashes starting backwards from headNumber until first canonical hash
@@ -55,43 +59,80 @@ func NewChainView(chain blockchain, number uint64, hash common.Hash) *ChainView
 	return cv
 }
 
-// getBlockHash returns the block hash belonging to the given block number.
+// HeadNumber returns the head block number of the chain view.
+func (cv *ChainView) HeadNumber() uint64 {
+	return cv.headNumber
+}
+
+// BlockHash returns the block hash belonging to the given block number.
 // Note that the hash of the head block is not returned because ChainView might
 // represent a view where the head block is currently being created.
-func (cv *ChainView) getBlockHash(number uint64) common.Hash {
+func (cv *ChainView) BlockHash(number uint64) common.Hash {
-	if number >= cv.headNumber {
+	cv.lock.Lock()
+	defer cv.lock.Unlock()
+
+	if number > cv.headNumber {
 		panic("invalid block number")
 	}
 	return cv.blockHash(number)
 }
 
-// getBlockId returns the unique block id belonging to the given block number.
+// BlockId returns the unique block id belonging to the given block number.
 // Note that it is currently equal to the block hash. In the future it might
 // be a different id for future blocks if the log index root becomes part of
 // consensus and therefore rendering the index with the new head will happen
 // before the hash of that new head is available.
-func (cv *ChainView) getBlockId(number uint64) common.Hash {
+func (cv *ChainView) BlockId(number uint64) common.Hash {
+	cv.lock.Lock()
+	defer cv.lock.Unlock()
+
 	if number > cv.headNumber {
 		panic("invalid block number")
 	}
 	return cv.blockHash(number)
 }
 
-// getReceipts returns the set of receipts belonging to the block at the given
+// Header returns the block header at the given block number.
-// block number.
+func (cv *ChainView) Header(number uint64) *types.Header {
-func (cv *ChainView) getReceipts(number uint64) types.Receipts {
+	return cv.chain.GetHeader(cv.BlockHash(number), number)
-	if number > cv.headNumber {
-		panic("invalid block number")
-	}
-	return cv.chain.GetReceiptsByHash(cv.blockHash(number))
 }
 
-// limitedView returns a new chain view that is a truncated version of the parent view.
+// Receipts returns the set of receipts belonging to the block at the given
-func (cv *ChainView) limitedView(newHead uint64) *ChainView {
+// block number.
-	if newHead >= cv.headNumber {
+func (cv *ChainView) Receipts(number uint64) types.Receipts {
-		return cv
+	blockHash := cv.BlockHash(number)
+	if blockHash == (common.Hash{}) {
+		log.Error("Chain view: block hash unavailable", "number", number, "head", cv.headNumber)
+		return nil
 	}
-	return NewChainView(cv.chain, newHead, cv.blockHash(newHead))
+	return cv.chain.GetReceiptsByHash(blockHash)
+}
+
+// RawReceipts returns the set of receipts belonging to the block at the given
+// block number. Does not derive the fields of the receipts, should only be
+// used during creation of the filter maps, please use cv.Receipts during querying.
+func (cv *ChainView) RawReceipts(number uint64) types.Receipts {
+	blockHash := cv.BlockHash(number)
+	if blockHash == (common.Hash{}) {
+		log.Error("Chain view: block hash unavailable", "number", number, "head", cv.headNumber)
+		return nil
+	}
+	return cv.chain.GetRawReceipts(blockHash, number)
+}
+
+// SharedRange returns the block range shared by two chain views.
+func (cv *ChainView) SharedRange(cv2 *ChainView) common.Range[uint64] {
+	cv.lock.Lock()
+	defer cv.lock.Unlock()
+
+	if cv == nil || cv2 == nil || !cv.extendNonCanonical() || !cv2.extendNonCanonical() {
+		return common.Range[uint64]{}
+	}
+	var sharedLen uint64
+	for n := min(cv.headNumber+1-uint64(len(cv.hashes)), cv2.headNumber+1-uint64(len(cv2.hashes))); n <= cv.headNumber && n <= cv2.headNumber && cv.blockHash(n) == cv2.blockHash(n); n++ {
+		sharedLen = n + 1
+	}
+	return common.NewRange(0, sharedLen)
 }
 
 // equalViews returns true if the two chain views are equivalent.
@@ -99,7 +140,7 @@ func equalViews(cv1, cv2 *ChainView) bool {
 	if cv1 == nil || cv2 == nil {
 		return false
 	}
-	return cv1.headNumber == cv2.headNumber && cv1.getBlockId(cv1.headNumber) == cv2.getBlockId(cv2.headNumber)
+	return cv1.headNumber == cv2.headNumber && cv1.BlockId(cv1.headNumber) == cv2.BlockId(cv2.headNumber)
 }
 
 // matchViews returns true if the two chain views are equivalent up until the
@@ -113,9 +154,9 @@ func matchViews(cv1, cv2 *ChainView, number uint64) bool {
 		return false
 	}
 	if number == cv1.headNumber || number == cv2.headNumber {
-		return cv1.getBlockId(number) == cv2.getBlockId(number)
+		return cv1.BlockId(number) == cv2.BlockId(number)
 	}
-	return cv1.getBlockHash(number) == cv2.getBlockHash(number)
+	return cv1.BlockHash(number) == cv2.BlockHash(number)
 }
 
 // extendNonCanonical checks whether the previously known reverse list of head

core/filtermaps/checkpoints_holesky.json

@@ -17,5 +17,6 @@
 {"blockNumber": 3101669, "blockId": "0xa6e66a18fed64d33178c7088f273c404be597662c34f6e6884cf2e24f3ea4ace", "firstIndex": 1073741353},
 {"blockNumber": 3221725, "blockId": "0xe771f897dece48b1583cc1d1d10de8015da57407eb1fdf239fdbe46eaab85143", "firstIndex": 1140850137},
 {"blockNumber": 3357164, "blockId": "0x6252d0aa54c79623b0680069c88d7b5c47983f0d5c4845b6c811b8d9b5e8ff3c", "firstIndex": 1207959453},
-{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542}
+{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542},
+{"blockNumber": 3546397, "blockId": "0xdabdef7defa4281180a57c5af121877b82274f15ccf074ea0096146f4c246df2", "firstIndex": 1342176778}
 ]

core/filtermaps/checkpoints_mainnet.json

@@ -260,5 +260,12 @@
 {"blockNumber": 21959188, "blockId": "0x6b9c482cc4822af2ad62a359b923062556ab6a046d1a39a8243b764848690601", "firstIndex": 17381193886},
 {"blockNumber": 21994911, "blockId": "0x0d99ef9dd42dd1c62fc5249eb4f618e7672ab93fa0ba7545c77360371cf972e5", "firstIndex": 17448302795},
 {"blockNumber": 22026007, "blockId": "0xf7cdc7f6694f2c2ed31fa8a607f65cfa59d0dd7d7424ab5c017f959ae2982c71", "firstIndex": 17515413036},
-{"blockNumber": 22059890, "blockId": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768}
+{"blockNumber": 22059890, "blockId": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768},
+{"blockNumber": 22090784, "blockId": "0xf97c2eaf9a550360ac24000c0ff17ffa388a2bdd6f73f2f36718e332edfa107a", "firstIndex": 17649630983},
+{"blockNumber": 22121157, "blockId": "0xa790025235db782e899f23d8b09663ec2d74ec149e4125d62989f98829b08e2d", "firstIndex": 17716724973},
+{"blockNumber": 22148056, "blockId": "0xbe25ac4f1bdd89a7db5782bf1157e6c4378d80220d67a029397853ef16cd1a4c", "firstIndex": 17783838366},
+{"blockNumber": 22168652, "blockId": "0x6ae43618c915e636794e2cc2d75dde9992766881c7405fe6479c045ed4bee57e", "firstIndex": 17850956277},
+{"blockNumber": 22190975, "blockId": "0x9437121647899a4b7b84d67fbea7cc6ff967481c2eab4328ccd86e2cefe19420", "firstIndex": 17918066140},
+{"blockNumber": 22234357, "blockId": "0x036030830134f9224160d5a0b62da35ec7813dc8855d554bd22e9d38545243ed", "firstIndex": 17985175075},
+{"blockNumber": 22276736, "blockId": "0x5ceb96d98aa1b4c1c2f2fa253ae9cdb1b04e0420c11bf795400e8762c0a1635c", "firstIndex": 18052284344}
 ]

core/filtermaps/checkpoints_sepolia.json

@@ -64,5 +64,9 @@
 {"blockNumber": 7730079, "blockId": "0x08e0d511a93e2a9c004b3456d77d687ae86247417ada1cdd1f85332a62dfed1d", "firstIndex": 4227846795},
 {"blockNumber": 7777515, "blockId": "0xeba8faed2e12bb9ed5e7fad19dd82662f15f6f4f512a0d232eedf1e676712428", "firstIndex": 4294966082},
 {"blockNumber": 7828860, "blockId": "0x62c054bd24be351dc4777460ee922a75fdcea5c9830455cbac6fa29552719c85", "firstIndex": 4362076087},
-{"blockNumber": 7869890, "blockId": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267}
+{"blockNumber": 7869890, "blockId": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267},
+{"blockNumber": 7911722, "blockId": "0x9a85e48e3135c97c51fc1786f2af0596c802e021b6c53cfca65a129cafcd23ed", "firstIndex": 4496287265},
+{"blockNumber": 7960147, "blockId": "0xc9359cc76d7090e1c8a031108f0ab7a8935d971efd4325fe53612a1d99562f6f", "firstIndex": 4563402388},
+{"blockNumber": 8030418, "blockId": "0x21867e68cd8327aed2da2601399d60f7f9e41dca4a4f2f9be982e5a2b9304a88", "firstIndex": 4630511616},
+{"blockNumber": 8087701, "blockId": "0x0fa8c8d7549cc9a8d308262706fe248efe759f8b63511efb1e7f3926e9af2dcb", "firstIndex": 4697614758}
 ]

core/filtermaps/filtermaps.go

@@ -17,7 +17,6 @@
 package filtermaps
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"os"
@@ -30,14 +29,30 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/ethdb/leveldb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+)
+
+var (
+	mapCountGauge           = metrics.NewRegisteredGauge("filtermaps/maps/count", nil)          // actual number of rendered maps
+	mapLogValueMeter        = metrics.NewRegisteredMeter("filtermaps/maps/logvalues", nil)      // number of log values processed
+	mapBlockMeter           = metrics.NewRegisteredMeter("filtermaps/maps/blocks", nil)         // number of block delimiters processed
+	mapRenderTimer          = metrics.NewRegisteredTimer("filtermaps/maps/rendertime", nil)     // time elapsed while rendering a single map
+	mapWriteTimer           = metrics.NewRegisteredTimer("filtermaps/maps/writetime", nil)      // time elapsed while writing a batch of finished maps to db
+	matchRequestTimer       = metrics.NewRegisteredTimer("filtermaps/match/requesttime", nil)   // processing time a matching request in a single epoch
+	matchEpochTimer         = metrics.NewRegisteredTimer("filtermaps/match/epochtime", nil)     // total processing time a matching request
+	matchBaseRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowaccess", nil) // number of accessed rows on layer 0
+	matchBaseRowSizeMeter   = metrics.NewRegisteredMeter("filtermaps/match/baserowsize", nil)   // size of accessed rows on layer 0
+	matchExtRowAccessMeter  = metrics.NewRegisteredMeter("filtermaps/match/extrowaccess", nil)  // number of accessed rows on higher layers
+	matchExtRowSizeMeter    = metrics.NewRegisteredMeter("filtermaps/match/extrowsize", nil)    // size of accessed rows on higher layers
+	matchLogLookup          = metrics.NewRegisteredMeter("filtermaps/match/loglookup", nil)     // number of log lookups based on potential matches
+	matchAllMeter           = metrics.NewRegisteredMeter("filtermaps/match/matchall", nil)      // number of requests returned with ErrMatchAll
 )
 
 const (
+	databaseVersion       = 2    // reindexed if database version does not match
 	cachedLastBlocks      = 1000 // last block of map pointers
 	cachedLvPointers      = 1000 // first log value pointer of block pointers
-	cachedBaseRows        = 100  // groups of base layer filter row data
 	cachedFilterMaps      = 3    // complete filter maps (cached by map renderer)
 	cachedRenderSnapshots = 8    // saved map renderer data at block boundaries
 )
@@ -55,10 +70,12 @@ type FilterMaps struct {
 	// We chose to implement disabling this way because it requires less special
 	// case logic in eth/filters.
 	disabled   bool
+	disabledCh chan struct{} // closed by indexer if disabled
 
 	closeCh        chan struct{}
 	closeWg        sync.WaitGroup
 	history        uint64
+	hashScheme     bool // use hashdb-safe delete range method
 	exportFileName string
 	Params
 
@@ -70,12 +87,19 @@ type FilterMaps struct {
 	indexLock    sync.RWMutex
 	indexedRange filterMapsRange
 	indexedView  *ChainView // always consistent with the log index
+	hasTempRange bool
+
+	// cleanedEpochsBefore indicates that all unindexed data before this point
+	// has been cleaned.
+	//
+	// This field is only accessed and modified within tryUnindexTail, so no
+	// explicit locking is required.
+	cleanedEpochsBefore uint32
 
 	// also accessed by indexer and matcher backend but no locking needed.
 	filterMapCache *lru.Cache[uint32, filterMap]
 	lastBlockCache *lru.Cache[uint32, lastBlockOfMap]
 	lvPointerCache *lru.Cache[uint64, uint64]
-	baseRowsCache  *lru.Cache[uint64, [][]uint32]
 
 	// the matchers set and the fields of FilterMapsMatcherBackend instances are
 	// read and written both by exported functions and the indexer.
@@ -94,7 +118,7 @@ type FilterMaps struct {
 	ptrTailUnindexMap                                            uint32
 
 	targetView            *ChainView
-	matcherSyncRequest    *FilterMapsMatcherBackend
+	matcherSyncRequests   []*FilterMapsMatcherBackend
 	historyCutoff         uint64
 	finalBlock, lastFinal uint64
 	lastFinalEpoch        uint32
@@ -108,6 +132,7 @@ type FilterMaps struct {
 
 	// test hooks
 	testDisableSnapshots, testSnapshotUsed bool
+	testProcessEventsHook                  func()
 }
 
 // filterMap is a full or partial in-memory representation of a filter map where
@@ -119,13 +144,25 @@ type FilterMaps struct {
 // as transparent (uncached/unchanged).
 type filterMap []FilterRow
 
-// copy returns a copy of the given filter map. Note that the row slices are
+// fastCopy returns a copy of the given filter map. Note that the row slices are
-// copied but their contents are not. This permits extending the rows further
+// copied but their contents are not. This permits appending to the rows further
 // (which happens during map rendering) without affecting the validity of
 // copies made for snapshots during rendering.
-func (fm filterMap) copy() filterMap {
+// Appending to the rows of both the original map and the fast copy, or two fast
+// copies of the same map would result in data corruption, therefore a fast copy
+// should always be used in a read only way.
+func (fm filterMap) fastCopy() filterMap {
+	return slices.Clone(fm)
+}
+
+// fullCopy returns a copy of the given filter map, also making a copy of each
+// individual filter row, ensuring that a modification to either one will never
+// affect the other.
+func (fm filterMap) fullCopy() filterMap {
 	c := make(filterMap, len(fm))
-	copy(c, fm)
+	for i, row := range fm {
+		c[i] = slices.Clone(row)
+	}
 	return c
 }
 
@@ -179,13 +216,18 @@ type Config struct {
 	// This option enables the checkpoint JSON file generator.
 	// If set, the given file will be updated with checkpoint information.
 	ExportFileName string
+
+	// expect trie nodes of hash based state scheme in the filtermaps key range;
+	// use safe iterator based implementation of DeleteRange that skips them
+	HashScheme bool
 }
 
 // NewFilterMaps creates a new FilterMaps and starts the indexer.
 func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, finalBlock uint64, params Params, config Config) *FilterMaps {
 	rs, initialized, err := rawdb.ReadFilterMapsRange(db)
-	if err != nil {
+	if err != nil || (initialized && rs.Version != databaseVersion) {
-		log.Error("Error reading log index range", "error", err)
+		rs, initialized = rawdb.FilterMapsRange{}, false
+		log.Warn("Invalid log index database version; resetting log index")
 	}
 	params.deriveFields()
 	f := &FilterMaps{
@@ -196,8 +238,12 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f
 		blockProcessingCh: make(chan bool, 1),
 		history:           config.History,
 		disabled:          config.Disabled,
+		hashScheme:        config.HashScheme,
+		disabledCh:        make(chan struct{}),
 		exportFileName:    config.ExportFileName,
 		Params:            params,
+		targetView:        initView,
+		indexedView:       initView,
 		indexedRange: filterMapsRange{
 			initialized:      initialized,
 			headIndexed:      rs.HeadIndexed,
@@ -206,24 +252,20 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f
 			maps:             common.NewRange(rs.MapsFirst, rs.MapsAfterLast-rs.MapsFirst),
 			tailPartialEpoch: rs.TailPartialEpoch,
 		},
+		// deleting last unindexed epoch might have been interrupted by shutdown
+		cleanedEpochsBefore: max(rs.MapsFirst>>params.logMapsPerEpoch, 1) - 1,
+
+		historyCutoff:   historyCutoff,
+		finalBlock:      finalBlock,
 		matcherSyncCh:   make(chan *FilterMapsMatcherBackend),
 		matchers:        make(map[*FilterMapsMatcherBackend]struct{}),
 		filterMapCache:  lru.NewCache[uint32, filterMap](cachedFilterMaps),
 		lastBlockCache:  lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks),
 		lvPointerCache:  lru.NewCache[uint64, uint64](cachedLvPointers),
-		baseRowsCache:   lru.NewCache[uint64, [][]uint32](cachedBaseRows),
 		renderSnapshots: lru.NewCache[uint64, *renderedMap](cachedRenderSnapshots),
 	}
+	f.checkRevertRange() // revert maps that are inconsistent with the current chain view
 
-	// Set initial indexer target.
-	f.targetView = initView
-	if f.indexedRange.initialized {
-		f.indexedView = f.initChainView(f.targetView)
-		f.indexedRange.headIndexed = f.indexedRange.blocks.AfterLast() == f.indexedView.headNumber+1
-		if !f.indexedRange.headIndexed {
-			f.indexedRange.headDelimiter = 0
-		}
-	}
 	if f.indexedRange.hasIndexedBlocks() {
 		log.Info("Initialized log indexer",
 			"first block", f.indexedRange.blocks.First(), "last block", f.indexedRange.blocks.Last(),
@@ -241,7 +283,8 @@ func (f *FilterMaps) Start() {
 			log.Error("Could not load head filter map snapshot", "error", err)
 		}
 	}
-	f.closeWg.Add(1)
+	f.closeWg.Add(2)
+	go f.removeBloomBits()
 	go f.indexerLoop()
 }
 
@@ -251,34 +294,50 @@ func (f *FilterMaps) Stop() {
 	f.closeWg.Wait()
 }
 
-// initChainView returns a chain view consistent with both the current target
+// checkRevertRange checks whether the existing index is consistent with the
-// view and the current state of the log index as found in the database, based
+// current indexed view and reverts inconsistent maps if necessary.
-// on the last block of stored maps.
+func (f *FilterMaps) checkRevertRange() {
-// Note that the returned view might be shorter than the existing index if
+	if f.indexedRange.maps.Count() == 0 {
-// the latest maps are not consistent with targetView.
+		return
-func (f *FilterMaps) initChainView(chainView *ChainView) *ChainView {
-	mapIndex := f.indexedRange.maps.AfterLast()
-	for {
-		var ok bool
-		mapIndex, ok = f.lastMapBoundaryBefore(mapIndex)
-		if !ok {
-			break
 	}
-		lastBlockNumber, lastBlockId, err := f.getLastBlockOfMap(mapIndex)
+	lastMap := f.indexedRange.maps.Last()
+	lastBlockNumber, lastBlockId, err := f.getLastBlockOfMap(lastMap)
 	if err != nil {
-			log.Error("Could not initialize indexed chain view", "error", err)
+		log.Error("Error initializing log index database; resetting log index", "error", err)
-			break
+		f.reset()
+		return
 	}
-		if lastBlockNumber <= chainView.headNumber && chainView.getBlockId(lastBlockNumber) == lastBlockId {
+	for lastBlockNumber > f.indexedView.HeadNumber() || f.indexedView.BlockId(lastBlockNumber) != lastBlockId {
-			return chainView.limitedView(lastBlockNumber)
+		// revert last map
+		if f.indexedRange.maps.Count() == 1 {
+			f.reset() // reset database if no rendered maps remained
+			return
 		}
+		lastMap--
+		newRange := f.indexedRange
+		newRange.maps.SetLast(lastMap)
+		lastBlockNumber, lastBlockId, err = f.getLastBlockOfMap(lastMap)
+		if err != nil {
+			log.Error("Error initializing log index database; resetting log index", "error", err)
+			f.reset()
+			return
+		}
+		newRange.blocks.SetAfterLast(lastBlockNumber) // lastBlockNumber is probably partially indexed
+		newRange.headIndexed = false
+		newRange.headDelimiter = 0
+		// only shorten range and leave map data; next head render will overwrite it
+		f.setRange(f.db, f.indexedView, newRange, false)
 	}
-	return chainView.limitedView(0)
 }
 
 // reset un-initializes the FilterMaps structure and removes all related data from
-// the database. The function returns true if everything was successfully removed.
+// the database.
-func (f *FilterMaps) reset() bool {
+// Note that in case of leveldb database the fallback implementation of DeleteRange
+// might take a long time to finish and deleting the entire database may be
+// interrupted by a shutdown. Deleting the filterMapsRange entry first does
+// guarantee though that the next init() will not return successfully until the
+// entire database has been cleaned.
+func (f *FilterMaps) reset() {
 	f.indexLock.Lock()
 	f.indexedRange = filterMapsRange{}
 	f.indexedView = nil
@@ -286,16 +345,30 @@ func (f *FilterMaps) reset() bool {
 	f.renderSnapshots.Purge()
 	f.lastBlockCache.Purge()
 	f.lvPointerCache.Purge()
-	f.baseRowsCache.Purge()
 	f.indexLock.Unlock()
 	// deleting the range first ensures that resetDb will be called again at next
 	// startup and any leftover data will be removed even if it cannot finish now.
 	rawdb.DeleteFilterMapsRange(f.db)
-	return f.removeDbWithPrefix([]byte(rawdb.FilterMapsPrefix), "Resetting log index database")
+	f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown)
+}
+
+// isShuttingDown returns true if FilterMaps is shutting down.
+func (f *FilterMaps) isShuttingDown() bool {
+	select {
+	case <-f.closeCh:
+		return true
+	default:
+		return false
+	}
 }
 
 // init initializes an empty log index according to the current targetView.
 func (f *FilterMaps) init() error {
+	// ensure that there is no remaining data in the filter maps key range
+	if err := f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown); err != nil {
+		return err
+	}
+
 	f.indexLock.Lock()
 	defer f.indexLock.Unlock()
 
@@ -306,7 +379,7 @@ func (f *FilterMaps) init() error {
 		for min < max {
 			mid := (min + max + 1) / 2
 			cp := checkpointList[mid-1]
-			if cp.BlockNumber <= f.targetView.headNumber && f.targetView.getBlockId(cp.BlockNumber) == cp.BlockId {
+			if cp.BlockNumber <= f.targetView.HeadNumber() && f.targetView.BlockId(cp.BlockNumber) == cp.BlockId {
 				min = mid
 			} else {
 				max = mid - 1
@@ -316,6 +389,13 @@ func (f *FilterMaps) init() error {
 			bestIdx, bestLen = idx, max
 		}
 	}
+	var initBlockNumber uint64
+	if bestLen > 0 {
+		initBlockNumber = checkpoints[bestIdx][bestLen-1].BlockNumber
+	}
+	if initBlockNumber < f.historyCutoff {
+		return errors.New("cannot start indexing before history cutoff point")
+	}
 	batch := f.db.NewBatch()
 	for epoch := range bestLen {
 		cp := checkpoints[bestIdx][epoch]
@@ -330,55 +410,58 @@ func (f *FilterMaps) init() error {
 		fmr.blocks = common.NewRange(cp.BlockNumber+1, 0)
 		fmr.maps = common.NewRange(uint32(bestLen)<<f.logMapsPerEpoch, 0)
 	}
-	f.setRange(batch, f.targetView, fmr)
+	f.setRange(batch, f.targetView, fmr, false)
 	return batch.Write()
 }
 
-// removeDbWithPrefix removes data with the given prefix from the database and
+// removeBloomBits removes old bloom bits data from the database.
-// returns true if everything was successfully removed.
+func (f *FilterMaps) removeBloomBits() {
-func (f *FilterMaps) removeDbWithPrefix(prefix []byte, action string) bool {
+	f.safeDeleteWithLogs(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database", f.isShuttingDown)
-	it := f.db.NewIterator(prefix, nil)
+	f.closeWg.Done()
-	hasData := it.Next()
-	it.Release()
-	if !hasData {
-		return true
 }
 
-	end := bytes.Clone(prefix)
+// safeDeleteWithLogs is a wrapper for a function that performs a safe range
-	end[len(end)-1]++
+// delete operation using rawdb.SafeDeleteRange. It emits log messages if the
-	start := time.Now()
+// process takes long enough to call the stop callback.
-	var retry bool
+func (f *FilterMaps) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error, action string, stopCb func() bool) error {
-	for {
+	var (
-		err := f.db.DeleteRange(prefix, end)
+		start          = time.Now()
-		if err == nil {
+		logPrinted     bool
-			log.Info(action+" finished", "elapsed", time.Since(start))
+		lastLogPrinted = start
-			return true
+	)
+	switch err := deleteFn(f.db, f.hashScheme, func(deleted bool) bool {
+		if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 {
+			log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start)))
+			logPrinted, lastLogPrinted = true, time.Now()
 		}
-		if err != leveldb.ErrTooManyKeys {
+		return stopCb()
-			log.Error(action+" failed", "error", err)
+	}); {
-			return false
+	case err == nil:
+		if logPrinted {
+			log.Info(action+" finished", "elapsed", common.PrettyDuration(time.Since(start)))
 		}
-		select {
+		return nil
-		case <-f.closeCh:
+	case errors.Is(err, rawdb.ErrDeleteRangeInterrupted):
-			return false
+		log.Warn(action+" interrupted", "elapsed", common.PrettyDuration(time.Since(start)))
+		return err
 	default:
-		}
+		log.Error(action+" failed", "error", err)
-		if !retry {
+		return err
-			log.Info(action + " in progress...")
-			retry = true
-		}
 	}
 }
 
 // setRange updates the indexed chain view and covered range and also adds the
 // changes to the given batch.
+//
 // Note that this function assumes that the index write lock is being held.
-func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, newRange filterMapsRange) {
+func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, newRange filterMapsRange, isTempRange bool) {
 	f.indexedView = newView
 	f.indexedRange = newRange
+	f.hasTempRange = isTempRange
 	f.updateMatchersValidRange()
 	if newRange.initialized {
 		rs := rawdb.FilterMapsRange{
+			Version:          databaseVersion,
 			HeadIndexed:      newRange.headIndexed,
 			HeadDelimiter:    newRange.headDelimiter,
 			BlocksFirst:      newRange.blocks.First(),
@@ -388,8 +471,12 @@ func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, ne
 			TailPartialEpoch: newRange.tailPartialEpoch,
 		}
 		rawdb.WriteFilterMapsRange(batch, rs)
+		if !isTempRange {
+			mapCountGauge.Update(int64(newRange.maps.Count() + newRange.tailPartialEpoch))
+		}
 	} else {
 		rawdb.DeleteFilterMapsRange(batch)
+		mapCountGauge.Update(0)
 	}
 }
 
@@ -399,6 +486,7 @@ func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, ne
 // Note that this function assumes that the log index structure is consistent
 // with the canonical chain at the point where the given log value index points.
 // If this is not the case then an invalid result or an error may be returned.
+//
 // Note that this function assumes that the indexer read lock is being held when
 // called from outside the indexerLoop goroutine.
 func (f *FilterMaps) getLogByLvIndex(lvIndex uint64) (*types.Log, error) {
@@ -435,7 +523,7 @@ func (f *FilterMaps) getLogByLvIndex(lvIndex uint64) (*types.Log, error) {
 		}
 	}
 	// get block receipts
-	receipts := f.indexedView.getReceipts(firstBlockNumber)
+	receipts := f.indexedView.Receipts(firstBlockNumber)
 	if receipts == nil {
 		return nil, fmt.Errorf("failed to retrieve receipts for block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err)
 	}
@@ -473,47 +561,69 @@ func (f *FilterMaps) getFilterMap(mapIndex uint32) (filterMap, error) {
 	}
 	fm := make(filterMap, f.mapHeight)
 	for rowIndex := range fm {
-		var err error
+		rows, err := f.getFilterMapRows([]uint32{mapIndex}, uint32(rowIndex), false)
-		fm[rowIndex], err = f.getFilterMapRow(mapIndex, uint32(rowIndex), false)
 		if err != nil {
 			return nil, fmt.Errorf("failed to load filter map %d from database: %v", mapIndex, err)
 		}
+		fm[rowIndex] = rows[0]
 	}
 	f.filterMapCache.Add(mapIndex, fm)
 	return fm, nil
 }
 
-// getFilterMapRow fetches the given filter map row. If baseLayerOnly is true
+// getFilterMapRows fetches a set of filter map rows at the corresponding map
-// then only the first baseRowLength entries are returned.
+// indices and a shared row index. If baseLayerOnly is true then only the first
-func (f *FilterMaps) getFilterMapRow(mapIndex, rowIndex uint32, baseLayerOnly bool) (FilterRow, error) {
+// baseRowLength entries are returned.
-	baseMapRowIndex := f.mapRowIndex(mapIndex&-f.baseRowGroupLength, rowIndex)
+func (f *FilterMaps) getFilterMapRows(mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) ([]FilterRow, error) {
-	baseRows, ok := f.baseRowsCache.Get(baseMapRowIndex)
+	rows := make([]FilterRow, len(mapIndices))
-	if !ok {
+	var ptr int
-		var err error
+	for len(mapIndices) > ptr {
-		baseRows, err = rawdb.ReadFilterMapBaseRows(f.db, baseMapRowIndex, f.baseRowGroupLength, f.logMapWidth)
+		baseRowGroup := mapIndices[ptr] / f.baseRowGroupLength
+		groupLength := 1
+		for ptr+groupLength < len(mapIndices) && mapIndices[ptr+groupLength]/f.baseRowGroupLength == baseRowGroup {
+			groupLength++
+		}
+		if err := f.getFilterMapRowsOfGroup(rows[ptr:ptr+groupLength], mapIndices[ptr:ptr+groupLength], rowIndex, baseLayerOnly); err != nil {
+			return nil, err
+		}
+		ptr += groupLength
+	}
+	return rows, nil
+}
+
+// getFilterMapRowsOfGroup fetches a set of filter map rows at map indices
+// belonging to the same base row group.
+func (f *FilterMaps) getFilterMapRowsOfGroup(target []FilterRow, mapIndices []uint32, rowIndex uint32, baseLayerOnly bool) error {
+	baseRowGroup := mapIndices[0] / f.baseRowGroupLength
+	baseMapRowIndex := f.mapRowIndex(baseRowGroup*f.baseRowGroupLength, rowIndex)
+	baseRows, err := rawdb.ReadFilterMapBaseRows(f.db, baseMapRowIndex, f.baseRowGroupLength, f.logMapWidth)
 	if err != nil {
-			return nil, fmt.Errorf("failed to retrieve filter map %d base rows %d: %v", mapIndex, rowIndex, err)
+		return fmt.Errorf("failed to retrieve base row group %d of row %d: %v", baseRowGroup, rowIndex, err)
 	}
-		f.baseRowsCache.Add(baseMapRowIndex, baseRows)
+	for i, mapIndex := range mapIndices {
-	}
+		if mapIndex/f.baseRowGroupLength != baseRowGroup {
-	baseRow := baseRows[mapIndex&(f.baseRowGroupLength-1)]
+			panic("mapIndices are not in the same base row group")
-	if baseLayerOnly {
-		return baseRow, nil
 		}
+		row := baseRows[mapIndex&(f.baseRowGroupLength-1)]
+		if !baseLayerOnly {
 			extRow, err := rawdb.ReadFilterMapExtRow(f.db, f.mapRowIndex(mapIndex, rowIndex), f.logMapWidth)
 			if err != nil {
-		return nil, fmt.Errorf("failed to retrieve filter map %d extended row %d: %v", mapIndex, rowIndex, err)
+				return fmt.Errorf("failed to retrieve filter map %d extended row %d: %v", mapIndex, rowIndex, err)
 			}
-	return FilterRow(append(baseRow, extRow...)), nil
+			row = append(row, extRow...)
+		}
+		target[i] = row
+	}
+	return nil
 }
 
 // storeFilterMapRows stores a set of filter map rows at the corresponding map
 // indices and a shared row index.
 func (f *FilterMaps) storeFilterMapRows(batch ethdb.Batch, mapIndices []uint32, rowIndex uint32, rows []FilterRow) error {
 	for len(mapIndices) > 0 {
-		baseMapIndex := mapIndices[0] & -f.baseRowGroupLength
+		baseRowGroup := mapIndices[0] / f.baseRowGroupLength
 		groupLength := 1
-		for groupLength < len(mapIndices) && mapIndices[groupLength]&-f.baseRowGroupLength == baseMapIndex {
+		for groupLength < len(mapIndices) && mapIndices[groupLength]/f.baseRowGroupLength == baseRowGroup {
 			groupLength++
 		}
 		if err := f.storeFilterMapRowsOfGroup(batch, mapIndices[:groupLength], rowIndex, rows[:groupLength]); err != nil {
@@ -527,24 +637,20 @@ func (f *FilterMaps) storeFilterMapRows(batch ethdb.Batch, mapIndices []uint32,
 // storeFilterMapRowsOfGroup stores a set of filter map rows at map indices
 // belonging to the same base row group.
 func (f *FilterMaps) storeFilterMapRowsOfGroup(batch ethdb.Batch, mapIndices []uint32, rowIndex uint32, rows []FilterRow) error {
-	baseMapIndex := mapIndices[0] & -f.baseRowGroupLength
+	baseRowGroup := mapIndices[0] / f.baseRowGroupLength
-	baseMapRowIndex := f.mapRowIndex(baseMapIndex, rowIndex)
+	baseMapRowIndex := f.mapRowIndex(baseRowGroup*f.baseRowGroupLength, rowIndex)
 	var baseRows [][]uint32
 	if uint32(len(mapIndices)) != f.baseRowGroupLength { // skip base rows read if all rows are replaced
-		var ok bool
-		baseRows, ok = f.baseRowsCache.Get(baseMapRowIndex)
-		if !ok {
 		var err error
 		baseRows, err = rawdb.ReadFilterMapBaseRows(f.db, baseMapRowIndex, f.baseRowGroupLength, f.logMapWidth)
 		if err != nil {
-				return fmt.Errorf("failed to retrieve filter map %d base rows %d for modification: %v", mapIndices[0]&-f.baseRowGroupLength, rowIndex, err)
+			return fmt.Errorf("failed to retrieve base row group %d of row %d for modification: %v", baseRowGroup, rowIndex, err)
-			}
 		}
 	} else {
 		baseRows = make([][]uint32, f.baseRowGroupLength)
 	}
 	for i, mapIndex := range mapIndices {
-		if mapIndex&-f.baseRowGroupLength != baseMapIndex {
+		if mapIndex/f.baseRowGroupLength != baseRowGroup {
 			panic("mapIndices are not in the same base row group")
 		}
 		baseRow := []uint32(rows[i])
@@ -556,7 +662,6 @@ func (f *FilterMaps) storeFilterMapRowsOfGroup(batch ethdb.Batch, mapIndices []u
 		baseRows[mapIndex&(f.baseRowGroupLength-1)] = baseRow
 		rawdb.WriteFilterMapExtRow(batch, f.mapRowIndex(mapIndex, rowIndex), extRow, f.logMapWidth)
 	}
-	f.baseRowsCache.Add(baseMapRowIndex, baseRows)
 	rawdb.WriteFilterMapBaseRows(batch, baseMapRowIndex, baseRows, f.logMapWidth)
 	return nil
 }
@@ -573,14 +678,11 @@ func (f *FilterMaps) mapRowIndex(mapIndex, rowIndex uint32) uint64 {
 }
 
 // getBlockLvPointer returns the starting log value index where the log values
-// generated by the given block are located. If blockNumber is beyond the current
+// generated by the given block are located.
-// head then the first unoccupied log value index is returned.
+//
 // Note that this function assumes that the indexer read lock is being held when
 // called from outside the indexerLoop goroutine.
 func (f *FilterMaps) getBlockLvPointer(blockNumber uint64) (uint64, error) {
-	if blockNumber >= f.indexedRange.blocks.AfterLast() && f.indexedRange.headIndexed {
-		return f.indexedRange.headDelimiter, nil
-	}
 	if lvPointer, ok := f.lvPointerCache.Get(blockNumber); ok {
 		return lvPointer, nil
 	}
@@ -634,57 +736,106 @@ func (f *FilterMaps) deleteLastBlockOfMap(batch ethdb.Batch, mapIndex uint32) {
 	rawdb.DeleteFilterMapLastBlock(batch, mapIndex)
 }
 
-// deleteTailEpoch deletes index data from the earliest, either fully or partially
+// deleteTailEpoch deletes index data from the specified epoch. The last block
-// indexed epoch. The last block pointer for the last map of the epoch and the
+// pointer for the last map of the epoch and the corresponding block log value
-// corresponding block log value pointer are retained as these are always assumed
+// pointer are retained as these are always assumed to be available for each
-// to be available for each epoch.
+// epoch as boundary markers.
-func (f *FilterMaps) deleteTailEpoch(epoch uint32) error {
+// The function returns true if all index data related to the epoch (except for
+// the boundary markers) has been fully removed.
+func (f *FilterMaps) deleteTailEpoch(epoch uint32) (bool, error) {
 	f.indexLock.Lock()
 	defer f.indexLock.Unlock()
 
+	// determine epoch boundaries
 	firstMap := epoch << f.logMapsPerEpoch
 	lastBlock, _, err := f.getLastBlockOfMap(firstMap + f.mapsPerEpoch - 1)
 	if err != nil {
-		return fmt.Errorf("failed to retrieve last block of deleted epoch %d: %v", epoch, err)
+		return false, fmt.Errorf("failed to retrieve last block of deleted epoch %d: %v", epoch, err)
 	}
 	var firstBlock uint64
 	if epoch > 0 {
 		firstBlock, _, err = f.getLastBlockOfMap(firstMap - 1)
 		if err != nil {
-			return fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err)
+			return false, fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err)
 		}
 		firstBlock++
 	}
-	fmr := f.indexedRange
+	// update rendered range if necessary
-	if f.indexedRange.maps.First() == firstMap &&
+	var (
-		f.indexedRange.maps.AfterLast() > firstMap+f.mapsPerEpoch &&
+		fmr            = f.indexedRange
-		f.indexedRange.tailPartialEpoch == 0 {
+		firstEpoch     = f.indexedRange.maps.First() >> f.logMapsPerEpoch
-		fmr.maps.SetFirst(firstMap + f.mapsPerEpoch)
+		afterLastEpoch = (f.indexedRange.maps.AfterLast() + f.mapsPerEpoch - 1) >> f.logMapsPerEpoch
-		fmr.blocks.SetFirst(lastBlock + 1)
+	)
-	} else if f.indexedRange.maps.First() == firstMap+f.mapsPerEpoch {
+	if f.indexedRange.tailPartialEpoch != 0 && firstEpoch > 0 {
-		fmr.tailPartialEpoch = 0
+		firstEpoch--
-	} else {
-		return errors.New("invalid tail epoch number")
 	}
-	f.setRange(f.db, f.indexedView, fmr)
+	switch {
+	case epoch < firstEpoch:
+	// cleanup of already unindexed epoch; range not affected
+	case epoch == firstEpoch && epoch+1 < afterLastEpoch:
+		// first fully or partially rendered epoch and there is at least one
+		// rendered map in the next epoch; remove from indexed range
+		fmr.tailPartialEpoch = 0
+		fmr.maps.SetFirst((epoch + 1) << f.logMapsPerEpoch)
+		fmr.blocks.SetFirst(lastBlock + 1)
+		f.setRange(f.db, f.indexedView, fmr, false)
+	default:
+		// cannot be cleaned or unindexed; return with error
+		return false, errors.New("invalid tail epoch number")
+	}
+	// remove index data
+	deleteFn := func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error {
 		first := f.mapRowIndex(firstMap, 0)
 		count := f.mapRowIndex(firstMap+f.mapsPerEpoch, 0) - first
-	rawdb.DeleteFilterMapRows(f.db, common.NewRange(first, count))
+		if err := rawdb.DeleteFilterMapRows(f.db, common.NewRange(first, count), hashScheme, stopCb); err != nil {
+			return err
+		}
 		for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch; mapIndex++ {
 			f.filterMapCache.Remove(mapIndex)
 		}
-	rawdb.DeleteFilterMapLastBlocks(f.db, common.NewRange(firstMap, f.mapsPerEpoch-1)) // keep last enrty
+		delMapRange := common.NewRange(firstMap, f.mapsPerEpoch-1) // keep last entry
+		if err := rawdb.DeleteFilterMapLastBlocks(f.db, delMapRange, hashScheme, stopCb); err != nil {
+			return err
+		}
 		for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch-1; mapIndex++ {
 			f.lastBlockCache.Remove(mapIndex)
 		}
-	rawdb.DeleteBlockLvPointers(f.db, common.NewRange(firstBlock, lastBlock-firstBlock)) // keep last enrty
+		delBlockRange := common.NewRange(firstBlock, lastBlock-firstBlock) // keep last entry
+		if err := rawdb.DeleteBlockLvPointers(f.db, delBlockRange, hashScheme, stopCb); err != nil {
+			return err
+		}
 		for blockNumber := firstBlock; blockNumber < lastBlock; blockNumber++ {
 			f.lvPointerCache.Remove(blockNumber)
 		}
 		return nil
 	}
+	action := fmt.Sprintf("Deleting tail epoch #%d", epoch)
+	stopFn := func() bool {
+		f.processEvents()
+		return f.stop || !f.targetHeadIndexed()
+	}
+	if err := f.safeDeleteWithLogs(deleteFn, action, stopFn); err == nil {
+		// everything removed; mark as cleaned and report success
+		if f.cleanedEpochsBefore == epoch {
+			f.cleanedEpochsBefore = epoch + 1
+		}
+		return true, nil
+	} else {
+		// more data left in epoch range; mark as dirty and report unfinished
+		if f.cleanedEpochsBefore > epoch {
+			f.cleanedEpochsBefore = epoch
+		}
+		if errors.Is(err, rawdb.ErrDeleteRangeInterrupted) {
+			return false, nil
+		}
+		return false, err
+	}
+}
 
 // exportCheckpoints exports epoch checkpoints in the format used by checkpoints.go.
+//
+// Note: acquiring the indexLock read lock is unnecessary here, as this function
+// is always called within the indexLoop.
 func (f *FilterMaps) exportCheckpoints() {
 	finalLvPtr, err := f.getBlockLvPointer(f.finalBlock + 1)
 	if err != nil {