Merge pull request #17 from taurusgroup/release_v1.13.2

Release v1.13.2

.travis.yml

@@ -13,7 +13,7 @@ jobs:
     - stage: lint
       os: linux
       dist: bionic
-      go: 1.20.x
+      go: 1.21.x
       env:
         - lint
       git:
@@ -28,7 +28,7 @@ jobs:
       os: linux
       arch: amd64
       dist: bionic
-      go: 1.20.x
+      go: 1.21.x
       env:
         - docker
       services:
@@ -45,7 +45,7 @@ jobs:
       os: linux
       arch: arm64
       dist: bionic
-      go: 1.20.x
+      go: 1.21.x
       env:
         - docker
       services:
@@ -63,10 +63,9 @@ jobs:
       os: linux
       dist: bionic
       sudo: required
-      go: 1.20.x
+      go: 1.21.x
       env:
         - azure-linux
-        - GO111MODULE=on
       git:
         submodules: false # avoid cloning ethereum/tests
       addons:
@@ -97,10 +96,9 @@ jobs:
     - stage: build
       if: type = push
       os: osx
-      go: 1.20.x
+      go: 1.21.x
       env:
         - azure-osx
-        - GO111MODULE=on
       git:
         submodules: false # avoid cloning ethereum/tests
       script:
@@ -112,41 +110,34 @@ jobs:
       os: linux
       arch: amd64
       dist: bionic
-      go: 1.20.x
+      go: 1.21.x
-      env:
-        - GO111MODULE=on
       script:
-        - go run build/ci.go test $TEST_PACKAGES
+        - travis_wait 30 go run build/ci.go test $TEST_PACKAGES
 
     - stage: build
       if: type = pull_request
       os: linux
       arch: arm64
       dist: bionic
-      go: 1.19.x
+      go: 1.20.x
-      env:
-        - GO111MODULE=on
       script:
-        - go run build/ci.go test $TEST_PACKAGES
+        - travis_wait 30 go run build/ci.go test $TEST_PACKAGES
 
     - stage: build
       os: linux
       dist: bionic
-      go: 1.19.x
+      go: 1.20.x
-      env:
-        - GO111MODULE=on
       script:
-        - go run build/ci.go test $TEST_PACKAGES
+        - travis_wait 30 go run build/ci.go test $TEST_PACKAGES
 
     # This builder does the Ubuntu PPA nightly uploads
     - stage: build
       if: type = cron || (type = push && tag ~= /^v[0-9]/)
       os: linux
       dist: bionic
-      go: 1.20.x
+      go: 1.21.x
       env:
         - ubuntu-ppa
-        - GO111MODULE=on
       git:
         submodules: false # avoid cloning ethereum/tests
       addons:
@@ -167,10 +158,9 @@ jobs:
       if: type = cron
       os: linux
       dist: bionic
-      go: 1.20.x
+      go: 1.21.x
       env:
         - azure-purge
-        - GO111MODULE=on
       git:
         submodules: false # avoid cloning ethereum/tests
       script:
@@ -181,9 +171,7 @@ jobs:
       if: type = cron
       os: linux
       dist: bionic
-      go: 1.20.x
+      go: 1.21.x
-      env:
-        - GO111MODULE=on
       script:
-        - go run build/ci.go test  -race $TEST_PACKAGES
+        - travis_wait 30 go run build/ci.go test  -race $TEST_PACKAGES
 

Dockerfile

@@ -4,7 +4,7 @@ ARG VERSION=""
 ARG BUILDNUM=""
 
 # Build Geth in a stock Go builder container
-FROM golang:1.20-alpine as builder
+FROM golang:1.21-alpine as builder
 
 RUN apk add --no-cache gcc musl-dev linux-headers git
 

Makefile

@@ -6,7 +6,7 @@
 
 GOBIN = ./build/bin
 GO ?= latest
-GORUN = env GO111MODULE=on go run
+GORUN = go run
 
 geth:
 	$(GORUN) build/ci.go install ./cmd/geth
@@ -23,7 +23,7 @@ lint: ## Run linters.
 	$(GORUN) build/ci.go lint
 
 clean:
-	env GO111MODULE=on go clean -cache
+	go clean -cache
 	rm -fr build/_workspace/pkg/ $(GOBIN)/*
 
 # The devtools target installs tools required for 'go generate'.

accounts/abi/abi.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -246,24 +247,65 @@ func (abi *ABI) HasReceive() bool {
 // revertSelector is a special function selector for revert reason unpacking.
 var revertSelector = crypto.Keccak256([]byte("Error(string)"))[:4]
 
+// panicSelector is a special function selector for panic reason unpacking.
+var panicSelector = crypto.Keccak256([]byte("Panic(uint256)"))[:4]
+
+// panicReasons map is for readable panic codes
+// see this linkage for the deails
+// https://docs.soliditylang.org/en/v0.8.21/control-structures.html#panic-via-assert-and-error-via-require
+// the reason string list is copied from ether.js
+// https://github.com/ethers-io/ethers.js/blob/fa3a883ff7c88611ce766f58bdd4b8ac90814470/src.ts/abi/interface.ts#L207-L218
+var panicReasons = map[uint64]string{
+	0x00: "generic panic",
+	0x01: "assert(false)",
+	0x11: "arithmetic underflow or overflow",
+	0x12: "division or modulo by zero",
+	0x21: "enum overflow",
+	0x22: "invalid encoded storage byte array accessed",
+	0x31: "out-of-bounds array access; popping on an empty array",
+	0x32: "out-of-bounds access of an array or bytesN",
+	0x41: "out of memory",
+	0x51: "uninitialized function",
+}
+
 // UnpackRevert resolves the abi-encoded revert reason. According to the solidity
 // spec https://solidity.readthedocs.io/en/latest/control-structures.html#revert,
-// the provided revert reason is abi-encoded as if it were a call to a function
+// the provided revert reason is abi-encoded as if it were a call to function
-// `Error(string)`. So it's a special tool for it.
+// `Error(string)` or `Panic(uint256)`. So it's a special tool for it.
 func UnpackRevert(data []byte) (string, error) {
 	if len(data) < 4 {
 		return "", errors.New("invalid data for unpacking")
 	}
-	if !bytes.Equal(data[:4], revertSelector) {
+	switch {
+	case bytes.Equal(data[:4], revertSelector):
+		typ, err := NewType("string", "", nil)
+		if err != nil {
+			return "", err
+		}
+		unpacked, err := (Arguments{{Type: typ}}).Unpack(data[4:])
+		if err != nil {
+			return "", err
+		}
+		return unpacked[0].(string), nil
+	case bytes.Equal(data[:4], panicSelector):
+		typ, err := NewType("uint256", "", nil)
+		if err != nil {
+			return "", err
+		}
+		unpacked, err := (Arguments{{Type: typ}}).Unpack(data[4:])
+		if err != nil {
+			return "", err
+		}
+		pCode := unpacked[0].(*big.Int)
+		// uint64 safety check for future
+		// but the code is not bigger than MAX(uint64) now
+		if pCode.IsUint64() {
+			if reason, ok := panicReasons[pCode.Uint64()]; ok {
+				return reason, nil
+			}
+		}
+		return fmt.Sprintf("unknown panic code: %#x", pCode), nil
+	default:
 		return "", errors.New("invalid data for unpacking")
 	}
-	typ, err := NewType("string", "", nil)
-	if err != nil {
-		return "", err
-	}
-	unpacked, err := (Arguments{{Type: typ}}).Unpack(data[4:])
-	if err != nil {
-		return "", err
-	}
-	return unpacked[0].(string), nil
 }

accounts/abi/abi_test.go

@@ -1173,6 +1173,8 @@ func TestUnpackRevert(t *testing.T) {
 		{"", "", errors.New("invalid data for unpacking")},
 		{"08c379a1", "", errors.New("invalid data for unpacking")},
 		{"08c379a00000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000000000000000000000000000000000000000d72657665727420726561736f6e00000000000000000000000000000000000000", "revert reason", nil},
+		{"4e487b710000000000000000000000000000000000000000000000000000000000000000", "generic panic", nil},
+		{"4e487b7100000000000000000000000000000000000000000000000000000000000000ff", "unknown panic code: 0xff", nil},
 	}
 	for index, c := range cases {
 		t.Run(fmt.Sprintf("case %d", index), func(t *testing.T) {

accounts/abi/bind/backends/simulated.go

@@ -199,7 +199,6 @@ func (b *SimulatedBackend) CodeAt(ctx context.Context, contract common.Address,
 	if err != nil {
 		return nil, err
 	}
-
 	return stateDB.GetCode(contract), nil
 }
 
@@ -212,7 +211,6 @@ func (b *SimulatedBackend) BalanceAt(ctx context.Context, contract common.Addres
 	if err != nil {
 		return nil, err
 	}
-
 	return stateDB.GetBalance(contract), nil
 }
 
@@ -225,7 +223,6 @@ func (b *SimulatedBackend) NonceAt(ctx context.Context, contract common.Address,
 	if err != nil {
 		return 0, err
 	}
-
 	return stateDB.GetNonce(contract), nil
 }
 
@@ -238,7 +235,6 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres
 	if err != nil {
 		return nil, err
 	}
-
 	val := stateDB.GetState(contract, key)
 	return val[:], nil
 }
@@ -700,8 +696,10 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 		}
 		block.AddTxWithChain(b.blockchain, tx)
 	})
-	stateDB, _ := b.blockchain.State()
+	stateDB, err := b.blockchain.State()
-
+	if err != nil {
+		return err
+	}
 	b.pendingBlock = blocks[0]
 	b.pendingState, _ = state.New(b.pendingBlock.Root(), stateDB.Database(), nil)
 	b.pendingReceipts = receipts[0]
@@ -821,11 +819,12 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
 	blocks, _ := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {
 		block.OffsetTime(int64(adjustment.Seconds()))
 	})
-	stateDB, _ := b.blockchain.State()
+	stateDB, err := b.blockchain.State()
-
+	if err != nil {
+		return err
+	}
 	b.pendingBlock = blocks[0]
 	b.pendingState, _ = state.New(b.pendingBlock.Root(), stateDB.Database(), nil)
-
 	return nil
 }
 
@@ -892,7 +891,7 @@ func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (typ
 }
 
 func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) {
-	logs := rawdb.ReadLogs(fb.db, hash, number, fb.bc.Config())
+	logs := rawdb.ReadLogs(fb.db, hash, number)
 	return logs, nil
 }
 

accounts/abi/method.go

@@ -127,11 +127,12 @@ func NewMethod(name string, rawName string, funType FunctionType, mutability str
 		state = state + " "
 	}
 	identity := fmt.Sprintf("function %v", rawName)
-	if funType == Fallback {
+	switch funType {
+	case Fallback:
 		identity = "fallback"
-	} else if funType == Receive {
+	case Receive:
 		identity = "receive"
-	} else if funType == Constructor {
+	case Constructor:
 		identity = "constructor"
 	}
 	str := fmt.Sprintf("%v(%v) %sreturns(%v)", identity, strings.Join(inputNames, ", "), state, strings.Join(outputNames, ", "))

accounts/abi/method_test.go

@@ -84,11 +84,12 @@ func TestMethodString(t *testing.T) {
 
 	for _, test := range table {
 		var got string
-		if test.method == "fallback" {
+		switch test.method {
+		case "fallback":
 			got = abi.Fallback.String()
-		} else if test.method == "receive" {
+		case "receive":
 			got = abi.Receive.String()
-		} else {
+		default:
 			got = abi.Methods[test.method].String()
 		}
 		if got != test.expectation {

accounts/abi/unpack.go

@@ -160,13 +160,14 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error)
 	// this value will become our slice or our array, depending on the type
 	var refSlice reflect.Value
 
-	if t.T == SliceTy {
+	switch t.T {
+	case SliceTy:
 		// declare our slice
 		refSlice = reflect.MakeSlice(t.GetType(), size, size)
-	} else if t.T == ArrayTy {
+	case ArrayTy:
 		// declare our array
 		refSlice = reflect.New(t.GetType()).Elem()
-	} else {
+	default:
 		return nil, errors.New("abi: invalid type in array/slice unpacking stage")
 	}
 

accounts/keystore/watch.go

@@ -20,6 +20,7 @@
 package keystore
 
 import (
+	"os"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -77,7 +78,9 @@ func (w *watcher) loop() {
 	}
 	defer watcher.Close()
 	if err := watcher.Add(w.ac.keydir); err != nil {
-		logger.Warn("Failed to watch keystore folder", "err", err)
+		if !os.IsNotExist(err) {
+			logger.Warn("Failed to watch keystore folder", "err", err)
+		}
 		return
 	}
 

beacon/engine/errors.go

@@ -80,6 +80,7 @@ var (
 	InvalidPayloadAttributes = &EngineAPIError{code: -38003, msg: "Invalid payload attributes"}
 	TooLargeRequest          = &EngineAPIError{code: -38004, msg: "Too large request"}
 	InvalidParams            = &EngineAPIError{code: -32602, msg: "Invalid parameters"}
+	UnsupportedFork          = &EngineAPIError{code: -38005, msg: "Unsupported fork"}
 
 	STATUS_INVALID         = ForkChoiceResponse{PayloadStatus: PayloadStatusV1{Status: INVALID}, PayloadID: nil}
 	STATUS_SYNCING         = ForkChoiceResponse{PayloadStatus: PayloadStatusV1{Status: SYNCING}, PayloadID: nil}

beacon/engine/gen_blockparams.go

@@ -20,12 +20,14 @@ func (p PayloadAttributes) MarshalJSON() ([]byte, error) {
 		Random                common.Hash         `json:"prevRandao"            gencodec:"required"`
 		SuggestedFeeRecipient common.Address      `json:"suggestedFeeRecipient" gencodec:"required"`
 		Withdrawals           []*types.Withdrawal `json:"withdrawals"`
+		BeaconRoot            *common.Hash        `json:"parentBeaconBlockRoot"`
 	}
 	var enc PayloadAttributes
 	enc.Timestamp = hexutil.Uint64(p.Timestamp)
 	enc.Random = p.Random
 	enc.SuggestedFeeRecipient = p.SuggestedFeeRecipient
 	enc.Withdrawals = p.Withdrawals
+	enc.BeaconRoot = p.BeaconRoot
 	return json.Marshal(&enc)
 }
 
@@ -36,6 +38,7 @@ func (p *PayloadAttributes) UnmarshalJSON(input []byte) error {
 		Random                *common.Hash        `json:"prevRandao"            gencodec:"required"`
 		SuggestedFeeRecipient *common.Address     `json:"suggestedFeeRecipient" gencodec:"required"`
 		Withdrawals           []*types.Withdrawal `json:"withdrawals"`
+		BeaconRoot            *common.Hash        `json:"parentBeaconBlockRoot"`
 	}
 	var dec PayloadAttributes
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -56,5 +59,8 @@ func (p *PayloadAttributes) UnmarshalJSON(input []byte) error {
 	if dec.Withdrawals != nil {
 		p.Withdrawals = dec.Withdrawals
 	}
+	if dec.BeaconRoot != nil {
+		p.BeaconRoot = dec.BeaconRoot
+	}
 	return nil
 }

beacon/engine/gen_epe.go

@@ -18,11 +18,13 @@ func (e ExecutionPayloadEnvelope) MarshalJSON() ([]byte, error) {
 		ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
 		BlockValue       *hexutil.Big    `json:"blockValue"  gencodec:"required"`
 		BlobsBundle      *BlobsBundleV1  `json:"blobsBundle"`
+		Override         bool            `json:"shouldOverrideBuilder"`
 	}
 	var enc ExecutionPayloadEnvelope
 	enc.ExecutionPayload = e.ExecutionPayload
 	enc.BlockValue = (*hexutil.Big)(e.BlockValue)
 	enc.BlobsBundle = e.BlobsBundle
+	enc.Override = e.Override
 	return json.Marshal(&enc)
 }
 
@@ -32,6 +34,7 @@ func (e *ExecutionPayloadEnvelope) UnmarshalJSON(input []byte) error {
 		ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
 		BlockValue       *hexutil.Big    `json:"blockValue"  gencodec:"required"`
 		BlobsBundle      *BlobsBundleV1  `json:"blobsBundle"`
+		Override         *bool           `json:"shouldOverrideBuilder"`
 	}
 	var dec ExecutionPayloadEnvelope
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -48,5 +51,8 @@ func (e *ExecutionPayloadEnvelope) UnmarshalJSON(input []byte) error {
 	if dec.BlobsBundle != nil {
 		e.BlobsBundle = dec.BlobsBundle
 	}
+	if dec.Override != nil {
+		e.Override = *dec.Override
+	}
 	return nil
 }

beacon/engine/types.go

@@ -23,7 +23,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/trie"
 )
 
@@ -36,6 +35,7 @@ type PayloadAttributes struct {
 	Random                common.Hash         `json:"prevRandao"            gencodec:"required"`
 	SuggestedFeeRecipient common.Address      `json:"suggestedFeeRecipient" gencodec:"required"`
 	Withdrawals           []*types.Withdrawal `json:"withdrawals"`
+	BeaconRoot            *common.Hash        `json:"parentBeaconBlockRoot"`
 }
 
 // JSON type overrides for PayloadAttributes.
@@ -86,6 +86,7 @@ type ExecutionPayloadEnvelope struct {
 	ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
 	BlockValue       *big.Int        `json:"blockValue"  gencodec:"required"`
 	BlobsBundle      *BlobsBundleV1  `json:"blobsBundle"`
+	Override         bool            `json:"shouldOverrideBuilder"`
 }
 
 type BlobsBundleV1 struct {
@@ -172,7 +173,7 @@ func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) {
 // and that the blockhash of the constructed block matches the parameters. Nil
 // Withdrawals value will propagate through the returned block. Empty
 // Withdrawals value must be passed via non-nil, length 0 value in params.
-func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash) (*types.Block, error) {
+func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash, beaconRoot *common.Hash) (*types.Block, error) {
 	txs, err := decodeTransactions(params.Transactions)
 	if err != nil {
 		return nil, err
@@ -208,24 +209,25 @@ func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash)
 		withdrawalsRoot = &h
 	}
 	header := &types.Header{
-		ParentHash:      params.ParentHash,
+		ParentHash:       params.ParentHash,
-		UncleHash:       types.EmptyUncleHash,
+		UncleHash:        types.EmptyUncleHash,
-		Coinbase:        params.FeeRecipient,
+		Coinbase:         params.FeeRecipient,
-		Root:            params.StateRoot,
+		Root:             params.StateRoot,
-		TxHash:          types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)),
+		TxHash:           types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)),
-		ReceiptHash:     params.ReceiptsRoot,
+		ReceiptHash:      params.ReceiptsRoot,
-		Bloom:           types.BytesToBloom(params.LogsBloom),
+		Bloom:            types.BytesToBloom(params.LogsBloom),
-		Difficulty:      common.Big0,
+		Difficulty:       common.Big0,
-		Number:          new(big.Int).SetUint64(params.Number),
+		Number:           new(big.Int).SetUint64(params.Number),
-		GasLimit:        params.GasLimit,
+		GasLimit:         params.GasLimit,
-		GasUsed:         params.GasUsed,
+		GasUsed:          params.GasUsed,
-		Time:            params.Timestamp,
+		Time:             params.Timestamp,
-		BaseFee:         params.BaseFeePerGas,
+		BaseFee:          params.BaseFeePerGas,
-		Extra:           params.ExtraData,
+		Extra:            params.ExtraData,
-		MixDigest:       params.Random,
+		MixDigest:        params.Random,
-		WithdrawalsHash: withdrawalsRoot,
+		WithdrawalsHash:  withdrawalsRoot,
-		ExcessBlobGas:   params.ExcessBlobGas,
+		ExcessBlobGas:    params.ExcessBlobGas,
-		BlobGasUsed:     params.BlobGasUsed,
+		BlobGasUsed:      params.BlobGasUsed,
+		ParentBeaconRoot: beaconRoot,
 	}
 	block := types.NewBlockWithHeader(header).WithBody(txs, nil /* uncles */).WithWithdrawals(params.Withdrawals)
 	if block.Hash() != params.BlockHash {
@@ -236,7 +238,7 @@ func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash)
 
 // BlockToExecutableData constructs the ExecutableData structure by filling the
 // fields from the given block. It assumes the given block is post-merge block.
-func BlockToExecutableData(block *types.Block, fees *big.Int, blobs []kzg4844.Blob, commitments []kzg4844.Commitment, proofs []kzg4844.Proof) *ExecutionPayloadEnvelope {
+func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types.BlobTxSidecar) *ExecutionPayloadEnvelope {
 	data := &ExecutableData{
 		BlockHash:     block.Hash(),
 		ParentHash:    block.ParentHash(),
@@ -256,17 +258,19 @@ func BlockToExecutableData(block *types.Block, fees *big.Int, blobs []kzg4844.Bl
 		BlobGasUsed:   block.BlobGasUsed(),
 		ExcessBlobGas: block.ExcessBlobGas(),
 	}
-	blobsBundle := BlobsBundleV1{
+	bundle := BlobsBundleV1{
 		Commitments: make([]hexutil.Bytes, 0),
 		Blobs:       make([]hexutil.Bytes, 0),
 		Proofs:      make([]hexutil.Bytes, 0),
 	}
-	for i := range blobs {
+	for _, sidecar := range sidecars {
-		blobsBundle.Blobs = append(blobsBundle.Blobs, hexutil.Bytes(blobs[i][:]))
+		for j := range sidecar.Blobs {
-		blobsBundle.Commitments = append(blobsBundle.Commitments, hexutil.Bytes(commitments[i][:]))
+			bundle.Blobs = append(bundle.Blobs, hexutil.Bytes(sidecar.Blobs[j][:]))
-		blobsBundle.Proofs = append(blobsBundle.Proofs, hexutil.Bytes(proofs[i][:]))
+			bundle.Commitments = append(bundle.Commitments, hexutil.Bytes(sidecar.Commitments[j][:]))
+			bundle.Proofs = append(bundle.Proofs, hexutil.Bytes(sidecar.Proofs[j][:]))
+		}
 	}
-	return &ExecutionPayloadEnvelope{ExecutionPayload: data, BlockValue: fees, BlobsBundle: &blobsBundle}
+	return &ExecutionPayloadEnvelope{ExecutionPayload: data, BlockValue: fees, BlobsBundle: &bundle, Override: false}
 }
 
 // ExecutionPayloadBodyV1 is used in the response to GetPayloadBodiesByHashV1 and GetPayloadBodiesByRangeV1

build/checksums.txt

@@ -1,20 +1,25 @@
 # This file contains sha256 checksums of optional build dependencies.
 
-2c5ee9c9ec1e733b0dbbc2bdfed3f62306e51d8172bf38f4f4e542b27520f597  go1.20.7.src.tar.gz
+# https://github.com/ethereum/execution-spec-tests/releases
-785170eab380a8985d53896808b0a71336d0ea60e0a26099b4ccec77798b1cf4  go1.20.7.darwin-amd64.tar.gz
+24bac679f3a2d8240d8e08e7f6a70b70c2dabf673317d924cf1d1887b9fe1f81  fixtures.tar.gz
-eea1e7e4c2f75c72629050e6a6c7c46c446d64056732a7787fb3ba16ace1982e  go1.20.7.darwin-arm64.tar.gz
-d8cff0357ac24eb06f3f280535397eeaacf95611d29f9b2abc3060f3d6dce3b4  go1.20.7.freebsd-386.tar.gz
-26918dcebf474a9e81ccf9f648cdf36968dfb76b481518cf615d78455dda4416  go1.20.7.freebsd-amd64.tar.gz
-ddb48145f05bda2f4617a22c979d4e94b22802cdb1a1fde1b1974e733b26f091  go1.20.7.linux-386.tar.gz
-f0a87f1bcae91c4b69f8dc2bc6d7e6bfcd7524fceec130af525058c0c17b1b44  go1.20.7.linux-amd64.tar.gz
-44781ae3b153c3b07651d93b6bc554e835a36e2d72a696281c1e4dad9efffe43  go1.20.7.linux-arm64.tar.gz
-7cc231b415b94f2f7065870a73f67dd2b0ec12b5a98052b7ee0121c42bc04f8d  go1.20.7.linux-armv6l.tar.gz
-6318a1db307c12b8afe68808bd6fae4fba1e558a85b958216096869ed506dcb3  go1.20.7.linux-ppc64le.tar.gz
-26aea2ede8722ceecbd9db883328a8d963136fd96c11dacc356c44c4c19c6515  go1.20.7.linux-s390x.tar.gz
-5b0ef6f58d3e04d6cc003aa98e9172f41ba9e091b1c98e7339b41c4c87fb78a1  go1.20.7.windows-386.zip
-736dc6c7fcab1c96b682c8c93e38d7e371e62a17d34cb2c37d451a1147f66af9  go1.20.7.windows-amd64.zip
-fc6f79c1e1ed9e506c65f2112ac4e387479916f1accb0d046a6a19ff6938baa5  go1.20.7.windows-arm64.zip
 
+# https://go.dev/dl/
+bfa36bf75e9a1e9cbbdb9abcf9d1707e479bd3a07880a8ae3564caee5711cb99  go1.21.1.src.tar.gz
+809f5b0ef4f7dcdd5f51e9630a5b2e5a1006f22a047126d61560cdc365678a19  go1.21.1.darwin-amd64.tar.gz
+ffd40391a1e995855488b008ad9326ff8c2e81803a6e80894401003bae47fcf1  go1.21.1.darwin-arm64.tar.gz
+9919a9a4dc82371aba3da5b7c830bcb6249fc1502cd26d959eb340a60e41ee01  go1.21.1.freebsd-386.tar.gz
+2571f10f6047e04d87c1f5986a05e5e8f7b511faf98803ef12b66d563845d2a1  go1.21.1.freebsd-amd64.tar.gz
+b93850666cdadbd696a986cf7b03111fe99db8c34a9aaa113d7c96d0081e1901  go1.21.1.linux-386.tar.gz
+b3075ae1ce5dab85f89bc7905d1632de23ca196bd8336afd93fa97434cfa55ae  go1.21.1.linux-amd64.tar.gz
+7da1a3936a928fd0b2602ed4f3ef535b8cd1990f1503b8d3e1acc0fa0759c967  go1.21.1.linux-arm64.tar.gz
+f3716a43f59ae69999841d6007b42c9e286e8d8ce470656fb3e70d7be2d7ca85  go1.21.1.linux-armv6l.tar.gz
+eddf018206f8a5589bda75252b72716d26611efebabdca5d0083ec15e9e41ab7  go1.21.1.linux-ppc64le.tar.gz
+a83b3e8eb4dbf76294e773055eb51397510ff4d612a247bad9903560267bba6d  go1.21.1.linux-s390x.tar.gz
+170256c820f466f29d64876f25f4dfa4029ed9902a0a9095d8bd603aecf4d83b  go1.21.1.windows-386.zip
+10a4f5b63215d11d1770453733dbcbf024f3f74872f84e28d7ea59f0250316c6  go1.21.1.windows-amd64.zip
+41135ce6e0ced4bc1e459cb96bd4090c9dc2062e24179c3f337d855af9b560ef  go1.21.1.windows-arm64.zip
+
+# https://github.com/golangci/golangci-lint/releases
 fba08acc4027f69f07cef48fbff70b8a7ecdfaa1c2aba9ad3fb31d60d9f5d4bc  golangci-lint-1.51.1-darwin-amd64.tar.gz
 75b8f0ff3a4e68147156be4161a49d4576f1be37a0b506473f8c482140c1e7f2  golangci-lint-1.51.1-darwin-arm64.tar.gz
 e06b3459aaed356e1667580be00b05f41f3b2e29685d12cdee571c23e1edb414  golangci-lint-1.51.1-freebsd-386.tar.gz

build/ci.go

@@ -120,15 +120,15 @@ var (
 	// Distros for which packages are created.
 	// Note: vivid is unsupported because there is no golang-1.6 package for it.
 	// Note: the following Ubuntu releases have been officially deprecated on Launchpad:
-	//   wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite, impish
+	//   wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy, hirsuite, impish,
+	//   kinetic
 	debDistroGoBoots = map[string]string{
-		"trusty":  "golang-1.11", // EOL: 04/2024
+		"trusty": "golang-1.11", // EOL: 04/2024
-		"xenial":  "golang-go",   // EOL: 04/2026
+		"xenial": "golang-go",   // EOL: 04/2026
-		"bionic":  "golang-go",   // EOL: 04/2028
+		"bionic": "golang-go",   // EOL: 04/2028
-		"focal":   "golang-go",   // EOL: 04/2030
+		"focal":  "golang-go",   // EOL: 04/2030
-		"jammy":   "golang-go",   // EOL: 04/2032
+		"jammy":  "golang-go",   // EOL: 04/2032
-		"kinetic": "golang-go",   // EOL: 07/2023
+		"lunar":  "golang-go",   // EOL: 01/2024
-		"lunar":   "golang-go",   // EOL: 01/2024
 	}
 
 	debGoBootPaths = map[string]string{
@@ -139,7 +139,7 @@ var (
 	// This is the version of Go that will be downloaded by
 	//
 	//     go run ci.go install -dlgo
-	dlgoVersion = "1.20.7"
+	dlgoVersion = "1.21.1"
 
 	// This is the version of Go that will be used to bootstrap the PPA builder.
 	//
@@ -148,6 +148,13 @@ var (
 	// we need to switch over to a recursive builder to jumpt across supported
 	// versions.
 	gobootVersion = "1.19.6"
+
+	// This is the version of execution-spec-tests that we are using.
+	// When updating, you must also update build/checksums.txt.
+	executionSpecTestsVersion = "1.0.2"
+
+	// This is where the tests should be unpacked.
+	executionSpecTestsDir = "tests/spec-tests"
 )
 
 var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin"))
@@ -294,13 +301,17 @@ func doTest(cmdline []string) {
 		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
 		verbose  = flag.Bool("v", false, "Whether to log verbosely")
 		race     = flag.Bool("race", false, "Execute the race detector")
+		cachedir = flag.String("cachedir", "./build/cache", "directory for caching downloads")
 	)
 	flag.CommandLine.Parse(cmdline)
 
+	// Get test fixtures.
+	csdb := build.MustLoadChecksums("build/checksums.txt")
+	downloadSpecTestFixtures(csdb, *cachedir)
+
 	// Configure the toolchain.
 	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
 	if *dlgo {
-		csdb := build.MustLoadChecksums("build/checksums.txt")
 		tc.Root = build.DownloadGo(csdb, dlgoVersion)
 	}
 	gotest := tc.Go("test")
@@ -332,6 +343,21 @@ func doTest(cmdline []string) {
 	build.MustRun(gotest)
 }
 
+// downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
+func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string {
+	ext := ".tar.gz"
+	base := "fixtures" // TODO(MariusVanDerWijden) rename once the version becomes part of the filename
+	url := fmt.Sprintf("https://github.com/ethereum/execution-spec-tests/releases/download/v%s/%s%s", executionSpecTestsVersion, base, ext)
+	archivePath := filepath.Join(cachedir, base+ext)
+	if err := csdb.DownloadFile(url, archivePath); err != nil {
+		log.Fatal(err)
+	}
+	if err := build.ExtractArchive(archivePath, executionSpecTestsDir); err != nil {
+		log.Fatal(err)
+	}
+	return filepath.Join(cachedir, base)
+}
+
 // doLint runs golangci-lint on requested packages.
 func doLint(cmdline []string) {
 	var (

cmd/devp2p/README.md

@@ -44,7 +44,7 @@ set to standard output. The following filters are supported:
 - `-limit <N>` limits the output set to N entries, taking the top N nodes by score
 - `-ip <CIDR>` filters nodes by IP subnet
 - `-min-age <duration>` filters nodes by 'first seen' time
-- `-eth-network <mainnet/goerli/sepolia>` filters nodes by "eth" ENR entry
+- `-eth-network <mainnet/goerli/sepolia/holesky>` filters nodes by "eth" ENR entry
 - `-les-server` filters nodes by LES server support
 - `-snap` filters nodes by snap protocol support
 

cmd/devp2p/crawl.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"errors"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -51,7 +52,14 @@ type resolver interface {
 	RequestENR(*enode.Node) (*enode.Node, error)
 }
 
-func newCrawler(input nodeSet, disc resolver, iters ...enode.Iterator) *crawler {
+func newCrawler(input nodeSet, bootnodes []*enode.Node, disc resolver, iters ...enode.Iterator) (*crawler, error) {
+	if len(input) == 0 {
+		input.add(bootnodes...)
+	}
+	if len(input) == 0 {
+		return nil, errors.New("no input nodes to start crawling")
+	}
+
 	c := &crawler{
 		input:     input,
 		output:    make(nodeSet, len(input)),
@@ -67,7 +75,7 @@ func newCrawler(input nodeSet, disc resolver, iters ...enode.Iterator) *crawler
 	for id, n := range input {
 		c.output[id] = n
 	}
-	return c
+	return c, nil
 }
 
 func (c *crawler) run(timeout time.Duration, nthreads int) nodeSet {

cmd/devp2p/discv4cmd.go

@@ -143,7 +143,7 @@ var discoveryNodeFlags = []cli.Flag{
 
 func discv4Ping(ctx *cli.Context) error {
 	n := getNodeArg(ctx)
-	disc := startV4(ctx)
+	disc, _ := startV4(ctx)
 	defer disc.Close()
 
 	start := time.Now()
@@ -156,7 +156,7 @@ func discv4Ping(ctx *cli.Context) error {
 
 func discv4RequestRecord(ctx *cli.Context) error {
 	n := getNodeArg(ctx)
-	disc := startV4(ctx)
+	disc, _ := startV4(ctx)
 	defer disc.Close()
 
 	respN, err := disc.RequestENR(n)
@@ -169,7 +169,7 @@ func discv4RequestRecord(ctx *cli.Context) error {
 
 func discv4Resolve(ctx *cli.Context) error {
 	n := getNodeArg(ctx)
-	disc := startV4(ctx)
+	disc, _ := startV4(ctx)
 	defer disc.Close()
 
 	fmt.Println(disc.Resolve(n).String())
@@ -196,10 +196,13 @@ func discv4ResolveJSON(ctx *cli.Context) error {
 		nodeargs = append(nodeargs, n)
 	}
 
-	// Run the crawler.
+	disc, config := startV4(ctx)
-	disc := startV4(ctx)
 	defer disc.Close()
-	c := newCrawler(inputSet, disc, enode.IterNodes(nodeargs))
+
+	c, err := newCrawler(inputSet, config.Bootnodes, disc, enode.IterNodes(nodeargs))
+	if err != nil {
+		return err
+	}
 	c.revalidateInterval = 0
 	output := c.run(0, 1)
 	writeNodesJSON(nodesFile, output)
@@ -211,14 +214,18 @@ func discv4Crawl(ctx *cli.Context) error {
 		return errors.New("need nodes file as argument")
 	}
 	nodesFile := ctx.Args().First()
-	var inputSet nodeSet
+	inputSet := make(nodeSet)
 	if common.FileExist(nodesFile) {
 		inputSet = loadNodesJSON(nodesFile)
 	}
 
-	disc := startV4(ctx)
+	disc, config := startV4(ctx)
 	defer disc.Close()
-	c := newCrawler(inputSet, disc, disc.RandomNodes())
+
+	c, err := newCrawler(inputSet, config.Bootnodes, disc, disc.RandomNodes())
+	if err != nil {
+		return err
+	}
 	c.revalidateInterval = 10 * time.Minute
 	output := c.run(ctx.Duration(crawlTimeoutFlag.Name), ctx.Int(crawlParallelismFlag.Name))
 	writeNodesJSON(nodesFile, output)
@@ -238,14 +245,14 @@ func discv4Test(ctx *cli.Context) error {
 }
 
 // startV4 starts an ephemeral discovery V4 node.
-func startV4(ctx *cli.Context) *discover.UDPv4 {
+func startV4(ctx *cli.Context) (*discover.UDPv4, discover.Config) {
 	ln, config := makeDiscoveryConfig(ctx)
 	socket := listen(ctx, ln)
 	disc, err := discover.ListenV4(socket, ln, config)
 	if err != nil {
 		exit(err)
 	}
-	return disc
+	return disc, config
 }
 
 func makeDiscoveryConfig(ctx *cli.Context) (*enode.LocalNode, discover.Config) {

cmd/devp2p/discv5cmd.go

@@ -81,7 +81,7 @@ var (
 
 func discv5Ping(ctx *cli.Context) error {
 	n := getNodeArg(ctx)
-	disc := startV5(ctx)
+	disc, _ := startV5(ctx)
 	defer disc.Close()
 
 	fmt.Println(disc.Ping(n))
@@ -90,7 +90,7 @@ func discv5Ping(ctx *cli.Context) error {
 
 func discv5Resolve(ctx *cli.Context) error {
 	n := getNodeArg(ctx)
-	disc := startV5(ctx)
+	disc, _ := startV5(ctx)
 	defer disc.Close()
 
 	fmt.Println(disc.Resolve(n))
@@ -102,14 +102,18 @@ func discv5Crawl(ctx *cli.Context) error {
 		return errors.New("need nodes file as argument")
 	}
 	nodesFile := ctx.Args().First()
-	var inputSet nodeSet
+	inputSet := make(nodeSet)
 	if common.FileExist(nodesFile) {
 		inputSet = loadNodesJSON(nodesFile)
 	}
 
-	disc := startV5(ctx)
+	disc, config := startV5(ctx)
 	defer disc.Close()
-	c := newCrawler(inputSet, disc, disc.RandomNodes())
+
+	c, err := newCrawler(inputSet, config.Bootnodes, disc, disc.RandomNodes())
+	if err != nil {
+		return err
+	}
 	c.revalidateInterval = 10 * time.Minute
 	output := c.run(ctx.Duration(crawlTimeoutFlag.Name), ctx.Int(crawlParallelismFlag.Name))
 	writeNodesJSON(nodesFile, output)
@@ -127,7 +131,7 @@ func discv5Test(ctx *cli.Context) error {
 }
 
 func discv5Listen(ctx *cli.Context) error {
-	disc := startV5(ctx)
+	disc, _ := startV5(ctx)
 	defer disc.Close()
 
 	fmt.Println(disc.Self())
@@ -135,12 +139,12 @@ func discv5Listen(ctx *cli.Context) error {
 }
 
 // startV5 starts an ephemeral discovery v5 node.
-func startV5(ctx *cli.Context) *discover.UDPv5 {
+func startV5(ctx *cli.Context) (*discover.UDPv5, discover.Config) {
 	ln, config := makeDiscoveryConfig(ctx)
 	socket := listen(ctx, ln)
 	disc, err := discover.ListenV5(socket, ln, config)
 	if err != nil {
 		exit(err)
 	}
-	return disc
+	return disc, config
 }

cmd/devp2p/internal/ethtest/chain.go

@@ -77,7 +77,7 @@ func (c *Chain) RootAt(height int) common.Hash {
 
 // ForkID gets the fork id of the chain.
 func (c *Chain) ForkID() forkid.ID {
-	return forkid.NewID(c.chainConfig, c.blocks[0].Hash(), uint64(c.Len()), c.blocks[0].Time())
+	return forkid.NewID(c.chainConfig, c.blocks[0], uint64(c.Len()), c.blocks[0].Time())
 }
 
 // Shorten returns a copy chain of a desired height from the imported

cmd/devp2p/internal/ethtest/suite_test.go

@@ -120,6 +120,7 @@ func setupGeth(stack *node.Node) error {
 	if err != nil {
 		return err
 	}
+	backend.SetSynced()
 
 	_, err = backend.BlockChain().InsertChain(chain.blocks[1:])
 	return err

cmd/devp2p/nodesetcmd.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/forkid"
 	"github.com/ethereum/go-ethereum/p2p/enr"
 	"github.com/ethereum/go-ethereum/params"
@@ -228,11 +229,13 @@ func ethFilter(args []string) (nodeFilter, error) {
 	var filter forkid.Filter
 	switch args[0] {
 	case "mainnet":
-		filter = forkid.NewStaticFilter(params.MainnetChainConfig, params.MainnetGenesisHash)
+		filter = forkid.NewStaticFilter(params.MainnetChainConfig, core.DefaultGenesisBlock().ToBlock())
 	case "goerli":
-		filter = forkid.NewStaticFilter(params.GoerliChainConfig, params.GoerliGenesisHash)
+		filter = forkid.NewStaticFilter(params.GoerliChainConfig, core.DefaultGoerliGenesisBlock().ToBlock())
 	case "sepolia":
-		filter = forkid.NewStaticFilter(params.SepoliaChainConfig, params.SepoliaGenesisHash)
+		filter = forkid.NewStaticFilter(params.SepoliaChainConfig, core.DefaultSepoliaGenesisBlock().ToBlock())
+	case "holesky":
+		filter = forkid.NewStaticFilter(params.HoleskyChainConfig, core.DefaultHoleskyGenesisBlock().ToBlock())
 	default:
 		return nil, fmt.Errorf("unknown network %q", args[0])
 	}

cmd/evm/blockrunner.go

@@ -22,9 +22,9 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/tests"
 	"github.com/urfave/cli/v2"
 )
@@ -40,10 +40,7 @@ func blockTestCmd(ctx *cli.Context) error {
 	if len(ctx.Args().First()) == 0 {
 		return errors.New("path-to-test argument required")
 	}
-	// Configure the go-ethereum logger
+
-	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
-	glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name)))
-	log.Root().SetHandler(glogger)
 	var tracer vm.EVMLogger
 	// Configure the EVM logger
 	if ctx.Bool(MachineFlag.Name) {
@@ -64,7 +61,7 @@ func blockTestCmd(ctx *cli.Context) error {
 		return err
 	}
 	for i, test := range tests {
-		if err := test.Run(false, tracer); err != nil {
+		if err := test.Run(false, rawdb.HashScheme, tracer); err != nil {
 			return fmt.Errorf("test %v: %w", i, err)
 		}
 	}

cmd/evm/internal/t8ntool/execution.go

@@ -69,25 +69,26 @@ type ommer struct {
 
 //go:generate go run github.com/fjl/gencodec -type stEnv -field-override stEnvMarshaling -out gen_stenv.go
 type stEnv struct {
-	Coinbase            common.Address                      `json:"currentCoinbase"   gencodec:"required"`
+	Coinbase              common.Address                      `json:"currentCoinbase"   gencodec:"required"`
-	Difficulty          *big.Int                            `json:"currentDifficulty"`
+	Difficulty            *big.Int                            `json:"currentDifficulty"`
-	Random              *big.Int                            `json:"currentRandom"`
+	Random                *big.Int                            `json:"currentRandom"`
-	ParentDifficulty    *big.Int                            `json:"parentDifficulty"`
+	ParentDifficulty      *big.Int                            `json:"parentDifficulty"`
-	ParentBaseFee       *big.Int                            `json:"parentBaseFee,omitempty"`
+	ParentBaseFee         *big.Int                            `json:"parentBaseFee,omitempty"`
-	ParentGasUsed       uint64                              `json:"parentGasUsed,omitempty"`
+	ParentGasUsed         uint64                              `json:"parentGasUsed,omitempty"`
-	ParentGasLimit      uint64                              `json:"parentGasLimit,omitempty"`
+	ParentGasLimit        uint64                              `json:"parentGasLimit,omitempty"`
-	GasLimit            uint64                              `json:"currentGasLimit"   gencodec:"required"`
+	GasLimit              uint64                              `json:"currentGasLimit"   gencodec:"required"`
-	Number              uint64                              `json:"currentNumber"     gencodec:"required"`
+	Number                uint64                              `json:"currentNumber"     gencodec:"required"`
-	Timestamp           uint64                              `json:"currentTimestamp"  gencodec:"required"`
+	Timestamp             uint64                              `json:"currentTimestamp"  gencodec:"required"`
-	ParentTimestamp     uint64                              `json:"parentTimestamp,omitempty"`
+	ParentTimestamp       uint64                              `json:"parentTimestamp,omitempty"`
-	BlockHashes         map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
+	BlockHashes           map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
-	Ommers              []ommer                             `json:"ommers,omitempty"`
+	Ommers                []ommer                             `json:"ommers,omitempty"`
-	Withdrawals         []*types.Withdrawal                 `json:"withdrawals,omitempty"`
+	Withdrawals           []*types.Withdrawal                 `json:"withdrawals,omitempty"`
-	BaseFee             *big.Int                            `json:"currentBaseFee,omitempty"`
+	BaseFee               *big.Int                            `json:"currentBaseFee,omitempty"`
-	ParentUncleHash     common.Hash                         `json:"parentUncleHash"`
+	ParentUncleHash       common.Hash                         `json:"parentUncleHash"`
-	ExcessBlobGas       *uint64                             `json:"excessBlobGas,omitempty"`
+	ExcessBlobGas         *uint64                             `json:"excessBlobGas,omitempty"`
-	ParentExcessBlobGas *uint64                             `json:"parentExcessBlobGas,omitempty"`
+	ParentExcessBlobGas   *uint64                             `json:"parentExcessBlobGas,omitempty"`
-	ParentBlobGasUsed   *uint64                             `json:"parentBlobGasUsed,omitempty"`
+	ParentBlobGasUsed     *uint64                             `json:"parentBlobGasUsed,omitempty"`
+	ParentBeaconBlockRoot *common.Hash                        `json:"parentBeaconBlockRoot"`
 }
 
 type stEnvMarshaling struct {
@@ -182,6 +183,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		chainConfig.DAOForkBlock.Cmp(new(big.Int).SetUint64(pre.Env.Number)) == 0 {
 		misc.ApplyDAOHardFork(statedb)
 	}
+	if beaconRoot := pre.Env.ParentBeaconBlockRoot; beaconRoot != nil {
+		evm := vm.NewEVM(vmContext, vm.TxContext{}, statedb, chainConfig, vmConfig)
+		core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb)
+	}
 	var blobGasUsed uint64
 	for i, tx := range txs {
 		if tx.Type() == types.BlobTxType && vmContext.ExcessBlobGas == nil {

cmd/evm/internal/t8ntool/gen_stenv.go

@@ -17,25 +17,26 @@ var _ = (*stEnvMarshaling)(nil)
 // MarshalJSON marshals as JSON.
 func (s stEnv) MarshalJSON() ([]byte, error) {
 	type stEnv struct {
-		Coinbase            common.UnprefixedAddress            `json:"currentCoinbase"   gencodec:"required"`
+		Coinbase              common.UnprefixedAddress            `json:"currentCoinbase"   gencodec:"required"`
-		Difficulty          *math.HexOrDecimal256               `json:"currentDifficulty"`
+		Difficulty            *math.HexOrDecimal256               `json:"currentDifficulty"`
-		Random              *math.HexOrDecimal256               `json:"currentRandom"`
+		Random                *math.HexOrDecimal256               `json:"currentRandom"`
-		ParentDifficulty    *math.HexOrDecimal256               `json:"parentDifficulty"`
+		ParentDifficulty      *math.HexOrDecimal256               `json:"parentDifficulty"`
-		ParentBaseFee       *math.HexOrDecimal256               `json:"parentBaseFee,omitempty"`
+		ParentBaseFee         *math.HexOrDecimal256               `json:"parentBaseFee,omitempty"`
-		ParentGasUsed       math.HexOrDecimal64                 `json:"parentGasUsed,omitempty"`
+		ParentGasUsed         math.HexOrDecimal64                 `json:"parentGasUsed,omitempty"`
-		ParentGasLimit      math.HexOrDecimal64                 `json:"parentGasLimit,omitempty"`
+		ParentGasLimit        math.HexOrDecimal64                 `json:"parentGasLimit,omitempty"`
-		GasLimit            math.HexOrDecimal64                 `json:"currentGasLimit"   gencodec:"required"`
+		GasLimit              math.HexOrDecimal64                 `json:"currentGasLimit"   gencodec:"required"`
-		Number              math.HexOrDecimal64                 `json:"currentNumber"     gencodec:"required"`
+		Number                math.HexOrDecimal64                 `json:"currentNumber"     gencodec:"required"`
-		Timestamp           math.HexOrDecimal64                 `json:"currentTimestamp"  gencodec:"required"`
+		Timestamp             math.HexOrDecimal64                 `json:"currentTimestamp"  gencodec:"required"`
-		ParentTimestamp     math.HexOrDecimal64                 `json:"parentTimestamp,omitempty"`
+		ParentTimestamp       math.HexOrDecimal64                 `json:"parentTimestamp,omitempty"`
-		BlockHashes         map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
+		BlockHashes           map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
-		Ommers              []ommer                             `json:"ommers,omitempty"`
+		Ommers                []ommer                             `json:"ommers,omitempty"`
-		Withdrawals         []*types.Withdrawal                 `json:"withdrawals,omitempty"`
+		Withdrawals           []*types.Withdrawal                 `json:"withdrawals,omitempty"`
-		BaseFee             *math.HexOrDecimal256               `json:"currentBaseFee,omitempty"`
+		BaseFee               *math.HexOrDecimal256               `json:"currentBaseFee,omitempty"`
-		ParentUncleHash     common.Hash                         `json:"parentUncleHash"`
+		ParentUncleHash       common.Hash                         `json:"parentUncleHash"`
-		ExcessBlobGas       *math.HexOrDecimal64                `json:"excessBlobGas,omitempty"`
+		ExcessBlobGas         *math.HexOrDecimal64                `json:"excessBlobGas,omitempty"`
-		ParentExcessBlobGas *math.HexOrDecimal64                `json:"parentExcessBlobGas,omitempty"`
+		ParentExcessBlobGas   *math.HexOrDecimal64                `json:"parentExcessBlobGas,omitempty"`
-		ParentBlobGasUsed   *math.HexOrDecimal64                `json:"parentBlobGasUsed,omitempty"`
+		ParentBlobGasUsed     *math.HexOrDecimal64                `json:"parentBlobGasUsed,omitempty"`
+		ParentBeaconBlockRoot *common.Hash                        `json:"parentBeaconBlockRoot"`
 	}
 	var enc stEnv
 	enc.Coinbase = common.UnprefixedAddress(s.Coinbase)
@@ -57,31 +58,33 @@ func (s stEnv) MarshalJSON() ([]byte, error) {
 	enc.ExcessBlobGas = (*math.HexOrDecimal64)(s.ExcessBlobGas)
 	enc.ParentExcessBlobGas = (*math.HexOrDecimal64)(s.ParentExcessBlobGas)
 	enc.ParentBlobGasUsed = (*math.HexOrDecimal64)(s.ParentBlobGasUsed)
+	enc.ParentBeaconBlockRoot = s.ParentBeaconBlockRoot
 	return json.Marshal(&enc)
 }
 
 // UnmarshalJSON unmarshals from JSON.
 func (s *stEnv) UnmarshalJSON(input []byte) error {
 	type stEnv struct {
-		Coinbase            *common.UnprefixedAddress           `json:"currentCoinbase"   gencodec:"required"`
+		Coinbase              *common.UnprefixedAddress           `json:"currentCoinbase"   gencodec:"required"`
-		Difficulty          *math.HexOrDecimal256               `json:"currentDifficulty"`
+		Difficulty            *math.HexOrDecimal256               `json:"currentDifficulty"`
-		Random              *math.HexOrDecimal256               `json:"currentRandom"`
+		Random                *math.HexOrDecimal256               `json:"currentRandom"`
-		ParentDifficulty    *math.HexOrDecimal256               `json:"parentDifficulty"`
+		ParentDifficulty      *math.HexOrDecimal256               `json:"parentDifficulty"`
-		ParentBaseFee       *math.HexOrDecimal256               `json:"parentBaseFee,omitempty"`
+		ParentBaseFee         *math.HexOrDecimal256               `json:"parentBaseFee,omitempty"`
-		ParentGasUsed       *math.HexOrDecimal64                `json:"parentGasUsed,omitempty"`
+		ParentGasUsed         *math.HexOrDecimal64                `json:"parentGasUsed,omitempty"`
-		ParentGasLimit      *math.HexOrDecimal64                `json:"parentGasLimit,omitempty"`
+		ParentGasLimit        *math.HexOrDecimal64                `json:"parentGasLimit,omitempty"`
-		GasLimit            *math.HexOrDecimal64                `json:"currentGasLimit"   gencodec:"required"`
+		GasLimit              *math.HexOrDecimal64                `json:"currentGasLimit"   gencodec:"required"`
-		Number              *math.HexOrDecimal64                `json:"currentNumber"     gencodec:"required"`
+		Number                *math.HexOrDecimal64                `json:"currentNumber"     gencodec:"required"`
-		Timestamp           *math.HexOrDecimal64                `json:"currentTimestamp"  gencodec:"required"`
+		Timestamp             *math.HexOrDecimal64                `json:"currentTimestamp"  gencodec:"required"`
-		ParentTimestamp     *math.HexOrDecimal64                `json:"parentTimestamp,omitempty"`
+		ParentTimestamp       *math.HexOrDecimal64                `json:"parentTimestamp,omitempty"`
-		BlockHashes         map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
+		BlockHashes           map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"`
-		Ommers              []ommer                             `json:"ommers,omitempty"`
+		Ommers                []ommer                             `json:"ommers,omitempty"`
-		Withdrawals         []*types.Withdrawal                 `json:"withdrawals,omitempty"`
+		Withdrawals           []*types.Withdrawal                 `json:"withdrawals,omitempty"`
-		BaseFee             *math.HexOrDecimal256               `json:"currentBaseFee,omitempty"`
+		BaseFee               *math.HexOrDecimal256               `json:"currentBaseFee,omitempty"`
-		ParentUncleHash     *common.Hash                        `json:"parentUncleHash"`
+		ParentUncleHash       *common.Hash                        `json:"parentUncleHash"`
-		ExcessBlobGas       *math.HexOrDecimal64                `json:"excessBlobGas,omitempty"`
+		ExcessBlobGas         *math.HexOrDecimal64                `json:"excessBlobGas,omitempty"`
-		ParentExcessBlobGas *math.HexOrDecimal64                `json:"parentExcessBlobGas,omitempty"`
+		ParentExcessBlobGas   *math.HexOrDecimal64                `json:"parentExcessBlobGas,omitempty"`
-		ParentBlobGasUsed   *math.HexOrDecimal64                `json:"parentBlobGasUsed,omitempty"`
+		ParentBlobGasUsed     *math.HexOrDecimal64                `json:"parentBlobGasUsed,omitempty"`
+		ParentBeaconBlockRoot *common.Hash                        `json:"parentBeaconBlockRoot"`
 	}
 	var dec stEnv
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -148,5 +151,8 @@ func (s *stEnv) UnmarshalJSON(input []byte) error {
 	if dec.ParentBlobGasUsed != nil {
 		s.ParentBlobGasUsed = (*uint64)(dec.ParentBlobGasUsed)
 	}
+	if dec.ParentBeaconBlockRoot != nil {
+		s.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot
+	}
 	return nil
 }

cmd/evm/internal/t8ntool/transition.go

@@ -192,105 +192,20 @@ func Transition(ctx *cli.Context) error {
 	// Set the chain id
 	chainConfig.ChainID = big.NewInt(ctx.Int64(ChainIDFlag.Name))
 
-	var txsWithKeys []*txWithKey
+	if txs, err = loadTransactions(txStr, inputData, prestate.Env, chainConfig); err != nil {
-	if txStr != stdinSelector {
+		return err
-		inFile, err := os.Open(txStr)
-		if err != nil {
-			return NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err))
-		}
-		defer inFile.Close()
-		decoder := json.NewDecoder(inFile)
-		if strings.HasSuffix(txStr, ".rlp") {
-			var body hexutil.Bytes
-			if err := decoder.Decode(&body); err != nil {
-				return err
-			}
-			var txs types.Transactions
-			if err := rlp.DecodeBytes(body, &txs); err != nil {
-				return err
-			}
-			for _, tx := range txs {
-				txsWithKeys = append(txsWithKeys, &txWithKey{
-					key: nil,
-					tx:  tx,
-				})
-			}
-		} else {
-			if err := decoder.Decode(&txsWithKeys); err != nil {
-				return NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err))
-			}
-		}
-	} else {
-		if len(inputData.TxRlp) > 0 {
-			// Decode the body of already signed transactions
-			body := common.FromHex(inputData.TxRlp)
-			var txs types.Transactions
-			if err := rlp.DecodeBytes(body, &txs); err != nil {
-				return err
-			}
-			for _, tx := range txs {
-				txsWithKeys = append(txsWithKeys, &txWithKey{
-					key: nil,
-					tx:  tx,
-				})
-			}
-		} else {
-			// JSON encoded transactions
-			txsWithKeys = inputData.Txs
-		}
 	}
-	// We may have to sign the transactions.
+	if err := applyLondonChecks(&prestate.Env, chainConfig); err != nil {
-	signer := types.MakeSigner(chainConfig, big.NewInt(int64(prestate.Env.Number)), prestate.Env.Timestamp)
+		return err
-
-	if txs, err = signUnsignedTransactions(txsWithKeys, signer); err != nil {
-		return NewError(ErrorJson, fmt.Errorf("failed signing transactions: %v", err))
 	}
-	// Sanity check, to not `panic` in state_transition
+	if err := applyShanghaiChecks(&prestate.Env, chainConfig); err != nil {
-	if chainConfig.IsLondon(big.NewInt(int64(prestate.Env.Number))) {
+		return err
-		if prestate.Env.BaseFee != nil {
-			// Already set, base fee has precedent over parent base fee.
-		} else if prestate.Env.ParentBaseFee != nil && prestate.Env.Number != 0 {
-			parent := &types.Header{
-				Number:   new(big.Int).SetUint64(prestate.Env.Number - 1),
-				BaseFee:  prestate.Env.ParentBaseFee,
-				GasUsed:  prestate.Env.ParentGasUsed,
-				GasLimit: prestate.Env.ParentGasLimit,
-			}
-			prestate.Env.BaseFee = eip1559.CalcBaseFee(chainConfig, parent)
-		} else {
-			return NewError(ErrorConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section"))
-		}
 	}
-	if chainConfig.IsShanghai(big.NewInt(int64(prestate.Env.Number)), prestate.Env.Timestamp) && prestate.Env.Withdrawals == nil {
+	if err := applyMergeChecks(&prestate.Env, chainConfig); err != nil {
-		return NewError(ErrorConfig, errors.New("Shanghai config but missing 'withdrawals' in env section"))
+		return err
 	}
-	isMerged := chainConfig.TerminalTotalDifficulty != nil && chainConfig.TerminalTotalDifficulty.BitLen() == 0
+	if err := applyCancunChecks(&prestate.Env, chainConfig); err != nil {
-	env := prestate.Env
+		return err
-	if isMerged {
-		// post-merge:
-		// - random must be supplied
-		// - difficulty must be zero
-		switch {
-		case env.Random == nil:
-			return NewError(ErrorConfig, errors.New("post-merge requires currentRandom to be defined in env"))
-		case env.Difficulty != nil && env.Difficulty.BitLen() != 0:
-			return NewError(ErrorConfig, errors.New("post-merge difficulty must be zero (or omitted) in env"))
-		}
-		prestate.Env.Difficulty = nil
-	} else if env.Difficulty == nil {
-		// pre-merge:
-		// If difficulty was not provided by caller, we need to calculate it.
-		switch {
-		case env.ParentDifficulty == nil:
-			return NewError(ErrorConfig, errors.New("currentDifficulty was not provided, and cannot be calculated due to missing parentDifficulty"))
-		case env.Number == 0:
-			return NewError(ErrorConfig, errors.New("currentDifficulty needs to be provided for block number 0"))
-		case env.Timestamp <= env.ParentTimestamp:
-			return NewError(ErrorConfig, fmt.Errorf("currentDifficulty cannot be calculated -- currentTime (%d) needs to be after parent time (%d)",
-				env.Timestamp, env.ParentTimestamp))
-		}
-		prestate.Env.Difficulty = calcDifficulty(chainConfig, env.Number, env.Timestamp,
-			env.ParentTimestamp, env.ParentDifficulty, env.ParentUncleHash)
 	}
 	// Run the test and aggregate the result
 	s, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer)
@@ -358,33 +273,149 @@ func (t *txWithKey) UnmarshalJSON(input []byte) error {
 // and secondly to read them with the standard tx json format
 func signUnsignedTransactions(txs []*txWithKey, signer types.Signer) (types.Transactions, error) {
 	var signedTxs []*types.Transaction
-	for i, txWithKey := range txs {
+	for i, tx := range txs {
-		tx := txWithKey.tx
+		var (
-		key := txWithKey.key
+			v, r, s = tx.tx.RawSignatureValues()
-		v, r, s := tx.RawSignatureValues()
+			signed  *types.Transaction
-		if key != nil && v.BitLen()+r.BitLen()+s.BitLen() == 0 {
+			err     error
-			// This transaction needs to be signed
+		)
-			var (
+		if tx.key == nil || v.BitLen()+r.BitLen()+s.BitLen() != 0 {
-				signed *types.Transaction
-				err    error
-			)
-			if txWithKey.protected {
-				signed, err = types.SignTx(tx, signer, key)
-			} else {
-				signed, err = types.SignTx(tx, types.FrontierSigner{}, key)
-			}
-			if err != nil {
-				return nil, NewError(ErrorJson, fmt.Errorf("tx %d: failed to sign tx: %v", i, err))
-			}
-			signedTxs = append(signedTxs, signed)
-		} else {
 			// Already signed
-			signedTxs = append(signedTxs, tx)
+			signedTxs = append(signedTxs, tx.tx)
+			continue
 		}
+		// This transaction needs to be signed
+		if tx.protected {
+			signed, err = types.SignTx(tx.tx, signer, tx.key)
+		} else {
+			signed, err = types.SignTx(tx.tx, types.FrontierSigner{}, tx.key)
+		}
+		if err != nil {
+			return nil, NewError(ErrorJson, fmt.Errorf("tx %d: failed to sign tx: %v", i, err))
+		}
+		signedTxs = append(signedTxs, signed)
 	}
 	return signedTxs, nil
 }
 
+func loadTransactions(txStr string, inputData *input, env stEnv, chainConfig *params.ChainConfig) (types.Transactions, error) {
+	var txsWithKeys []*txWithKey
+	var signed types.Transactions
+	if txStr != stdinSelector {
+		data, err := os.ReadFile(txStr)
+		if err != nil {
+			return nil, NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err))
+		}
+		if strings.HasSuffix(txStr, ".rlp") { // A file containing an rlp list
+			var body hexutil.Bytes
+			if err := json.Unmarshal(data, &body); err != nil {
+				return nil, err
+			}
+			// Already signed transactions
+			if err := rlp.DecodeBytes(body, &signed); err != nil {
+				return nil, err
+			}
+			return signed, nil
+		}
+		if err := json.Unmarshal(data, &txsWithKeys); err != nil {
+			return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err))
+		}
+	} else {
+		if len(inputData.TxRlp) > 0 {
+			// Decode the body of already signed transactions
+			body := common.FromHex(inputData.TxRlp)
+			// Already signed transactions
+			if err := rlp.DecodeBytes(body, &signed); err != nil {
+				return nil, err
+			}
+			return signed, nil
+		}
+		// JSON encoded transactions
+		txsWithKeys = inputData.Txs
+	}
+	// We may have to sign the transactions.
+	signer := types.MakeSigner(chainConfig, big.NewInt(int64(env.Number)), env.Timestamp)
+	return signUnsignedTransactions(txsWithKeys, signer)
+}
+
+func applyLondonChecks(env *stEnv, chainConfig *params.ChainConfig) error {
+	if !chainConfig.IsLondon(big.NewInt(int64(env.Number))) {
+		return nil
+	}
+	// Sanity check, to not `panic` in state_transition
+	if env.BaseFee != nil {
+		// Already set, base fee has precedent over parent base fee.
+		return nil
+	}
+	if env.ParentBaseFee == nil || env.Number == 0 {
+		return NewError(ErrorConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section"))
+	}
+	env.BaseFee = eip1559.CalcBaseFee(chainConfig, &types.Header{
+		Number:   new(big.Int).SetUint64(env.Number - 1),
+		BaseFee:  env.ParentBaseFee,
+		GasUsed:  env.ParentGasUsed,
+		GasLimit: env.ParentGasLimit,
+	})
+	return nil
+}
+
+func applyShanghaiChecks(env *stEnv, chainConfig *params.ChainConfig) error {
+	if !chainConfig.IsShanghai(big.NewInt(int64(env.Number)), env.Timestamp) {
+		return nil
+	}
+	if env.Withdrawals == nil {
+		return NewError(ErrorConfig, errors.New("Shanghai config but missing 'withdrawals' in env section"))
+	}
+	return nil
+}
+
+func applyMergeChecks(env *stEnv, chainConfig *params.ChainConfig) error {
+	isMerged := chainConfig.TerminalTotalDifficulty != nil && chainConfig.TerminalTotalDifficulty.BitLen() == 0
+	if !isMerged {
+		// pre-merge: If difficulty was not provided by caller, we need to calculate it.
+		if env.Difficulty != nil {
+			// already set
+			return nil
+		}
+		switch {
+		case env.ParentDifficulty == nil:
+			return NewError(ErrorConfig, errors.New("currentDifficulty was not provided, and cannot be calculated due to missing parentDifficulty"))
+		case env.Number == 0:
+			return NewError(ErrorConfig, errors.New("currentDifficulty needs to be provided for block number 0"))
+		case env.Timestamp <= env.ParentTimestamp:
+			return NewError(ErrorConfig, fmt.Errorf("currentDifficulty cannot be calculated -- currentTime (%d) needs to be after parent time (%d)",
+				env.Timestamp, env.ParentTimestamp))
+		}
+		env.Difficulty = calcDifficulty(chainConfig, env.Number, env.Timestamp,
+			env.ParentTimestamp, env.ParentDifficulty, env.ParentUncleHash)
+		return nil
+	}
+	// post-merge:
+	// - random must be supplied
+	// - difficulty must be zero
+	switch {
+	case env.Random == nil:
+		return NewError(ErrorConfig, errors.New("post-merge requires currentRandom to be defined in env"))
+	case env.Difficulty != nil && env.Difficulty.BitLen() != 0:
+		return NewError(ErrorConfig, errors.New("post-merge difficulty must be zero (or omitted) in env"))
+	}
+	env.Difficulty = nil
+	return nil
+}
+
+func applyCancunChecks(env *stEnv, chainConfig *params.ChainConfig) error {
+	if !chainConfig.IsCancun(big.NewInt(int64(env.Number)), env.Timestamp) {
+		env.ParentBeaconBlockRoot = nil // un-set it if it has been set too early
+		return nil
+	}
+	// Post-cancun
+	// We require EIP-4788 beacon root to be set in the env
+	if env.ParentBeaconBlockRoot == nil {
+		return NewError(ErrorConfig, errors.New("post-cancun env requires parentBeaconBlockRoot to be set"))
+	}
+	return nil
+}
+
 type Alloc map[common.Address]core.GenesisAccount
 
 func (g Alloc) OnRoot(common.Hash) {}

cmd/evm/main.go

@@ -23,107 +23,116 @@ import (
 	"os"
 
 	"github.com/ethereum/go-ethereum/cmd/evm/internal/t8ntool"
+	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/urfave/cli/v2"
 )
 
 var (
 	DebugFlag = &cli.BoolFlag{
-		Name:  "debug",
+		Name:     "debug",
-		Usage: "output full trace logs",
+		Usage:    "output full trace logs",
-	}
+		Category: flags.VMCategory,
-	MemProfileFlag = &cli.StringFlag{
-		Name:  "memprofile",
-		Usage: "creates a memory profile at the given path",
-	}
-	CPUProfileFlag = &cli.StringFlag{
-		Name:  "cpuprofile",
-		Usage: "creates a CPU profile at the given path",
 	}
 	StatDumpFlag = &cli.BoolFlag{
-		Name:  "statdump",
+		Name:     "statdump",
-		Usage: "displays stack and heap memory information",
+		Usage:    "displays stack and heap memory information",
+		Category: flags.VMCategory,
 	}
 	CodeFlag = &cli.StringFlag{
-		Name:  "code",
+		Name:     "code",
-		Usage: "EVM code",
+		Usage:    "EVM code",
+		Category: flags.VMCategory,
 	}
 	CodeFileFlag = &cli.StringFlag{
-		Name:  "codefile",
+		Name:     "codefile",
-		Usage: "File containing EVM code. If '-' is specified, code is read from stdin ",
+		Usage:    "File containing EVM code. If '-' is specified, code is read from stdin ",
+		Category: flags.VMCategory,
 	}
 	GasFlag = &cli.Uint64Flag{
-		Name:  "gas",
+		Name:     "gas",
-		Usage: "gas limit for the evm",
+		Usage:    "gas limit for the evm",
-		Value: 10000000000,
+		Value:    10000000000,
+		Category: flags.VMCategory,
 	}
 	PriceFlag = &flags.BigFlag{
-		Name:  "price",
+		Name:     "price",
-		Usage: "price set for the evm",
+		Usage:    "price set for the evm",
-		Value: new(big.Int),
+		Value:    new(big.Int),
+		Category: flags.VMCategory,
 	}
 	ValueFlag = &flags.BigFlag{
-		Name:  "value",
+		Name:     "value",
-		Usage: "value set for the evm",
+		Usage:    "value set for the evm",
-		Value: new(big.Int),
+		Value:    new(big.Int),
+		Category: flags.VMCategory,
 	}
 	DumpFlag = &cli.BoolFlag{
-		Name:  "dump",
+		Name:     "dump",
-		Usage: "dumps the state after the run",
+		Usage:    "dumps the state after the run",
+		Category: flags.VMCategory,
 	}
 	InputFlag = &cli.StringFlag{
-		Name:  "input",
+		Name:     "input",
-		Usage: "input for the EVM",
+		Usage:    "input for the EVM",
+		Category: flags.VMCategory,
 	}
 	InputFileFlag = &cli.StringFlag{
-		Name:  "inputfile",
+		Name:     "inputfile",
-		Usage: "file containing input for the EVM",
+		Usage:    "file containing input for the EVM",
-	}
+		Category: flags.VMCategory,
-	VerbosityFlag = &cli.IntFlag{
-		Name:  "verbosity",
-		Usage: "sets the verbosity level",
 	}
 	BenchFlag = &cli.BoolFlag{
-		Name:  "bench",
+		Name:     "bench",
-		Usage: "benchmark the execution",
+		Usage:    "benchmark the execution",
+		Category: flags.VMCategory,
 	}
 	CreateFlag = &cli.BoolFlag{
-		Name:  "create",
+		Name:     "create",
-		Usage: "indicates the action should be create rather than call",
+		Usage:    "indicates the action should be create rather than call",
+		Category: flags.VMCategory,
 	}
 	GenesisFlag = &cli.StringFlag{
-		Name:  "prestate",
+		Name:     "prestate",
-		Usage: "JSON file with prestate (genesis) config",
+		Usage:    "JSON file with prestate (genesis) config",
+		Category: flags.VMCategory,
 	}
 	MachineFlag = &cli.BoolFlag{
-		Name:  "json",
+		Name:     "json",
-		Usage: "output trace logs in machine readable format (json)",
+		Usage:    "output trace logs in machine readable format (json)",
+		Category: flags.VMCategory,
 	}
 	SenderFlag = &cli.StringFlag{
-		Name:  "sender",
+		Name:     "sender",
-		Usage: "The transaction origin",
+		Usage:    "The transaction origin",
+		Category: flags.VMCategory,
 	}
 	ReceiverFlag = &cli.StringFlag{
-		Name:  "receiver",
+		Name:     "receiver",
-		Usage: "The transaction receiver (execution context)",
+		Usage:    "The transaction receiver (execution context)",
+		Category: flags.VMCategory,
 	}
 	DisableMemoryFlag = &cli.BoolFlag{
-		Name:  "nomemory",
+		Name:     "nomemory",
-		Value: true,
+		Value:    true,
-		Usage: "disable memory output",
+		Usage:    "disable memory output",
+		Category: flags.VMCategory,
 	}
 	DisableStackFlag = &cli.BoolFlag{
-		Name:  "nostack",
+		Name:     "nostack",
-		Usage: "disable stack output",
+		Usage:    "disable stack output",
+		Category: flags.VMCategory,
 	}
 	DisableStorageFlag = &cli.BoolFlag{
-		Name:  "nostorage",
+		Name:     "nostorage",
-		Usage: "disable storage output",
+		Usage:    "disable storage output",
+		Category: flags.VMCategory,
 	}
 	DisableReturnDataFlag = &cli.BoolFlag{
-		Name:  "noreturndata",
+		Name:     "noreturndata",
-		Value: true,
+		Value:    true,
-		Usage: "enable return data output",
+		Usage:    "enable return data output",
+		Category: flags.VMCategory,
 	}
 )
 
@@ -183,34 +192,38 @@ var blockBuilderCommand = &cli.Command{
 	},
 }
 
+// vmFlags contains flags related to running the EVM.
+var vmFlags = []cli.Flag{
+	CodeFlag,
+	CodeFileFlag,
+	CreateFlag,
+	GasFlag,
+	PriceFlag,
+	ValueFlag,
+	InputFlag,
+	InputFileFlag,
+	GenesisFlag,
+	SenderFlag,
+	ReceiverFlag,
+}
+
+// traceFlags contains flags that configure tracing output.
+var traceFlags = []cli.Flag{
+	BenchFlag,
+	DebugFlag,
+	DumpFlag,
+	MachineFlag,
+	StatDumpFlag,
+	DisableMemoryFlag,
+	DisableStackFlag,
+	DisableStorageFlag,
+	DisableReturnDataFlag,
+}
+
 var app = flags.NewApp("the evm command line interface")
 
 func init() {
-	app.Flags = []cli.Flag{
+	app.Flags = flags.Merge(vmFlags, traceFlags, debug.Flags)
-		BenchFlag,
-		CreateFlag,
-		DebugFlag,
-		VerbosityFlag,
-		CodeFlag,
-		CodeFileFlag,
-		GasFlag,
-		PriceFlag,
-		ValueFlag,
-		DumpFlag,
-		InputFlag,
-		InputFileFlag,
-		MemProfileFlag,
-		CPUProfileFlag,
-		StatDumpFlag,
-		GenesisFlag,
-		MachineFlag,
-		SenderFlag,
-		ReceiverFlag,
-		DisableMemoryFlag,
-		DisableStackFlag,
-		DisableStorageFlag,
-		DisableReturnDataFlag,
-	}
 	app.Commands = []*cli.Command{
 		compileCommand,
 		disasmCommand,
@@ -221,6 +234,14 @@ func init() {
 		transactionCommand,
 		blockBuilderCommand,
 	}
+	app.Before = func(ctx *cli.Context) error {
+		flags.MigrateGlobalFlags(ctx)
+		return debug.Setup(ctx)
+	}
+	app.After = func(ctx *cli.Context) error {
+		debug.Exit()
+		return nil
+	}
 }
 
 func main() {

cmd/evm/runner.go

@@ -24,7 +24,6 @@ import (
 	"math/big"
 	"os"
 	goruntime "runtime"
-	"runtime/pprof"
 	"testing"
 	"time"
 
@@ -34,14 +33,13 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/core/vm/runtime"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
 	"github.com/ethereum/go-ethereum/internal/flags"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
 	"github.com/urfave/cli/v2"
 )
 
@@ -51,6 +49,7 @@ var runCommand = &cli.Command{
 	Usage:       "run arbitrary evm binary",
 	ArgsUsage:   "<code>",
 	Description: `The run command runs arbitrary EVM code.`,
+	Flags:       flags.Merge(vmFlags, traceFlags),
 }
 
 // readGenesis will read the given JSON format genesis file and return
@@ -108,9 +107,6 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []by
 }
 
 func runCmd(ctx *cli.Context) error {
-	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
-	glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name)))
-	log.Root().SetHandler(glogger)
 	logconfig := &logger.Config{
 		EnableMemory:     !ctx.Bool(DisableMemoryFlag.Name),
 		DisableStack:     ctx.Bool(DisableStackFlag.Name),
@@ -120,15 +116,14 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	var (
-		tracer        vm.EVMLogger
+		tracer      vm.EVMLogger
-		debugLogger   *logger.StructLogger
+		debugLogger *logger.StructLogger
-		statedb       *state.StateDB
+		statedb     *state.StateDB
-		chainConfig   *params.ChainConfig
+		chainConfig *params.ChainConfig
-		sender        = common.BytesToAddress([]byte("sender"))
+		sender      = common.BytesToAddress([]byte("sender"))
-		receiver      = common.BytesToAddress([]byte("receiver"))
+		receiver    = common.BytesToAddress([]byte("receiver"))
-		genesisConfig *core.Genesis
+		preimages   = ctx.Bool(DumpFlag.Name)
-		preimages     = ctx.Bool(DumpFlag.Name)
+		blobHashes  []common.Hash // TODO (MariusVanDerWijden) implement blob hashes in state tests
-		blobHashes    []common.Hash // TODO (MariusVanDerWijden) implement blob hashes in state tests
 	)
 	if ctx.Bool(MachineFlag.Name) {
 		tracer = logger.NewJSONLogger(logconfig, os.Stdout)
@@ -138,19 +133,30 @@ func runCmd(ctx *cli.Context) error {
 	} else {
 		debugLogger = logger.NewStructLogger(logconfig)
 	}
+
+	initialGas := ctx.Uint64(GasFlag.Name)
+	genesisConfig := new(core.Genesis)
+	genesisConfig.GasLimit = initialGas
 	if ctx.String(GenesisFlag.Name) != "" {
-		gen := readGenesis(ctx.String(GenesisFlag.Name))
+		genesisConfig = readGenesis(ctx.String(GenesisFlag.Name))
-		genesisConfig = gen
+		if genesisConfig.GasLimit != 0 {
-		db := rawdb.NewMemoryDatabase()
+			initialGas = genesisConfig.GasLimit
-		genesis := gen.MustCommit(db)
+		}
-		sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: preimages})
-		statedb, _ = state.New(genesis.Root(), sdb, nil)
-		chainConfig = gen.Config
 	} else {
-		sdb := state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: preimages})
+		genesisConfig.Config = params.AllEthashProtocolChanges
-		statedb, _ = state.New(types.EmptyRootHash, sdb, nil)
-		genesisConfig = new(core.Genesis)
 	}
+
+	db := rawdb.NewMemoryDatabase()
+	triedb := trie.NewDatabase(db, &trie.Config{
+		Preimages: preimages,
+		HashDB:    hashdb.Defaults,
+	})
+	defer triedb.Close()
+	genesis := genesisConfig.MustCommit(db, triedb)
+	sdb := state.NewDatabaseWithNodeDB(db, triedb)
+	statedb, _ = state.New(genesis.Root(), sdb, nil)
+	chainConfig = genesisConfig.Config
+
 	if ctx.String(SenderFlag.Name) != "" {
 		sender = common.HexToAddress(ctx.String(SenderFlag.Name))
 	}
@@ -204,10 +210,6 @@ func runCmd(ctx *cli.Context) error {
 		}
 		code = common.Hex2Bytes(bin)
 	}
-	initialGas := ctx.Uint64(GasFlag.Name)
-	if genesisConfig.GasLimit != 0 {
-		initialGas = genesisConfig.GasLimit
-	}
 	runtimeConfig := runtime.Config{
 		Origin:      sender,
 		State:       statedb,
@@ -224,19 +226,6 @@ func runCmd(ctx *cli.Context) error {
 		},
 	}
 
-	if cpuProfilePath := ctx.String(CPUProfileFlag.Name); cpuProfilePath != "" {
-		f, err := os.Create(cpuProfilePath)
-		if err != nil {
-			fmt.Println("could not create CPU profile: ", err)
-			os.Exit(1)
-		}
-		if err := pprof.StartCPUProfile(f); err != nil {
-			fmt.Println("could not start CPU profile: ", err)
-			os.Exit(1)
-		}
-		defer pprof.StopCPUProfile()
-	}
-
 	if chainConfig != nil {
 		runtimeConfig.ChainConfig = chainConfig
 	} else {
@@ -284,19 +273,6 @@ func runCmd(ctx *cli.Context) error {
 		fmt.Println(string(statedb.Dump(nil)))
 	}
 
-	if memProfilePath := ctx.String(MemProfileFlag.Name); memProfilePath != "" {
-		f, err := os.Create(memProfilePath)
-		if err != nil {
-			fmt.Println("could not create memory profile: ", err)
-			os.Exit(1)
-		}
-		if err := pprof.WriteHeapProfile(f); err != nil {
-			fmt.Println("could not write memory profile: ", err)
-			os.Exit(1)
-		}
-		f.Close()
-	}
-
 	if ctx.Bool(DebugFlag.Name) {
 		if debugLogger != nil {
 			fmt.Fprintln(os.Stderr, "#### TRACE ####")

cmd/evm/staterunner.go

@@ -23,10 +23,11 @@ import (
 	"os"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/tests"
 	"github.com/urfave/cli/v2"
 )
@@ -50,11 +51,6 @@ type StatetestResult struct {
 }
 
 func stateTestCmd(ctx *cli.Context) error {
-	// Configure the go-ethereum logger
-	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
-	glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name)))
-	log.Root().SetHandler(glogger)
-
 	// Configure the EVM logger
 	config := &logger.Config{
 		EnableMemory:     !ctx.Bool(DisableMemoryFlag.Name),
@@ -104,25 +100,23 @@ func runStateTest(fname string, cfg vm.Config, jsonOut, dump bool) error {
 		for _, st := range test.Subtests() {
 			// Run the test and aggregate the result
 			result := &StatetestResult{Name: key, Fork: st.Fork, Pass: true}
-			_, s, err := test.Run(st, cfg, false)
+			test.Run(st, cfg, false, rawdb.HashScheme, func(err error, snaps *snapshot.Tree, state *state.StateDB) {
-			// print state root for evmlab tracing
+				if state != nil {
-			if s != nil {
+					root := state.IntermediateRoot(false)
-				root := s.IntermediateRoot(false)
+					result.Root = &root
-				result.Root = &root
+					if jsonOut {
-				if jsonOut {
+						fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root)
-					fmt.Fprintf(os.Stderr, "{\"stateRoot\": \"%#x\"}\n", root)
+					}
 				}
-			}
+				if err != nil {
-			if err != nil {
+					// Test failed, mark as so and dump any state to aid debugging
-				// Test failed, mark as so and dump any state to aid debugging
+					result.Pass, result.Error = false, err.Error()
-				result.Pass, result.Error = false, err.Error()
+					if dump {
-				if dump && s != nil {
+						dump := state.RawDump(nil)
-					s, _ = state.New(*result.Root, s.Database(), nil)
+						result.State = &dump
-					dump := s.RawDump(nil)
+					}
-					result.State = &dump
 				}
-			}
+			})
-
 			results = append(results, *result)
 		}
 	}

cmd/evm/t8n_test.go

@@ -267,6 +267,14 @@ func TestT8n(t *testing.T) {
 			output: t8nOutput{alloc: true, result: true},
 			expOut: "exp.json",
 		},
+		{ // More cancun tests
+			base: "./testdata/29",
+			input: t8nInput{
+				"alloc.json", "txs.json", "env.json", "Cancun", "",
+			},
+			output: t8nOutput{alloc: true, result: true},
+			expOut: "exp.json",
+		},
 	} {
 		args := []string{"t8n"}
 		args = append(args, tc.output.get()...)

cmd/evm/testdata/28/env.json

@@ -18,5 +18,6 @@
     "parentBlobGasUsed" : "0x00",
     "blockHashes" : {
         "0" : "0x3a9b485972e7353edd9152712492f0c58d89ef80623686b6bf947a4a6dce6cb6"
-    }
+    },
+    "parentBeaconBlockRoot": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00"
 }

cmd/evm/testdata/29/alloc.json

@@ -0,0 +1,16 @@
+{
+  "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" : {
+    "balance" : "0x016345785d8a0000",
+    "code" : "0x",
+    "nonce" : "0x00",
+    "storage" : {
+    }
+  },
+  "0xbEac00dDB15f3B6d645C48263dC93862413A222D" : {
+    "balance" : "0x1",
+    "code" : "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500",
+    "nonce" : "0x00",
+    "storage" : {
+    }
+  }
+}

cmd/evm/testdata/29/env.json

@@ -0,0 +1,20 @@
+{
+    "currentCoinbase" : "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba",
+    "currentNumber" : "0x01",
+    "currentTimestamp" : "0x079e",
+    "currentGasLimit" : "0x7fffffffffffffff",
+    "previousHash" : "0x3a9b485972e7353edd9152712492f0c58d89ef80623686b6bf947a4a6dce6cb6",
+    "currentBlobGasUsed" : "0x00",
+    "parentTimestamp" : "0x03b6",
+    "parentDifficulty" : "0x00",
+    "parentUncleHash" : "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "currentRandom" : "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+    "withdrawals" : [
+    ],
+    "parentBaseFee" : "0x0a",
+    "parentGasUsed" : "0x00",
+    "parentGasLimit" : "0x7fffffffffffffff",
+    "parentExcessBlobGas" : "0x00",
+    "parentBlobGasUsed" : "0x00",
+    "parentBeaconBlockRoot": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00"
+}

cmd/evm/testdata/29/exp.json

@@ -0,0 +1,45 @@
+{
+  "alloc": {
+    "0xbeac00ddb15f3b6d645c48263dc93862413a222d": {
+      "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500",
+      "storage": {
+        "0x000000000000000000000000000000000000000000000000000000000000079e": "0x000000000000000000000000000000000000000000000000000000000000079e",
+        "0x000000000000000000000000000000000000000000000000000000000001879e": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00"
+      },
+      "balance": "0x1"
+    },
+    "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+      "balance": "0x16345785d871db8",
+      "nonce": "0x1"
+    }
+  },
+  "result": {
+    "stateRoot": "0x2db9f6bc233e8fd0af2d8023404493a19b37d9d69ace71f4e73158851fced574",
+    "txRoot": "0x248074fabe112f7d93917f292b64932394f835bb98da91f21501574d58ec92ab",
+    "receiptsRoot": "0xf78dfb743fbd92ade140711c8bbc542b5e307f0ab7984eff35d751969fe57efa",
+    "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+    "receipts": [
+      {
+        "type": "0x2",
+        "root": "0x",
+        "status": "0x1",
+        "cumulativeGasUsed": "0x5208",
+        "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        "logs": null,
+        "transactionHash": "0x84f70aba406a55628a0620f26d260f90aeb6ccc55fed6ec2ac13dd4f727032ed",
+        "contractAddress": "0x0000000000000000000000000000000000000000",
+        "gasUsed": "0x5208",
+        "effectiveGasPrice": null,
+        "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+        "transactionIndex": "0x0"
+      }
+    ],
+    "currentDifficulty": null,
+    "gasUsed": "0x5208",
+    "currentBaseFee": "0x9",
+    "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+    "currentExcessBlobGas": "0x0",
+    "currentBlobGasUsed": "0x0"
+  }
+}

cmd/evm/testdata/29/readme.md

@@ -0,0 +1,29 @@
+## EIP 4788
+
+This test contains testcases for EIP-4788. The 4788-contract is 
+located at address `0xbeac00ddb15f3b6d645c48263dc93862413a222d`, and this test executes a simple transaction. It also
+implicitly invokes the system tx, which sets calls the contract and sets the 
+storage values
+```
+$ dir=./testdata/29/ && go run . t8n --state.fork=Cancun  --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json --output.alloc=stdout
+INFO [08-15|20:07:56.335] Trie dumping started                     root=ecde45..2af8a7
+INFO [08-15|20:07:56.335] Trie dumping complete                    accounts=2 elapsed="225.848µs"
+INFO [08-15|20:07:56.335] Wrote file                               file=result.json
+{
+  "alloc": {
+    "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+      "balance": "0x16345785d871db8",
+      "nonce": "0x1"
+    },
+    "0xbeac00541d49391ed88abf392bfc1f4dea8c4143": {
+      "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500",
+      "storage": {
+        "0x000000000000000000000000000000000000000000000000000000000000079e": "0x000000000000000000000000000000000000000000000000000000000000079e",
+        "0x000000000000000000000000000000000000000000000000000000000001879e": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00"
+      },
+      "balance": "0x
+    }
+  }
+}
+
+```

cmd/evm/testdata/29/txs.json

@@ -0,0 +1,19 @@
+[
+  {
+    "input" : "0x",
+    "gas" : "0x10000000",
+    "nonce" : "0x0",
+    "to" : "0x1111111111111111111111111111111111111111",
+    "value" : "0x0",
+    "secretKey" : "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8",
+    "chainId" : "0x1",
+    "type" : "0x2",
+    "v": "0x0",
+    "r": "0x0",
+    "s": "0x0",
+    "maxFeePerGas" : "0xfa0",
+    "maxPriorityFeePerGas" : "0x0",
+    "accessList" : [
+    ]
+  }
+]

cmd/geth/chaincmd.go

@@ -39,7 +39,6 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/node"
-	"github.com/ethereum/go-ethereum/trie"
 	"github.com/urfave/cli/v2"
 )
 
@@ -49,7 +48,9 @@ var (
 		Name:      "init",
 		Usage:     "Bootstrap and initialize a new genesis block",
 		ArgsUsage: "<genesisPath>",
-		Flags:     flags.Merge([]cli.Flag{utils.CachePreimagesFlag}, utils.DatabasePathFlags),
+		Flags: flags.Merge([]cli.Flag{
+			utils.CachePreimagesFlag,
+		}, utils.DatabaseFlags),
 		Description: `
 The init command initializes a new genesis block and definition for the network.
 This is a destructive action and changes the network in which you will be
@@ -94,7 +95,9 @@ if one is set.  Otherwise it prints the genesis from the datadir.`,
 			utils.MetricsInfluxDBBucketFlag,
 			utils.MetricsInfluxDBOrganizationFlag,
 			utils.TxLookupLimitFlag,
-		}, utils.DatabasePathFlags),
+			utils.TransactionHistoryFlag,
+			utils.StateHistoryFlag,
+		}, utils.DatabaseFlags),
 		Description: `
 The import command imports blocks from an RLP-encoded form. The form can be one file
 with several RLP-encoded blocks, or several files can be used.
@@ -110,7 +113,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`,
 		Flags: flags.Merge([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabaseFlags),
 		Description: `
 Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
@@ -126,7 +129,7 @@ be gzipped.`,
 		Flags: flags.Merge([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabaseFlags),
 		Description: `
 The import-preimages command imports hash preimages from an RLP encoded stream.
 It's deprecated, please use "geth db import" instead.
@@ -140,7 +143,7 @@ It's deprecated, please use "geth db import" instead.
 		Flags: flags.Merge([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabaseFlags),
 		Description: `
 The export-preimages command exports hash preimages to an RLP encoded stream.
 It's deprecated, please use "geth db export" instead.
@@ -159,7 +162,7 @@ It's deprecated, please use "geth db export" instead.
 			utils.IncludeIncompletesFlag,
 			utils.StartKeyFlag,
 			utils.DumpLimitFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabaseFlags),
 		Description: `
 This command dumps out the state for a given block (or latest, if none provided).
 `,
@@ -195,14 +198,15 @@ func initGenesis(ctx *cli.Context) error {
 		if err != nil {
 			utils.Fatalf("Failed to open database: %v", err)
 		}
-		triedb := trie.NewDatabaseWithConfig(chaindb, &trie.Config{
+		defer chaindb.Close()
-			Preimages: ctx.Bool(utils.CachePreimagesFlag.Name),
+
-		})
+		triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false)
+		defer triedb.Close()
+
 		_, hash, err := core.SetupGenesisBlock(chaindb, triedb, genesis)
 		if err != nil {
 			utils.Fatalf("Failed to write genesis block: %v", err)
 		}
-		chaindb.Close()
 		log.Info("Successfully wrote genesis state", "database", name, "hash", hash)
 	}
 	return nil
@@ -241,7 +245,7 @@ func dumpGenesis(ctx *cli.Context) error {
 	if ctx.IsSet(utils.DataDirFlag.Name) {
 		utils.Fatalf("no existing datadir at %s", stack.Config().DataDir)
 	}
-	utils.Fatalf("no network preset provided.  no exisiting genesis in the default datadir")
+	utils.Fatalf("no network preset provided, no existing genesis in the default datadir")
 	return nil
 }
 
@@ -332,7 +336,8 @@ func exportChain(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	chain, _ := utils.MakeChain(ctx, stack, true)
+	chain, db := utils.MakeChain(ctx, stack, true)
+	defer db.Close()
 	start := time.Now()
 
 	var err error
@@ -372,6 +377,7 @@ func importPreimages(ctx *cli.Context) error {
 	defer stack.Close()
 
 	db := utils.MakeChainDatabase(ctx, stack, false)
+	defer db.Close()
 	start := time.Now()
 
 	if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil {
@@ -390,6 +396,7 @@ func exportPreimages(ctx *cli.Context) error {
 	defer stack.Close()
 
 	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
 	start := time.Now()
 
 	if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil {
@@ -401,6 +408,8 @@ func exportPreimages(ctx *cli.Context) error {
 
 func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) {
 	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+
 	var header *types.Header
 	if ctx.NArg() > 1 {
 		return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg())
@@ -465,10 +474,10 @@ func dump(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	config := &trie.Config{
+	triedb := utils.MakeTrieDatabase(ctx, db, true, false) // always enable preimage lookup
-		Preimages: true, // always enable preimage lookup
+	defer triedb.Close()
-	}
+
-	state, err := state.New(root, state.NewDatabaseWithConfig(db, config), nil)
+	state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil)
 	if err != nil {
 		return err
 	}

cmd/geth/config.go

@@ -22,10 +22,10 @@ import (
 	"fmt"
 	"os"
 	"reflect"
+	"runtime"
+	"strings"
 	"unicode"
 
-	"github.com/urfave/cli/v2"
-
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/external"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
@@ -43,6 +43,7 @@ import (
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/naoina/toml"
+	"github.com/urfave/cli/v2"
 )
 
 var (
@@ -177,6 +178,20 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	}
 	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
 
+	// Create gauge with geth system and build information
+	if eth != nil { // The 'eth' backend may be nil in light mode
+		var protos []string
+		for _, p := range eth.Protocols() {
+			protos = append(protos, fmt.Sprintf("%v/%d", p.Name, p.Version))
+		}
+		metrics.NewRegisteredGaugeInfo("geth/info", nil).Update(metrics.GaugeInfoValue{
+			"arch":      runtime.GOARCH,
+			"os":        runtime.GOOS,
+			"version":   cfg.Node.Version,
+			"protocols": strings.Join(protos, ","),
+		})
+	}
+
 	// Configure log filter RPC API.
 	filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth)
 

cmd/geth/dbcmd.go

@@ -48,7 +48,7 @@ var (
 		Name:      "removedb",
 		Usage:     "Remove blockchain and state databases",
 		ArgsUsage: "",
-		Flags:     utils.DatabasePathFlags,
+		Flags:     utils.DatabaseFlags,
 		Description: `
 Remove blockchain and state databases`,
 	}
@@ -77,7 +77,7 @@ Remove blockchain and state databases`,
 		ArgsUsage: "<prefix> <start>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Usage:       "Inspect the storage size for each type of data in the database",
 		Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
 	}
@@ -85,7 +85,7 @@ Remove blockchain and state databases`,
 		Action:    checkStateContent,
 		Name:      "check-state-content",
 		ArgsUsage: "<start (optional)>",
-		Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+		Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 		Usage:     "Verify that state data is cryptographically correct",
 		Description: `This command iterates the entire database for 32-byte keys, looking for rlp-encoded trie nodes.
 For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates
@@ -97,7 +97,7 @@ a data corruption.`,
 		Usage:  "Print leveldb statistics",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 	}
 	dbCompactCmd = &cli.Command{
 		Action: dbCompact,
@@ -107,7 +107,7 @@ a data corruption.`,
 			utils.SyncModeFlag,
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: `This command performs a database compaction. 
 WARNING: This operation may take a very long time to finish, and may cause database
 corruption if it is aborted during execution'!`,
@@ -119,7 +119,7 @@ corruption if it is aborted during execution'!`,
 		ArgsUsage: "<hex-encoded key>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDeleteCmd = &cli.Command{
@@ -129,7 +129,7 @@ corruption if it is aborted during execution'!`,
 		ArgsUsage: "<hex-encoded key>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: `This command deletes the specified database key from the database. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -140,7 +140,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<hex-encoded key> <hex-encoded value>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: `This command sets a given database key to the given value. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -151,7 +151,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<hex-encoded state root> <hex-encoded account hash> <hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDumpFreezerIndex = &cli.Command{
@@ -161,7 +161,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<freezer-type> <table-type> <start (int)> <end (int)>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "This command displays information about the freezer index.",
 	}
 	dbImportCmd = &cli.Command{
@@ -171,7 +171,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<dumpfile> <start (optional)",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "The import command imports the specific chain data from an RLP encoded stream.",
 	}
 	dbExportCmd = &cli.Command{
@@ -181,7 +181,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<type> <dumpfile>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
 	}
 	dbMetadataCmd = &cli.Command{
@@ -190,7 +190,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Usage:  "Shows metadata about the chain status.",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Shows metadata about the chain status.",
 	}
 )
@@ -482,6 +482,9 @@ func dbDumpTrie(ctx *cli.Context) error {
 	db := utils.MakeChainDatabase(ctx, stack, true)
 	defer db.Close()
 
+	triedb := utils.MakeTrieDatabase(ctx, db, false, true)
+	defer triedb.Close()
+
 	var (
 		state   []byte
 		storage []byte
@@ -515,7 +518,7 @@ func dbDumpTrie(ctx *cli.Context) error {
 		}
 	}
 	id := trie.StorageTrieID(common.BytesToHash(state), common.BytesToHash(account), common.BytesToHash(storage))
-	theTrie, err := trie.New(id, trie.NewDatabase(db))
+	theTrie, err := trie.New(id, triedb)
 	if err != nil {
 		return err
 	}
@@ -591,6 +594,7 @@ func importLDBdata(ctx *cli.Context) error {
 		close(stop)
 	}()
 	db := utils.MakeChainDatabase(ctx, stack, false)
+	defer db.Close()
 	return utils.ImportLDBData(db, fName, int64(start), stop)
 }
 
@@ -687,6 +691,7 @@ func exportChaindata(ctx *cli.Context) error {
 		close(stop)
 	}()
 	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
 	return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop)
 }
 
@@ -694,6 +699,8 @@ func showMetaData(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+
 	ancients, err := db.Ancients()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error accessing ancients: %v", err)

cmd/geth/genesis_test.go

@@ -176,12 +176,12 @@ func TestCustomBackend(t *testing.T) {
 		{ // Can't start pebble on top of leveldb
 			initArgs:   []string{"--db.engine", "leveldb"},
 			execArgs:   []string{"--db.engine", "pebble"},
-			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
+			execExpect: `Fatal: Could not open database: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
 		},
 		{ // Can't start leveldb on top of pebble
 			initArgs:   []string{"--db.engine", "pebble"},
 			execArgs:   []string{"--db.engine", "leveldb"},
-			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
+			execExpect: `Fatal: Could not open database: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
 		},
 		{ // Reject invalid backend choice
 			initArgs:   []string{"--db.engine", "mssql"},

cmd/geth/main.go

@@ -88,6 +88,8 @@ var (
 		utils.GCModeFlag,
 		utils.SnapshotFlag,
 		utils.TxLookupLimitFlag,
+		utils.TransactionHistoryFlag,
+		utils.StateHistoryFlag,
 		utils.LightServeFlag,
 		utils.LightIngressFlag,
 		utils.LightEgressFlag,
@@ -142,7 +144,7 @@ var (
 		utils.GpoMaxGasPriceFlag,
 		utils.GpoIgnoreGasPriceFlag,
 		configFileFlag,
-	}, utils.NetworkFlags, utils.DatabasePathFlags)
+	}, utils.NetworkFlags, utils.DatabaseFlags)
 
 	rpcFlags = []cli.Flag{
 		utils.HTTPEnabledFlag,
@@ -241,11 +243,16 @@ func init() {
 		debug.Flags,
 		metricsFlags,
 	)
+	flags.AutoEnvVars(app.Flags, "GETH")
 
 	app.Before = func(ctx *cli.Context) error {
 		maxprocs.Set() // Automatically set GOMAXPROCS to match Linux container CPU quota.
 		flags.MigrateGlobalFlags(ctx)
-		return debug.Setup(ctx)
+		if err := debug.Setup(ctx); err != nil {
+			return err
+		}
+		flags.CheckEnvVars(ctx, app.Flags, "GETH")
+		return nil
 	}
 	app.After = func(ctx *cli.Context) error {
 		debug.Exit()
@@ -272,6 +279,9 @@ func prepare(ctx *cli.Context) {
 	case ctx.IsSet(utils.SepoliaFlag.Name):
 		log.Info("Starting Geth on Sepolia testnet...")
 
+	case ctx.IsSet(utils.HoleskyFlag.Name):
+		log.Info("Starting Geth on Holesky testnet...")
+
 	case ctx.IsSet(utils.DeveloperFlag.Name):
 		log.Info("Starting Geth in ephemeral dev mode...")
 		log.Warn(`You are running Geth in --dev mode. Please note the following:
@@ -296,7 +306,8 @@ func prepare(ctx *cli.Context) {
 	// If we're a full node on mainnet without --cache specified, bump default cache allowance
 	if ctx.String(utils.SyncModeFlag.Name) != "light" && !ctx.IsSet(utils.CacheFlag.Name) && !ctx.IsSet(utils.NetworkIdFlag.Name) {
 		// Make sure we're not on any supported preconfigured testnet either
-		if !ctx.IsSet(utils.SepoliaFlag.Name) &&
+		if !ctx.IsSet(utils.HoleskyFlag.Name) &&
+			!ctx.IsSet(utils.SepoliaFlag.Name) &&
 			!ctx.IsSet(utils.GoerliFlag.Name) &&
 			!ctx.IsSet(utils.DeveloperFlag.Name) {
 			// Nope, we're really on mainnet. Bump that cache up!

cmd/geth/snapshot.go

@@ -51,7 +51,7 @@ var (
 				Action:    pruneState,
 				Flags: flags.Merge([]cli.Flag{
 					utils.BloomFilterSizeFlag,
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
+				}, utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth snapshot prune-state <state-root>
 will prune historical state data with the help of the state snapshot.
@@ -61,10 +61,7 @@ two version states are available: genesis and the specific one.
 
 The default pruning target is the HEAD-127 state.
 
-WARNING: It's necessary to delete the trie clean cache after the pruning.
+WARNING: it's only supported in hash mode(--state.scheme=hash)".
-If you specify another directory for the trie clean cache via "--cache.trie.journal"
-during the use of Geth, please also specify it here for correct deletion. Otherwise
-the trie clean cache with default directory will be deleted.
 `,
 			},
 			{
@@ -72,7 +69,7 @@ the trie clean cache with default directory will be deleted.
 				Usage:     "Recalculate state hash based on the snapshot for verification",
 				ArgsUsage: "<root>",
 				Action:    verifyState,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth snapshot verify-state <state-root>
 will traverse the whole accounts and storages set based on the specified
@@ -85,7 +82,7 @@ In other words, this command does the snapshot to trie conversion.
 				Usage:     "Check that there is no 'dangling' snap storage",
 				ArgsUsage: "<root>",
 				Action:    checkDanglingStorage,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth snapshot check-dangling-storage <state-root> traverses the snap storage 
 data, and verifies that all snapshot storage data has a corresponding account. 
@@ -96,7 +93,7 @@ data, and verifies that all snapshot storage data has a corresponding account.
 				Usage:     "Check all snapshot layers for the a specific account",
 				ArgsUsage: "<address | hash>",
 				Action:    checkAccount,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth snapshot inspect-account <address | hash> checks all snapshot layers and prints out
 information about the specified address. 
@@ -107,7 +104,7 @@ information about the specified address.
 				Usage:     "Traverse the state with given root hash and perform quick verification",
 				ArgsUsage: "<root>",
 				Action:    traverseState,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth snapshot traverse-state <state-root>
 will traverse the whole state from the given state root and will abort if any
@@ -122,7 +119,7 @@ It's also usable without snapshot enabled.
 				Usage:     "Traverse the state with given root hash and perform detailed verification",
 				ArgsUsage: "<root>",
 				Action:    traverseRawState,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth snapshot traverse-rawstate <state-root>
 will traverse the whole state from the given root and will abort if any referenced
@@ -143,7 +140,7 @@ It's also usable without snapshot enabled.
 					utils.ExcludeStorageFlag,
 					utils.StartKeyFlag,
 					utils.DumpLimitFlag,
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
+				}, utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 This command is semantically equivalent to 'geth dump', but uses the snapshots
 as the backend data source, making this command a lot faster. 
@@ -165,6 +162,9 @@ func pruneState(ctx *cli.Context) error {
 	chaindb := utils.MakeChainDatabase(ctx, stack, false)
 	defer chaindb.Close()
 
+	if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme {
+		log.Crit("Offline pruning is not required for path scheme")
+	}
 	prunerconfig := pruner.Config{
 		Datadir:   stack.ResolvePath(""),
 		BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name),
@@ -205,13 +205,16 @@ func verifyState(ctx *cli.Context) error {
 		log.Error("Failed to load head block")
 		return errors.New("no head block")
 	}
-	snapconfig := snapshot.Config{
+	triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
+	defer triedb.Close()
+
+	snapConfig := snapshot.Config{
 		CacheSize:  256,
 		Recovery:   false,
 		NoBuild:    true,
 		AsyncBuild: false,
 	}
-	snaptree, err := snapshot.New(snapconfig, chaindb, trie.NewDatabase(chaindb), headBlock.Root())
+	snaptree, err := snapshot.New(snapConfig, chaindb, triedb, headBlock.Root())
 	if err != nil {
 		log.Error("Failed to open snapshot tree", "err", err)
 		return err
@@ -242,7 +245,9 @@ func checkDanglingStorage(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	return snapshot.CheckDanglingStorage(utils.MakeChainDatabase(ctx, stack, true))
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+	return snapshot.CheckDanglingStorage(db)
 }
 
 // traverseState is a helper function used for pruning verification.
@@ -253,6 +258,11 @@ func traverseState(ctx *cli.Context) error {
 	defer stack.Close()
 
 	chaindb := utils.MakeChainDatabase(ctx, stack, true)
+	defer chaindb.Close()
+
+	triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
+	defer triedb.Close()
+
 	headBlock := rawdb.ReadHeadBlock(chaindb)
 	if headBlock == nil {
 		log.Error("Failed to load head block")
@@ -277,7 +287,6 @@ func traverseState(ctx *cli.Context) error {
 		root = headBlock.Root()
 		log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
 	}
-	triedb := trie.NewDatabase(chaindb)
 	t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb)
 	if err != nil {
 		log.Error("Failed to open trie", "root", root, "err", err)
@@ -318,6 +327,11 @@ func traverseState(ctx *cli.Context) error {
 			storageIter := trie.NewIterator(storageIt)
 			for storageIter.Next() {
 				slots += 1
+
+				if time.Since(lastReport) > time.Second*8 {
+					log.Info("Traversing state", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))
+					lastReport = time.Now()
+				}
 			}
 			if storageIter.Err != nil {
 				log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Err)
@@ -353,6 +367,11 @@ func traverseRawState(ctx *cli.Context) error {
 	defer stack.Close()
 
 	chaindb := utils.MakeChainDatabase(ctx, stack, true)
+	defer chaindb.Close()
+
+	triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true)
+	defer triedb.Close()
+
 	headBlock := rawdb.ReadHeadBlock(chaindb)
 	if headBlock == nil {
 		log.Error("Failed to load head block")
@@ -377,7 +396,6 @@ func traverseRawState(ctx *cli.Context) error {
 		root = headBlock.Root()
 		log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64())
 	}
-	triedb := trie.NewDatabase(chaindb)
 	t, err := trie.NewStateTrie(trie.StateTrieID(root), triedb)
 	if err != nil {
 		log.Error("Failed to open trie", "root", root, "err", err)
@@ -398,6 +416,11 @@ func traverseRawState(ctx *cli.Context) error {
 		log.Error("Failed to open iterator", "root", root, "err", err)
 		return err
 	}
+	reader, err := triedb.Reader(root)
+	if err != nil {
+		log.Error("State is non-existent", "root", root)
+		return nil
+	}
 	for accIter.Next(true) {
 		nodes += 1
 		node := accIter.Hash()
@@ -405,7 +428,7 @@ func traverseRawState(ctx *cli.Context) error {
 		// Check the present for non-empty hash node(embedded node doesn't
 		// have their own hash).
 		if node != (common.Hash{}) {
-			blob := rawdb.ReadLegacyTrieNode(chaindb, node)
+			blob, _ := reader.Node(common.Hash{}, accIter.Path(), node)
 			if len(blob) == 0 {
 				log.Error("Missing trie node(account)", "hash", node)
 				return errors.New("missing account")
@@ -446,7 +469,7 @@ func traverseRawState(ctx *cli.Context) error {
 					// Check the presence for non-empty hash node(embedded node doesn't
 					// have their own hash).
 					if node != (common.Hash{}) {
-						blob := rawdb.ReadLegacyTrieNode(chaindb, node)
+						blob, _ := reader.Node(common.BytesToHash(accIter.LeafKey()), storageIter.Path(), node)
 						if len(blob) == 0 {
 							log.Error("Missing trie node(storage)", "hash", node)
 							return errors.New("missing storage")
@@ -463,6 +486,10 @@ func traverseRawState(ctx *cli.Context) error {
 					if storageIter.Leaf() {
 						slots += 1
 					}
+					if time.Since(lastReport) > time.Second*8 {
+						log.Info("Traversing state", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))
+						lastReport = time.Now()
+					}
 				}
 				if storageIter.Error() != nil {
 					log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Error())
@@ -506,13 +533,16 @@ func dumpState(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	triedb := utils.MakeTrieDatabase(ctx, db, false, true)
+	defer triedb.Close()
+
 	snapConfig := snapshot.Config{
 		CacheSize:  256,
 		Recovery:   false,
 		NoBuild:    true,
 		AsyncBuild: false,
 	}
-	snaptree, err := snapshot.New(snapConfig, db, trie.NewDatabase(db), root)
+	snaptree, err := snapshot.New(snapConfig, db, triedb, root)
 	if err != nil {
 		return err
 	}

cmd/geth/testdata/vcheck/minisig-sigs-new/data.json.minisig

@@ -0,0 +1,4 @@
+untrusted comment: signature from minisign secret key
+RUQkliYstQBOKLK05Sy5f3bVRMBqJT26ABo6Vbp3BNJAVjejoqYCu4GWE/+7qcDfHBqYIniDCbFIUvYEnOHxV6vZ93wO1xJWDQw=
+trusted comment: timestamp:1693986492	file:data.json	hashed
+6Fdw2H+W1ZXK7QXSF77Z5AWC7+AEFAfDmTSxNGylU5HLT1AuSJQmxslj+VjtUBamYCvOuET7plbXza942AlWDw==

cmd/geth/verkle.go

@@ -45,7 +45,7 @@ var (
 				Usage:     "verify the conversion of a MPT into a verkle tree",
 				ArgsUsage: "<root>",
 				Action:    verifyVerkle,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth verkle verify <state-root>
 This command takes a root commitment and attempts to rebuild the tree.
@@ -56,7 +56,7 @@ This command takes a root commitment and attempts to rebuild the tree.
 				Usage:     "Dump a verkle tree to a DOT file",
 				ArgsUsage: "<root> <key1> [<key 2> ...]",
 				Action:    expandVerkle,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth verkle dump <state-root> <key 1> [<key 2> ...]
 This command will produce a dot file representing the tree, rooted at <root>.
@@ -115,6 +115,7 @@ func verifyVerkle(ctx *cli.Context) error {
 	defer stack.Close()
 
 	chaindb := utils.MakeChainDatabase(ctx, stack, true)
+	defer chaindb.Close()
 	headBlock := rawdb.ReadHeadBlock(chaindb)
 	if headBlock == nil {
 		log.Error("Failed to load head block")
@@ -163,6 +164,7 @@ func expandVerkle(ctx *cli.Context) error {
 	defer stack.Close()
 
 	chaindb := utils.MakeChainDatabase(ctx, stack, true)
+	defer chaindb.Close()
 	var (
 		rootC   common.Hash
 		keylist [][]byte

cmd/geth/version_check_test.go

@@ -30,17 +30,24 @@ import (
 )
 
 func TestVerification(t *testing.T) {
-	// Signatures generated with `minisign`
+	// Signatures generated with `minisign`. Legacy format, not pre-hashed file.
-	t.Run("minisig", func(t *testing.T) {
+	t.Run("minisig-legacy", func(t *testing.T) {
-		// For this test, the pubkey is in testdata/minisign.pub
+		// For this test, the pubkey is in testdata/vcheck/minisign.pub
 		// (the privkey is `minisign.sec`, if we want to expand this test. Password 'test' )
 		pub := "RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp"
 		testVerification(t, pub, "./testdata/vcheck/minisig-sigs/")
 	})
+	t.Run("minisig-new", func(t *testing.T) {
+		// For this test, the pubkey is in testdata/vcheck/minisign.pub
+		// (the privkey is `minisign.sec`, if we want to expand this test. Password 'test' )
+		// `minisign -S -s ./minisign.sec  -m data.json  -x ./minisig-sigs-new/data.json.minisig`
+		pub := "RWQkliYstQBOKOdtClfgC3IypIPX6TAmoEi7beZ4gyR3wsaezvqOMWsp"
+		testVerification(t, pub, "./testdata/vcheck/minisig-sigs-new/")
+	})
 	// Signatures generated with `signify-openbsd`
 	t.Run("signify-openbsd", func(t *testing.T) {
 		t.Skip("This currently fails, minisign expects 4 lines of data, signify provides only 2")
-		// For this test, the pubkey is in testdata/signifykey.pub
+		// For this test, the pubkey is in testdata/vcheck/signifykey.pub
 		// (the privkey is `signifykey.sec`, if we want to expand this test. Password 'test' )
 		pub := "RWSKLNhZb0KdATtRT7mZC/bybI3t3+Hv/O2i3ye04Dq9fnT9slpZ1a2/"
 		testVerification(t, pub, "./testdata/vcheck/signify-sigs/")
@@ -58,6 +65,9 @@ func testVerification(t *testing.T, pubkey, sigdir string) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	if len(files) == 0 {
+		t.Fatal("Missing tests")
+	}
 	for _, f := range files {
 		sig, err := os.ReadFile(filepath.Join(sigdir, f.Name()))
 		if err != nil {

cmd/utils/flags.go

@@ -48,7 +48,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/eth"
-	ethcatalyst "github.com/ethereum/go-ethereum/eth/catalyst"
+	"github.com/ethereum/go-ethereum/eth/catalyst"
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/eth/filters"
@@ -74,6 +74,9 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
+	"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
 	pcsclite "github.com/gballet/go-libpcsclite"
 	gopsutil "github.com/shirou/gopsutil/mem"
 	"github.com/urfave/cli/v2"
@@ -133,7 +136,7 @@ var (
 	}
 	NetworkIdFlag = &cli.Uint64Flag{
 		Name:     "networkid",
-		Usage:    "Explicitly set network id (integer)(For testnets: use --goerli, --sepolia instead)",
+		Usage:    "Explicitly set network id (integer)(For testnets: use --goerli, --sepolia, --holesky instead)",
 		Value:    ethconfig.Defaults.NetworkId,
 		Category: flags.EthCategory,
 	}
@@ -152,7 +155,11 @@ var (
 		Usage:    "Sepolia network: pre-configured proof-of-work test network",
 		Category: flags.EthCategory,
 	}
-
+	HoleskyFlag = &cli.BoolFlag{
+		Name:     "holesky",
+		Usage:    "Holesky network: pre-configured proof-of-stake test network",
+		Category: flags.EthCategory,
+	}
 	// Dev mode
 	DeveloperFlag = &cli.BoolFlag{
 		Name:     "dev",
@@ -218,30 +225,12 @@ var (
 	}
 
 	defaultSyncMode = ethconfig.Defaults.SyncMode
-	SyncModeFlag    = &flags.TextMarshalerFlag{
+	SnapshotFlag    = &cli.BoolFlag{
-		Name:     "syncmode",
-		Usage:    `Blockchain sync mode ("snap", "full" or "light")`,
-		Value:    &defaultSyncMode,
-		Category: flags.EthCategory,
-	}
-	GCModeFlag = &cli.StringFlag{
-		Name:     "gcmode",
-		Usage:    `Blockchain garbage collection mode ("full", "archive")`,
-		Value:    "full",
-		Category: flags.EthCategory,
-	}
-	SnapshotFlag = &cli.BoolFlag{
 		Name:     "snapshot",
 		Usage:    `Enables snapshot-database mode (default = enable)`,
 		Value:    true,
 		Category: flags.EthCategory,
 	}
-	TxLookupLimitFlag = &cli.Uint64Flag{
-		Name:     "txlookuplimit",
-		Usage:    "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)",
-		Value:    ethconfig.Defaults.TxLookupLimit,
-		Category: flags.EthCategory,
-	}
 	LightKDFFlag = &cli.BoolFlag{
 		Name:     "lightkdf",
 		Usage:    "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
@@ -268,6 +257,36 @@ var (
 		Usage:    "Manually specify the Verkle fork timestamp, overriding the bundled setting",
 		Category: flags.EthCategory,
 	}
+	SyncModeFlag = &flags.TextMarshalerFlag{
+		Name:     "syncmode",
+		Usage:    `Blockchain sync mode ("snap", "full" or "light")`,
+		Value:    &defaultSyncMode,
+		Category: flags.StateCategory,
+	}
+	GCModeFlag = &cli.StringFlag{
+		Name:     "gcmode",
+		Usage:    `Blockchain garbage collection mode, only relevant in state.scheme=hash ("full", "archive")`,
+		Value:    "full",
+		Category: flags.StateCategory,
+	}
+	StateSchemeFlag = &cli.StringFlag{
+		Name:     "state.scheme",
+		Usage:    "Scheme to use for storing ethereum state ('hash' or 'path')",
+		Value:    rawdb.HashScheme,
+		Category: flags.StateCategory,
+	}
+	StateHistoryFlag = &cli.Uint64Flag{
+		Name:     "history.state",
+		Usage:    "Number of recent blocks to retain state history for (default = 90,000 blocks, 0 = entire chain)",
+		Value:    ethconfig.Defaults.StateHistory,
+		Category: flags.StateCategory,
+	}
+	TransactionHistoryFlag = &cli.Uint64Flag{
+		Name:     "history.transactions",
+		Usage:    "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain)",
+		Value:    ethconfig.Defaults.TransactionHistory,
+		Category: flags.StateCategory,
+	}
 	// Light server and client settings
 	LightServeFlag = &cli.IntFlag{
 		Name:     "light.serve",
@@ -937,22 +956,24 @@ var (
 	TestnetFlags = []cli.Flag{
 		GoerliFlag,
 		SepoliaFlag,
+		HoleskyFlag,
 	}
 	// NetworkFlags is the flag group of all built-in supported networks.
 	NetworkFlags = append([]cli.Flag{MainnetFlag}, TestnetFlags...)
 
-	// DatabasePathFlags is the flag group of all database path flags.
+	// DatabaseFlags is the flag group of all database flags.
-	DatabasePathFlags = []cli.Flag{
+	DatabaseFlags = []cli.Flag{
 		DataDirFlag,
 		AncientFlag,
 		RemoteDBFlag,
+		StateSchemeFlag,
 		HttpHeaderFlag,
 	}
 )
 
 func init() {
 	if rawdb.PebbleEnabled {
-		DatabasePathFlags = append(DatabasePathFlags, DBEngineFlag)
+		DatabaseFlags = append(DatabaseFlags, DBEngineFlag)
 	}
 }
 
@@ -967,6 +988,9 @@ func MakeDataDir(ctx *cli.Context) string {
 		if ctx.Bool(SepoliaFlag.Name) {
 			return filepath.Join(path, "sepolia")
 		}
+		if ctx.Bool(HoleskyFlag.Name) {
+			return filepath.Join(path, "holesky")
+		}
 		return path
 	}
 	Fatalf("Cannot determine default data directory, please set manually (--datadir)")
@@ -975,7 +999,7 @@ func MakeDataDir(ctx *cli.Context) string {
 
 // setNodeKey creates a node key from set command line flags, either loading it
 // from a file or as a specified hex value. If neither flags were provided, this
-// method returns nil and an emphemeral key is to be generated.
+// method returns nil and an ephemeral key is to be generated.
 func setNodeKey(ctx *cli.Context, cfg *p2p.Config) {
 	var (
 		hex  = ctx.String(NodeKeyHexFlag.Name)
@@ -1008,33 +1032,45 @@ func setNodeUserIdent(ctx *cli.Context, cfg *node.Config) {
 
 // setBootstrapNodes creates a list of bootstrap nodes from the command line
 // flags, reverting to pre-configured ones if none have been specified.
+// Priority order for bootnodes configuration:
+//
+// 1. --bootnodes flag
+// 2. Config file
+// 3. Network preset flags (e.g. --goerli)
+// 4. default to mainnet nodes
 func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
 	urls := params.MainnetBootnodes
-	switch {
+	if ctx.IsSet(BootnodesFlag.Name) {
-	case ctx.IsSet(BootnodesFlag.Name):
 		urls = SplitAndTrim(ctx.String(BootnodesFlag.Name))
-	case ctx.Bool(SepoliaFlag.Name):
+	} else {
-		urls = params.SepoliaBootnodes
+		if cfg.BootstrapNodes != nil {
-	case ctx.Bool(GoerliFlag.Name):
+			return // Already set by config file, don't apply defaults.
-		urls = params.GoerliBootnodes
+		}
+		switch {
+		case ctx.Bool(HoleskyFlag.Name):
+			urls = params.HoleskyBootnodes
+		case ctx.Bool(SepoliaFlag.Name):
+			urls = params.SepoliaBootnodes
+		case ctx.Bool(GoerliFlag.Name):
+			urls = params.GoerliBootnodes
+		}
 	}
+	cfg.BootstrapNodes = mustParseBootnodes(urls)
+}
 
-	// don't apply defaults if BootstrapNodes is already set
+func mustParseBootnodes(urls []string) []*enode.Node {
-	if cfg.BootstrapNodes != nil {
+	nodes := make([]*enode.Node, 0, len(urls))
-		return
-	}
-
-	cfg.BootstrapNodes = make([]*enode.Node, 0, len(urls))
 	for _, url := range urls {
 		if url != "" {
 			node, err := enode.Parse(enode.ValidSchemes, url)
 			if err != nil {
 				log.Crit("Bootstrap URL invalid", "enode", url, "err", err)
-				continue
+				return nil
 			}
-			cfg.BootstrapNodes = append(cfg.BootstrapNodes, node)
+			nodes = append(nodes, node)
 		}
 	}
+	return nodes
 }
 
 // setBootstrapNodesV5 creates a list of bootstrap nodes from the command line
@@ -1465,6 +1501,8 @@ func SetDataDir(ctx *cli.Context, cfg *node.Config) {
 		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli")
 	case ctx.Bool(SepoliaFlag.Name) && cfg.DataDir == node.DefaultDataDir():
 		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "sepolia")
+	case ctx.Bool(HoleskyFlag.Name) && cfg.DataDir == node.DefaultDataDir():
+		cfg.DataDir = filepath.Join(node.DefaultDataDir(), "holesky")
 	}
 }
 
@@ -1621,16 +1659,11 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) {
 // SetEthConfig applies eth-related command line flags to the config.
 func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	// Avoid conflicting network flags
-	CheckExclusive(ctx, MainnetFlag, DeveloperFlag, GoerliFlag, SepoliaFlag)
+	CheckExclusive(ctx, MainnetFlag, DeveloperFlag, GoerliFlag, SepoliaFlag, HoleskyFlag)
 	CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light")
 	CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer
-	if ctx.String(GCModeFlag.Name) == "archive" && ctx.Uint64(TxLookupLimitFlag.Name) != 0 {
+
-		ctx.Set(TxLookupLimitFlag.Name, "0")
+	// Set configurations from CLI flags
-		log.Warn("Disable transaction unindexing for archive node")
-	}
-	if ctx.IsSet(LightServeFlag.Name) && ctx.Uint64(TxLookupLimitFlag.Name) != 0 {
-		log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited")
-	}
 	setEtherbase(ctx, cfg)
 	setGPO(ctx, &cfg.GPO, ctx.String(SyncModeFlag.Name) == "light")
 	setTxPool(ctx, &cfg.TxPool)
@@ -1687,8 +1720,36 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		cfg.Preimages = true
 		log.Info("Enabling recording of key preimages since archive mode is used")
 	}
-	if ctx.IsSet(TxLookupLimitFlag.Name) {
+	if ctx.IsSet(StateHistoryFlag.Name) {
-		cfg.TxLookupLimit = ctx.Uint64(TxLookupLimitFlag.Name)
+		cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name)
+	}
+	// Parse state scheme, abort the process if it's not compatible.
+	chaindb := tryMakeReadOnlyDatabase(ctx, stack)
+	scheme, err := ParseStateScheme(ctx, chaindb)
+	chaindb.Close()
+	if err != nil {
+		Fatalf("%v", err)
+	}
+	cfg.StateScheme = scheme
+
+	// Parse transaction history flag, if user is still using legacy config
+	// file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'.
+	if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit {
+		log.Warn("The config option 'TxLookupLimit' is deprecated and will be removed, please use 'TransactionHistory'")
+		cfg.TransactionHistory = cfg.TxLookupLimit
+	}
+	if ctx.IsSet(TransactionHistoryFlag.Name) {
+		cfg.TransactionHistory = ctx.Uint64(TransactionHistoryFlag.Name)
+	} else if ctx.IsSet(TxLookupLimitFlag.Name) {
+		log.Warn("The flag --txlookuplimit is deprecated and will be removed, please use --history.transactions")
+		cfg.TransactionHistory = ctx.Uint64(TxLookupLimitFlag.Name)
+	}
+	if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 {
+		cfg.TransactionHistory = 0
+		log.Warn("Disabled transaction unindexing for archive node")
+	}
+	if ctx.IsSet(LightServeFlag.Name) && cfg.TransactionHistory != 0 {
+		log.Warn("LES server cannot serve old transaction status and cannot connect below les/4 protocol version if transaction lookup index is limited")
 	}
 	if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) {
 		cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100
@@ -1751,6 +1812,12 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		}
 		cfg.Genesis = core.DefaultGenesisBlock()
 		SetDNSDiscoveryDefaults(cfg, params.MainnetGenesisHash)
+	case ctx.Bool(HoleskyFlag.Name):
+		if !ctx.IsSet(NetworkIdFlag.Name) {
+			cfg.NetworkId = 17000
+		}
+		cfg.Genesis = core.DefaultHoleskyGenesisBlock()
+		SetDNSDiscoveryDefaults(cfg, params.HoleskyGenesisHash)
 	case ctx.Bool(SepoliaFlag.Name):
 		if !ctx.IsSet(NetworkIdFlag.Name) {
 			cfg.NetworkId = 11155111
@@ -1814,15 +1881,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		// Create a new developer genesis block or reuse existing one
 		cfg.Genesis = core.DeveloperGenesisBlock(ctx.Uint64(DeveloperGasLimitFlag.Name), developer.Address)
 		if ctx.IsSet(DataDirFlag.Name) {
-			// If datadir doesn't exist we need to open db in write-mode
+			chaindb := tryMakeReadOnlyDatabase(ctx, stack)
-			// so leveldb can create files.
-			readonly := true
-			if !common.FileExist(stack.ResolvePath("chaindata")) {
-				readonly = false
-			}
-			// Check if we have an already initialized chain and fall back to
-			// that if so. Otherwise we need to generate a new genesis spec.
-			chaindb := MakeChainDatabase(ctx, stack, readonly)
 			if rawdb.ReadCanonicalHash(chaindb, 0) != (common.Hash{}) {
 				cfg.Genesis = nil // fallback to db content
 			}
@@ -1930,7 +1989,7 @@ func RegisterFullSyncTester(stack *node.Node, eth *eth.Ethereum, path string) {
 	if err := rlp.DecodeBytes(rlpBlob, &block); err != nil {
 		Fatalf("Failed to decode block: %v", err)
 	}
-	ethcatalyst.RegisterFullSyncTester(stack, eth, &block)
+	catalyst.RegisterFullSyncTester(stack, eth, &block)
 	log.Info("Registered full-sync tester", "number", block.NumberU64(), "hash", block.Hash())
 }
 
@@ -2040,6 +2099,18 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
 	return chainDb
 }
 
+// tryMakeReadOnlyDatabase try to open the chain database in read-only mode,
+// or fallback to write mode if the database is not initialized.
+func tryMakeReadOnlyDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {
+	// If the database doesn't exist we need to open it in write-mode to allow
+	// the engine to create files.
+	readonly := true
+	if rawdb.PreexistingDatabase(stack.ResolvePath("chaindata")) == "" {
+		readonly = false
+	}
+	return MakeChainDatabase(ctx, stack, readonly)
+}
+
 func IsNetworkPreset(ctx *cli.Context) bool {
 	for _, flag := range NetworkFlags {
 		bFlag, _ := flag.(*cli.BoolFlag)
@@ -2061,7 +2132,7 @@ func DialRPCWithHeaders(endpoint string, headers []string) (*rpc.Client, error)
 	}
 	var opts []rpc.ClientOption
 	if len(headers) > 0 {
-		var customHeaders = make(http.Header)
+		customHeaders := make(http.Header)
 		for _, h := range headers {
 			kv := strings.Split(h, ":")
 			if len(kv) != 2 {
@@ -2079,6 +2150,8 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis {
 	switch {
 	case ctx.Bool(MainnetFlag.Name):
 		genesis = core.DefaultGenesisBlock()
+	case ctx.Bool(HoleskyFlag.Name):
+		genesis = core.DefaultHoleskyGenesisBlock()
 	case ctx.Bool(SepoliaFlag.Name):
 		genesis = core.DefaultSepoliaGenesisBlock()
 	case ctx.Bool(GoerliFlag.Name):
@@ -2106,6 +2179,10 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
 	if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
 		Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
 	}
+	scheme, err := ParseStateScheme(ctx, chainDb)
+	if err != nil {
+		Fatalf("%v", err)
+	}
 	cache := &core.CacheConfig{
 		TrieCleanLimit:      ethconfig.Defaults.TrieCleanCache,
 		TrieCleanNoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name),
@@ -2114,6 +2191,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
 		TrieTimeLimit:       ethconfig.Defaults.TrieTimeout,
 		SnapshotLimit:       ethconfig.Defaults.SnapshotCache,
 		Preimages:           ctx.Bool(CachePreimagesFlag.Name),
+		StateScheme:         scheme,
+		StateHistory:        ctx.Uint64(StateHistoryFlag.Name),
 	}
 	if cache.TrieDirtyDisabled && !cache.Preimages {
 		cache.Preimages = true
@@ -2158,3 +2237,62 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
 	}
 	return preloads
 }
+
+// ParseStateScheme resolves scheme identifier from CLI flag. If the provided
+// state scheme is not compatible with the one of persistent scheme, an error
+// will be returned.
+//
+//   - none: use the scheme consistent with persistent state, or fallback
+//     to hash-based scheme if state is empty.
+//   - hash: use hash-based scheme or error out if not compatible with
+//     persistent state scheme.
+//   - path: use path-based scheme or error out if not compatible with
+//     persistent state scheme.
+func ParseStateScheme(ctx *cli.Context, disk ethdb.Database) (string, error) {
+	// If state scheme is not specified, use the scheme consistent
+	// with persistent state, or fallback to hash mode if database
+	// is empty.
+	stored := rawdb.ReadStateScheme(disk)
+	if !ctx.IsSet(StateSchemeFlag.Name) {
+		if stored == "" {
+			// use default scheme for empty database, flip it when
+			// path mode is chosen as default
+			log.Info("State schema set to default", "scheme", "hash")
+			return rawdb.HashScheme, nil
+		}
+		log.Info("State scheme set to already existing", "scheme", stored)
+		return stored, nil // reuse scheme of persistent scheme
+	}
+	// If state scheme is specified, ensure it's compatible with
+	// persistent state.
+	scheme := ctx.String(StateSchemeFlag.Name)
+	if stored == "" || scheme == stored {
+		log.Info("State scheme set by user", "scheme", scheme)
+		return scheme, nil
+	}
+	return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, scheme)
+}
+
+// MakeTrieDatabase constructs a trie database based on the configured scheme.
+func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) *trie.Database {
+	config := &trie.Config{
+		Preimages: preimage,
+	}
+	scheme, err := ParseStateScheme(ctx, disk)
+	if err != nil {
+		Fatalf("%v", err)
+	}
+	if scheme == rawdb.HashScheme {
+		// Read-only mode is not implemented in hash mode,
+		// ignore the parameter silently. TODO(rjl493456442)
+		// please config it if read mode is implemented.
+		config.HashDB = hashdb.Defaults
+		return trie.NewDatabase(disk, config)
+	}
+	if readOnly {
+		config.PathDB = pathdb.ReadOnly
+	} else {
+		config.PathDB = pathdb.Defaults
+	}
+	return trie.NewDatabase(disk, config)
+}

cmd/utils/flags_legacy.go

@@ -19,6 +19,7 @@ package utils
 import (
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/urfave/cli/v2"
 )
@@ -37,6 +38,7 @@ var DeprecatedFlags = []cli.Flag{
 	CacheTrieJournalFlag,
 	CacheTrieRejournalFlag,
 	LegacyDiscoveryV5Flag,
+	TxLookupLimitFlag,
 }
 
 var (
@@ -68,6 +70,13 @@ var (
 		Usage:    "Enables the experimental RLPx V5 (Topic Discovery) mechanism (deprecated, use --discv5 instead)",
 		Category: flags.DeprecatedCategory,
 	}
+	// Deprecated August 2023
+	TxLookupLimitFlag = &cli.Uint64Flag{
+		Name:     "txlookuplimit",
+		Usage:    "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain) (deprecated, use history.transactions instead)",
+		Value:    ethconfig.Defaults.TransactionHistory,
+		Category: flags.DeprecatedCategory,
+	}
 )
 
 // showDeprecated displays deprecated flags that will be soon removed from the codebase.

common/types_test.go

@@ -25,6 +25,7 @@ import (
 	"reflect"
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestBytesConversion(t *testing.T) {
@@ -583,3 +584,14 @@ func TestAddressEIP55(t *testing.T) {
 		t.Fatal("Unexpected address after unmarshal")
 	}
 }
+
+func BenchmarkPrettyDuration(b *testing.B) {
+	var x = PrettyDuration(time.Duration(int64(1203123912312)))
+	b.Logf("Pre %s", time.Duration(x).String())
+	var a string
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		a = x.String()
+	}
+	b.Logf("Post %s", a)
+}

consensus/beacon/consensus.go

@@ -269,15 +269,21 @@ func (beacon *Beacon) verifyHeader(chain consensus.ChainHeaderReader, header, pa
 	if !shanghai && header.WithdrawalsHash != nil {
 		return fmt.Errorf("invalid withdrawalsHash: have %x, expected nil", header.WithdrawalsHash)
 	}
-	// Verify the existence / non-existence of excessBlobGas
+	// Verify the existence / non-existence of cancun-specific header fields
 	cancun := chain.Config().IsCancun(header.Number, header.Time)
-	if !cancun && header.ExcessBlobGas != nil {
+	if !cancun {
-		return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas)
+		switch {
-	}
+		case header.ExcessBlobGas != nil:
-	if !cancun && header.BlobGasUsed != nil {
+			return fmt.Errorf("invalid excessBlobGas: have %d, expected nil", header.ExcessBlobGas)
-		return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed)
+		case header.BlobGasUsed != nil:
-	}
+			return fmt.Errorf("invalid blobGasUsed: have %d, expected nil", header.BlobGasUsed)
-	if cancun {
+		case header.ParentBeaconRoot != nil:
+			return fmt.Errorf("invalid parentBeaconRoot, have %#x, expected nil", header.ParentBeaconRoot)
+		}
+	} else {
+		if header.ParentBeaconRoot == nil {
+			return errors.New("header is missing beaconRoot")
+		}
 		if err := eip4844.VerifyEIP4844Header(parent, header); err != nil {
 			return err
 		}

consensus/misc/eip4844/eip4844.go

@@ -42,8 +42,8 @@ func VerifyEIP4844Header(parent, header *types.Header) error {
 		return errors.New("header is missing blobGasUsed")
 	}
 	// Verify that the blob gas used remains within reasonable limits.
-	if *header.BlobGasUsed > params.BlobTxMaxBlobGasPerBlock {
+	if *header.BlobGasUsed > params.MaxBlobGasPerBlock {
-		return fmt.Errorf("blob gas used %d exceeds maximum allowance %d", *header.BlobGasUsed, params.BlobTxMaxBlobGasPerBlock)
+		return fmt.Errorf("blob gas used %d exceeds maximum allowance %d", *header.BlobGasUsed, params.MaxBlobGasPerBlock)
 	}
 	if *header.BlobGasUsed%params.BlobTxBlobGasPerBlob != 0 {
 		return fmt.Errorf("blob gas used %d not a multiple of blob gas per blob %d", header.BlobGasUsed, params.BlobTxBlobGasPerBlob)

consensus/misc/eip4844/eip4844_test.go

@@ -45,14 +45,14 @@ func TestCalcExcessBlobGas(t *testing.T) {
 		// The excess blob gas should decrease by however much the target was
 		// under-shot, capped at zero.
 		{params.BlobTxTargetBlobGasPerBlock, params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob, params.BlobTxTargetBlobGasPerBlock},
-		{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, params.BlobTxBlobGasPerBlob},
+		{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, params.BlobTxTargetBlobGasPerBlock - params.BlobTxBlobGasPerBlob},
-		{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 2, 0},
+		{params.BlobTxTargetBlobGasPerBlock, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 2, params.BlobTxTargetBlobGasPerBlock - (2 * params.BlobTxBlobGasPerBlob)},
 		{params.BlobTxBlobGasPerBlob - 1, (params.BlobTxTargetBlobGasPerBlock / params.BlobTxBlobGasPerBlob) - 1, 0},
 	}
-	for _, tt := range tests {
+	for i, tt := range tests {
 		result := CalcExcessBlobGas(tt.excess, tt.blobs*params.BlobTxBlobGasPerBlob)
 		if result != tt.want {
-			t.Errorf("excess blob gas mismatch: have %v, want %v", result, tt.want)
+			t.Errorf("test %d: excess blob gas mismatch: have %v, want %v", i, result, tt.want)
 		}
 	}
 }
@@ -63,9 +63,9 @@ func TestCalcBlobFee(t *testing.T) {
 		blobfee       int64
 	}{
 		{0, 1},
-		{1542706, 1},
+		{2314057, 1},
-		{1542707, 2},
+		{2314058, 2},
-		{10 * 1024 * 1024, 111},
+		{10 * 1024 * 1024, 23},
 	}
 	for i, tt := range tests {
 		have := CalcBlobFee(tt.excessBlobGas)

consensus/misc/gaslimit.go

@@ -17,7 +17,6 @@
 package misc
 
 import (
-	"errors"
 	"fmt"
 
 	"github.com/ethereum/go-ethereum/params"
@@ -36,7 +35,7 @@ func VerifyGaslimit(parentGasLimit, headerGasLimit uint64) error {
 		return fmt.Errorf("invalid gas limit: have %d, want %d +-= %d", headerGasLimit, parentGasLimit, limit-1)
 	}
 	if headerGasLimit < params.MinGasLimit {
-		return errors.New("invalid gas limit below 5000")
+		return fmt.Errorf("invalid gas limit below %d", params.MinGasLimit)
 	}
 	return nil
 }

core/asm/compiler.go

@@ -17,6 +17,8 @@
 package asm
 
 import (
+	"encoding/hex"
+	"errors"
 	"fmt"
 	"math/big"
 	"os"
@@ -30,7 +32,7 @@ import (
 // and holds the tokens for the program.
 type Compiler struct {
 	tokens []token
-	binary []interface{}
+	out    []byte
 
 	labels map[string]int
 
@@ -50,12 +52,10 @@ func NewCompiler(debug bool) *Compiler {
 // Feed feeds tokens in to ch and are interpreted by
 // the compiler.
 //
-// feed is the first pass in the compile stage as it
+// feed is the first pass in the compile stage as it collects the used labels in the
-// collects the used labels in the program and keeps a
+// program and keeps a program counter which is used to determine the locations of the
-// program counter which is used to determine the locations
+// jump dests. The labels can than be used in the second stage to push labels and
-// of the jump dests. The labels can than be used in the
+// determine the right position.
-// second stage to push labels and determine the right
-// position.
 func (c *Compiler) Feed(ch <-chan token) {
 	var prev token
 	for i := range ch {
@@ -79,7 +79,6 @@ func (c *Compiler) Feed(ch <-chan token) {
 				c.pc++
 			}
 		}
-
 		c.tokens = append(c.tokens, i)
 		prev = i
 	}
@@ -88,12 +87,11 @@ func (c *Compiler) Feed(ch <-chan token) {
 	}
 }
 
-// Compile compiles the current tokens and returns a
+// Compile compiles the current tokens and returns a binary string that can be interpreted
-// binary string that can be interpreted by the EVM
+// by the EVM and an error if it failed.
-// and an error if it failed.
 //
-// compile is the second stage in the compile phase
+// compile is the second stage in the compile phase which compiles the tokens to EVM
-// which compiles the tokens to EVM instructions.
+// instructions.
 func (c *Compiler) Compile() (string, []error) {
 	var errors []error
 	// continue looping over the tokens until
@@ -105,16 +103,8 @@ func (c *Compiler) Compile() (string, []error) {
 	}
 
 	// turn the binary to hex
-	var bin strings.Builder
+	h := hex.EncodeToString(c.out)
-	for _, v := range c.binary {
+	return h, errors
-		switch v := v.(type) {
-		case vm.OpCode:
-			bin.WriteString(fmt.Sprintf("%x", []byte{byte(v)}))
-		case []byte:
-			bin.WriteString(fmt.Sprintf("%x", v))
-		}
-	}
-	return bin.String(), errors
 }
 
 // next returns the next token and increments the
@@ -156,87 +146,114 @@ func (c *Compiler) compileLine() error {
 	return nil
 }
 
-// compileNumber compiles the number to bytes
+// parseNumber compiles the number to bytes
-func (c *Compiler) compileNumber(element token) {
+func parseNumber(tok token) ([]byte, error) {
-	num := math.MustParseBig256(element.text).Bytes()
+	if tok.typ != number {
-	if len(num) == 0 {
+		panic("parseNumber of non-number token")
-		num = []byte{0}
 	}
-	c.pushBin(num)
+	num, ok := math.ParseBig256(tok.text)
+	if !ok {
+		return nil, errors.New("invalid number")
+	}
+	bytes := num.Bytes()
+	if len(bytes) == 0 {
+		bytes = []byte{0}
+	}
+	return bytes, nil
 }
 
 // compileElement compiles the element (push & label or both)
 // to a binary representation and may error if incorrect statements
 // where fed.
 func (c *Compiler) compileElement(element token) error {
-	// check for a jump. jumps must be read and compiled
+	switch {
-	// from right to left.
+	case isJump(element.text):
-	if isJump(element.text) {
+		return c.compileJump(element.text)
-		rvalue := c.next()
+	case isPush(element.text):
-		switch rvalue.typ {
+		return c.compilePush()
-		case number:
+	default:
-			// TODO figure out how to return the error properly
+		c.outputOpcode(toBinary(element.text))
-			c.compileNumber(rvalue)
-		case stringValue:
-			// strings are quoted, remove them.
-			c.pushBin(rvalue.text[1 : len(rvalue.text)-2])
-		case label:
-			c.pushBin(vm.PUSH4)
-			pos := big.NewInt(int64(c.labels[rvalue.text])).Bytes()
-			pos = append(make([]byte, 4-len(pos)), pos...)
-			c.pushBin(pos)
-		case lineEnd:
-			c.pos--
-		default:
-			return compileErr(rvalue, rvalue.text, "number, string or label")
-		}
-		// push the operation
-		c.pushBin(toBinary(element.text))
 		return nil
-	} else if isPush(element.text) {
-		// handle pushes. pushes are read from left to right.
-		var value []byte
-
-		rvalue := c.next()
-		switch rvalue.typ {
-		case number:
-			value = math.MustParseBig256(rvalue.text).Bytes()
-			if len(value) == 0 {
-				value = []byte{0}
-			}
-		case stringValue:
-			value = []byte(rvalue.text[1 : len(rvalue.text)-1])
-		case label:
-			value = big.NewInt(int64(c.labels[rvalue.text])).Bytes()
-			value = append(make([]byte, 4-len(value)), value...)
-		default:
-			return compileErr(rvalue, rvalue.text, "number, string or label")
-		}
-
-		if len(value) > 32 {
-			return fmt.Errorf("%d type error: unsupported string or number with size > 32", rvalue.lineno)
-		}
-
-		c.pushBin(vm.OpCode(int(vm.PUSH1) - 1 + len(value)))
-		c.pushBin(value)
-	} else {
-		c.pushBin(toBinary(element.text))
 	}
+}
 
+func (c *Compiler) compileJump(jumpType string) error {
+	rvalue := c.next()
+	switch rvalue.typ {
+	case number:
+		numBytes, err := parseNumber(rvalue)
+		if err != nil {
+			return err
+		}
+		c.outputBytes(numBytes)
+
+	case stringValue:
+		// strings are quoted, remove them.
+		str := rvalue.text[1 : len(rvalue.text)-2]
+		c.outputBytes([]byte(str))
+
+	case label:
+		c.outputOpcode(vm.PUSH4)
+		pos := big.NewInt(int64(c.labels[rvalue.text])).Bytes()
+		pos = append(make([]byte, 4-len(pos)), pos...)
+		c.outputBytes(pos)
+
+	case lineEnd:
+		// push without argument is supported, it just takes the destination from the stack.
+		c.pos--
+
+	default:
+		return compileErr(rvalue, rvalue.text, "number, string or label")
+	}
+	// push the operation
+	c.outputOpcode(toBinary(jumpType))
+	return nil
+}
+
+func (c *Compiler) compilePush() error {
+	// handle pushes. pushes are read from left to right.
+	var value []byte
+	rvalue := c.next()
+	switch rvalue.typ {
+	case number:
+		value = math.MustParseBig256(rvalue.text).Bytes()
+		if len(value) == 0 {
+			value = []byte{0}
+		}
+	case stringValue:
+		value = []byte(rvalue.text[1 : len(rvalue.text)-1])
+	case label:
+		value = big.NewInt(int64(c.labels[rvalue.text])).Bytes()
+		value = append(make([]byte, 4-len(value)), value...)
+	default:
+		return compileErr(rvalue, rvalue.text, "number, string or label")
+	}
+	if len(value) > 32 {
+		return fmt.Errorf("%d: string or number size > 32 bytes", rvalue.lineno+1)
+	}
+	c.outputOpcode(vm.OpCode(int(vm.PUSH1) - 1 + len(value)))
+	c.outputBytes(value)
 	return nil
 }
 
 // compileLabel pushes a jumpdest to the binary slice.
 func (c *Compiler) compileLabel() {
-	c.pushBin(vm.JUMPDEST)
+	c.outputOpcode(vm.JUMPDEST)
 }
 
-// pushBin pushes the value v to the binary stack.
+func (c *Compiler) outputOpcode(op vm.OpCode) {
-func (c *Compiler) pushBin(v interface{}) {
 	if c.debug {
-		fmt.Printf("%d: %v\n", len(c.binary), v)
+		fmt.Printf("%d: %v\n", len(c.out), op)
 	}
-	c.binary = append(c.binary, v)
+	c.out = append(c.out, byte(op))
+}
+
+// output pushes the value v to the binary stack.
+func (c *Compiler) outputBytes(b []byte) {
+	if c.debug {
+		fmt.Printf("%d: %x\n", len(c.out), b)
+	}
+	c.out = append(c.out, b...)
 }
 
 // isPush returns whether the string op is either any of
@@ -263,13 +280,13 @@ type compileError struct {
 }
 
 func (err compileError) Error() string {
-	return fmt.Sprintf("%d syntax error: unexpected %v, expected %v", err.lineno, err.got, err.want)
+	return fmt.Sprintf("%d: syntax error: unexpected %v, expected %v", err.lineno, err.got, err.want)
 }
 
 func compileErr(c token, got, want string) error {
 	return compileError{
 		got:    got,
 		want:   want,
-		lineno: c.lineno,
+		lineno: c.lineno + 1,
 	}
 }

core/asm/compiler_test.go

@@ -54,6 +54,14 @@ func TestCompiler(t *testing.T) {
 `,
 			output: "6300000006565b",
 		},
+		{
+			input: `
+	JUMP @label
+label: ;; comment
+	ADD ;; comment
+`,
+			output: "6300000006565b01",
+		},
 	}
 	for _, test := range tests {
 		ch := Lex([]byte(test.input), false)

core/asm/lex_test.go

@@ -72,6 +72,16 @@ func TestLexer(t *testing.T) {
 			input:  "@label123",
 			tokens: []token{{typ: lineStart}, {typ: label, text: "label123"}, {typ: eof}},
 		},
+		// comment after label
+		{
+			input:  "@label123 ;; comment",
+			tokens: []token{{typ: lineStart}, {typ: label, text: "label123"}, {typ: eof}},
+		},
+		// comment after instruction
+		{
+			input:  "push 3 ;; comment\nadd",
+			tokens: []token{{typ: lineStart}, {typ: element, text: "push"}, {typ: number, text: "3"}, {typ: lineEnd, text: "\n"}, {typ: lineStart, lineno: 1}, {typ: element, lineno: 1, text: "add"}, {typ: eof, lineno: 1}},
+		},
 	}
 
 	for _, test := range tests {

core/asm/lexer.go

@@ -42,6 +42,8 @@ type token struct {
 // is able to parse and return.
 type tokenType int
 
+//go:generate go run golang.org/x/tools/cmd/stringer -type tokenType
+
 const (
 	eof              tokenType = iota // end of file
 	lineStart                         // emitted when a line starts
@@ -52,31 +54,13 @@ const (
 	labelDef                          // label definition is emitted when a new label is found
 	number                            // number is emitted when a number is found
 	stringValue                       // stringValue is emitted when a string has been found
-
-	Numbers            = "1234567890"                                           // characters representing any decimal number
-	HexadecimalNumbers = Numbers + "aAbBcCdDeEfF"                               // characters representing any hexadecimal
-	Alpha              = "abcdefghijklmnopqrstuwvxyzABCDEFGHIJKLMNOPQRSTUWVXYZ" // characters representing alphanumeric
 )
 
-// String implements stringer
+const (
-func (it tokenType) String() string {
+	decimalNumbers = "1234567890"                                           // characters representing any decimal number
-	if int(it) > len(stringtokenTypes) {
+	hexNumbers     = decimalNumbers + "aAbBcCdDeEfF"                        // characters representing any hexadecimal
-		return "invalid"
+	alpha          = "abcdefghijklmnopqrstuwvxyzABCDEFGHIJKLMNOPQRSTUWVXYZ" // characters representing alphanumeric
-	}
+)
-	return stringtokenTypes[it]
-}
-
-var stringtokenTypes = []string{
-	eof:              "EOF",
-	lineStart:        "new line",
-	lineEnd:          "end of line",
-	invalidStatement: "invalid statement",
-	element:          "element",
-	label:            "label",
-	labelDef:         "label definition",
-	number:           "number",
-	stringValue:      "string",
-}
 
 // lexer is the basic construct for parsing
 // source code and turning them in to tokens.
@@ -200,7 +184,6 @@ func lexLine(l *lexer) stateFn {
 			l.emit(lineEnd)
 			l.ignore()
 			l.lineno++
-
 			l.emit(lineStart)
 		case r == ';' && l.peek() == ';':
 			return lexComment
@@ -225,6 +208,7 @@ func lexLine(l *lexer) stateFn {
 // of the line and discards the text.
 func lexComment(l *lexer) stateFn {
 	l.acceptRunUntil('\n')
+	l.backup()
 	l.ignore()
 
 	return lexLine
@@ -234,7 +218,7 @@ func lexComment(l *lexer) stateFn {
 // the lex text state function to advance the parsing
 // process.
 func lexLabel(l *lexer) stateFn {
-	l.acceptRun(Alpha + "_" + Numbers)
+	l.acceptRun(alpha + "_" + decimalNumbers)
 
 	l.emit(label)
 
@@ -253,9 +237,9 @@ func lexInsideString(l *lexer) stateFn {
 }
 
 func lexNumber(l *lexer) stateFn {
-	acceptance := Numbers
+	acceptance := decimalNumbers
 	if l.accept("xX") {
-		acceptance = HexadecimalNumbers
+		acceptance = hexNumbers
 	}
 	l.acceptRun(acceptance)
 
@@ -265,7 +249,7 @@ func lexNumber(l *lexer) stateFn {
 }
 
 func lexElement(l *lexer) stateFn {
-	l.acceptRun(Alpha + "_" + Numbers)
+	l.acceptRun(alpha + "_" + decimalNumbers)
 
 	if l.peek() == ':' {
 		l.emit(labelDef)

core/asm/tokentype_string.go

@@ -0,0 +1,31 @@
+// Code generated by "stringer -type tokenType"; DO NOT EDIT.
+
+package asm
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[eof-0]
+	_ = x[lineStart-1]
+	_ = x[lineEnd-2]
+	_ = x[invalidStatement-3]
+	_ = x[element-4]
+	_ = x[label-5]
+	_ = x[labelDef-6]
+	_ = x[number-7]
+	_ = x[stringValue-8]
+}
+
+const _tokenType_name = "eoflineStartlineEndinvalidStatementelementlabellabelDefnumberstringValue"
+
+var _tokenType_index = [...]uint8{0, 3, 12, 19, 35, 42, 47, 55, 61, 72}
+
+func (i tokenType) String() string {
+	if i < 0 || i >= tokenType(len(_tokenType_index)-1) {
+		return "tokenType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _tokenType_name[_tokenType_index[i]:_tokenType_index[i+1]]
+}

core/block_validator.go

@@ -68,6 +68,7 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 	if hash := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); hash != header.TxHash {
 		return fmt.Errorf("transaction root hash mismatch (header value %x, calculated %x)", header.TxHash, hash)
 	}
+
 	// Withdrawals are present after the Shanghai fork.
 	if header.WithdrawalsHash != nil {
 		// Withdrawals list must be present in body after Shanghai.
@@ -81,14 +82,23 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 		// Withdrawals are not allowed prior to Shanghai fork
 		return errors.New("withdrawals present in block body")
 	}
+
 	// Blob transactions may be present after the Cancun fork.
 	var blobs int
-	for _, tx := range block.Transactions() {
+	for i, tx := range block.Transactions() {
 		// Count the number of blobs to validate against the header's blobGasUsed
 		blobs += len(tx.BlobHashes())
+
+		// If the tx is a blob tx, it must NOT have a sidecar attached to be valid in a block.
+		if tx.BlobTxSidecar() != nil {
+			return fmt.Errorf("unexpected blob sidecar in transaction at index %d", i)
+		}
+
 		// The individual checks for blob validity (version-check + not empty)
-		// happens in the state_transition check.
+		// happens in StateTransition.
 	}
+
+	// Check blob gas usage.
 	if header.BlobGasUsed != nil {
 		if want := *header.BlobGasUsed / params.BlobTxBlobGasPerBlob; uint64(blobs) != want { // div because the header is surely good vs the body might be bloated
 			return fmt.Errorf("blob gas used mismatch (header %v, calculated %v)", *header.BlobGasUsed, blobs*params.BlobTxBlobGasPerBlob)
@@ -98,6 +108,8 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 			return errors.New("data blobs present in block body")
 		}
 	}
+
+	// Ancestor block must be known.
 	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
 		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
 			return consensus.ErrUnknownAncestor

core/block_validator_test.go

@@ -35,6 +35,11 @@ import (
 
 // Tests that simple header verification works, for both good and bad blocks.
 func TestHeaderVerification(t *testing.T) {
+	testHeaderVerification(t, rawdb.HashScheme)
+	testHeaderVerification(t, rawdb.PathScheme)
+}
+
+func testHeaderVerification(t *testing.T, scheme string) {
 	// Create a simple chain to verify
 	var (
 		gspec        = &Genesis{Config: params.TestChainConfig}
@@ -45,7 +50,7 @@ func TestHeaderVerification(t *testing.T) {
 		headers[i] = block.Header()
 	}
 	// Run the header checker for blocks one-by-one, checking for both valid and invalid nonces
-	chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
+	chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
 	defer chain.Stop()
 
 	for i := 0; i < len(blocks); i++ {

core/blockchain.go

@@ -48,6 +48,8 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
+	"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
 	"golang.org/x/exp/slices"
 )
 
@@ -58,6 +60,8 @@ var (
 	headFinalizedBlockGauge = metrics.NewRegisteredGauge("chain/head/finalized", nil)
 	headSafeBlockGauge      = metrics.NewRegisteredGauge("chain/head/safe", nil)
 
+	chainInfoGauge = metrics.NewRegisteredGaugeInfo("chain/info", nil)
+
 	accountReadTimer   = metrics.NewRegisteredTimer("chain/account/reads", nil)
 	accountHashTimer   = metrics.NewRegisteredTimer("chain/account/hashes", nil)
 	accountUpdateTimer = metrics.NewRegisteredTimer("chain/account/updates", nil)
@@ -128,7 +132,7 @@ const (
 )
 
 // CacheConfig contains the configuration values for the trie database
-// that's resident in a blockchain.
+// and state snapshot these are resident in a blockchain.
 type CacheConfig struct {
 	TrieCleanLimit      int           // Memory allowance (MB) to use for caching trie nodes in memory
 	TrieCleanNoPrefetch bool          // Whether to disable heuristic state prefetching for followup blocks
@@ -137,11 +141,31 @@ type CacheConfig struct {
 	TrieTimeLimit       time.Duration // Time limit after which to flush the current in-memory trie to disk
 	SnapshotLimit       int           // Memory allowance (MB) to use for caching snapshot entries in memory
 	Preimages           bool          // Whether to store preimage of trie key to the disk
+	StateHistory        uint64        // Number of blocks from head whose state histories are reserved.
+	StateScheme         string        // Scheme used to store ethereum states and merkle tree nodes on top
 
 	SnapshotNoBuild bool // Whether the background generation is allowed
 	SnapshotWait    bool // Wait for snapshot construction on startup. TODO(karalabe): This is a dirty hack for testing, nuke it
 }
 
+// triedbConfig derives the configures for trie database.
+func (c *CacheConfig) triedbConfig() *trie.Config {
+	config := &trie.Config{Preimages: c.Preimages}
+	if c.StateScheme == rawdb.HashScheme {
+		config.HashDB = &hashdb.Config{
+			CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
+		}
+	}
+	if c.StateScheme == rawdb.PathScheme {
+		config.PathDB = &pathdb.Config{
+			StateHistory:   c.StateHistory,
+			CleanCacheSize: c.TrieCleanLimit * 1024 * 1024,
+			DirtyCacheSize: c.TrieDirtyLimit * 1024 * 1024,
+		}
+	}
+	return config
+}
+
 // defaultCacheConfig are the default caching values if none are specified by the
 // user (also used during testing).
 var defaultCacheConfig = &CacheConfig{
@@ -150,6 +174,15 @@ var defaultCacheConfig = &CacheConfig{
 	TrieTimeLimit:  5 * time.Minute,
 	SnapshotLimit:  256,
 	SnapshotWait:   true,
+	StateScheme:    rawdb.HashScheme,
+}
+
+// DefaultCacheConfigWithScheme returns a deep copied default cache config with
+// a provided trie node scheme.
+func DefaultCacheConfigWithScheme(scheme string) *CacheConfig {
+	config := *defaultCacheConfig
+	config.StateScheme = scheme
+	return &config
 }
 
 // BlockChain represents the canonical chain given a database with a genesis
@@ -235,10 +268,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 		cacheConfig = defaultCacheConfig
 	}
 	// Open trie database with provided config
-	triedb := trie.NewDatabaseWithConfig(db, &trie.Config{
+	triedb := trie.NewDatabase(db, cacheConfig.triedbConfig())
-		Cache:     cacheConfig.TrieCleanLimit,
+
-		Preimages: cacheConfig.Preimages,
-	})
 	// Setup the genesis block, commit the provided genesis specification
 	// to database if the genesis block is not present yet, or load the
 	// stored one from database.
@@ -293,6 +324,9 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	bc.currentFinalBlock.Store(nil)
 	bc.currentSafeBlock.Store(nil)
 
+	// Update chain info data metrics
+	chainInfoGauge.Update(metrics.GaugeInfoValue{"chain_id": bc.chainConfig.ChainID.String()})
+
 	// If Geth is initialized with an external ancient store, re-initialize the
 	// missing chain indexes and chain flags. This procedure can survive crash
 	// and can be resumed in next restart since chain flags are updated in last step.
@@ -303,31 +337,41 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	if err := bc.loadLastState(); err != nil {
 		return nil, err
 	}
-	// Make sure the state associated with the block is available
+	// Make sure the state associated with the block is available, or log out
+	// if there is no available state, waiting for state sync.
 	head := bc.CurrentBlock()
 	if !bc.HasState(head.Root) {
-		// Head state is missing, before the state recovery, find out the
+		if head.Number.Uint64() == 0 {
-		// disk layer point of snapshot(if it's enabled). Make sure the
+			// The genesis state is missing, which is only possible in the path-based
-		// rewound point is lower than disk layer.
+			// scheme. This situation occurs when the initial state sync is not finished
-		var diskRoot common.Hash
+			// yet, or the chain head is rewound below the pivot point. In both scenario,
-		if bc.cacheConfig.SnapshotLimit > 0 {
+			// there is no possible recovery approach except for rerunning a snap sync.
-			diskRoot = rawdb.ReadSnapshotRoot(bc.db)
+			// Do nothing here until the state syncer picks it up.
-		}
+			log.Info("Genesis state is missing, wait state sync")
-		if diskRoot != (common.Hash{}) {
-			log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash(), "snaproot", diskRoot)
-
-			snapDisk, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, diskRoot, true)
-			if err != nil {
-				return nil, err
-			}
-			// Chain rewound, persist old snapshot number to indicate recovery procedure
-			if snapDisk != 0 {
-				rawdb.WriteSnapshotRecoveryNumber(bc.db, snapDisk)
-			}
 		} else {
-			log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash())
+			// Head state is missing, before the state recovery, find out the
-			if _, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, common.Hash{}, true); err != nil {
+			// disk layer point of snapshot(if it's enabled). Make sure the
-				return nil, err
+			// rewound point is lower than disk layer.
+			var diskRoot common.Hash
+			if bc.cacheConfig.SnapshotLimit > 0 {
+				diskRoot = rawdb.ReadSnapshotRoot(bc.db)
+			}
+			if diskRoot != (common.Hash{}) {
+				log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash(), "snaproot", diskRoot)
+
+				snapDisk, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, diskRoot, true)
+				if err != nil {
+					return nil, err
+				}
+				// Chain rewound, persist old snapshot number to indicate recovery procedure
+				if snapDisk != 0 {
+					rawdb.WriteSnapshotRecoveryNumber(bc.db, snapDisk)
+				}
+			} else {
+				log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash())
+				if _, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, common.Hash{}, true); err != nil {
+					return nil, err
+				}
 			}
 		}
 	}
@@ -632,7 +676,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 					if root != (common.Hash{}) && !beyondRoot && newHeadBlock.Root() == root {
 						beyondRoot, rootNumber = true, newHeadBlock.NumberU64()
 					}
-					if !bc.HasState(newHeadBlock.Root()) {
+					if !bc.HasState(newHeadBlock.Root()) && !bc.stateRecoverable(newHeadBlock.Root()) {
 						log.Trace("Block state missing, rewinding further", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
 						if pivot == nil || newHeadBlock.NumberU64() > *pivot {
 							parent := bc.GetBlock(newHeadBlock.ParentHash(), newHeadBlock.NumberU64()-1)
@@ -648,20 +692,14 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 						}
 					}
 					if beyondRoot || newHeadBlock.NumberU64() == 0 {
-						if newHeadBlock.NumberU64() == 0 {
+						if !bc.HasState(newHeadBlock.Root()) && bc.stateRecoverable(newHeadBlock.Root()) {
-							// Recommit the genesis state into disk in case the rewinding destination
+							// Rewind to a block with recoverable state. If the state is
-							// is genesis block and the relevant state is gone. In the future this
+							// missing, run the state recovery here.
-							// rewinding destination can be the earliest block stored in the chain
+							if err := bc.triedb.Recover(newHeadBlock.Root()); err != nil {
-							// if the historical chain pruning is enabled. In that case the logic
+								log.Crit("Failed to rollback state", "err", err) // Shouldn't happen
-							// needs to be improved here.
-							if !bc.HasState(bc.genesisBlock.Root()) {
-								if err := CommitGenesisState(bc.db, bc.triedb, bc.genesisBlock.Hash()); err != nil {
-									log.Crit("Failed to commit genesis state", "err", err)
-								}
-								log.Debug("Recommitted genesis state to disk")
 							}
+							log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
 						}
-						log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
 						break
 					}
 					log.Debug("Skipping block with threshold state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "root", newHeadBlock.Root())
@@ -676,6 +714,15 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 			// to low, so it's safe to update in-memory markers directly.
 			bc.currentBlock.Store(newHeadBlock.Header())
 			headBlockGauge.Update(int64(newHeadBlock.NumberU64()))
+
+			// The head state is missing, which is only possible in the path-based
+			// scheme. This situation occurs when the chain head is rewound below
+			// the pivot point. In this scenario, there is no possible recovery
+			// approach except for rerunning a snap sync. Do nothing here until the
+			// state syncer picks it up.
+			if !bc.HasState(newHeadBlock.Root()) {
+				log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number(), "hash", newHeadBlock.Hash())
+			}
 		}
 		// Rewind the snap block in a simpleton way to the target head
 		if currentSnapBlock := bc.CurrentSnapBlock(); currentSnapBlock != nil && header.Number.Uint64() < currentSnapBlock.Number.Uint64() {
@@ -772,7 +819,13 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
 	if block == nil {
 		return fmt.Errorf("non existent block [%x..]", hash[:4])
 	}
+	// Reset the trie database with the fresh snap synced state.
 	root := block.Root()
+	if bc.triedb.Scheme() == rawdb.PathScheme {
+		if err := bc.triedb.Enable(root); err != nil {
+			return err
+		}
+	}
 	if !bc.HasState(root) {
 		return fmt.Errorf("non existent state [%x..]", root[:4])
 	}
@@ -937,41 +990,47 @@ func (bc *BlockChain) Stop() {
 			log.Error("Failed to journal state snapshot", "err", err)
 		}
 	}
+	if bc.triedb.Scheme() == rawdb.PathScheme {
+		// Ensure that the in-memory trie nodes are journaled to disk properly.
+		if err := bc.triedb.Journal(bc.CurrentBlock().Root); err != nil {
+			log.Info("Failed to journal in-memory trie nodes", "err", err)
+		}
+	} else {
+		// Ensure the state of a recent block is also stored to disk before exiting.
+		// We're writing three different states to catch different restart scenarios:
+		//  - HEAD:     So we don't need to reprocess any blocks in the general case
+		//  - HEAD-1:   So we don't do large reorgs if our HEAD becomes an uncle
+		//  - HEAD-127: So we have a hard limit on the number of blocks reexecuted
+		if !bc.cacheConfig.TrieDirtyDisabled {
+			triedb := bc.triedb
 
-	// Ensure the state of a recent block is also stored to disk before exiting.
+			for _, offset := range []uint64{0, 1, TriesInMemory - 1} {
-	// We're writing three different states to catch different restart scenarios:
+				if number := bc.CurrentBlock().Number.Uint64(); number > offset {
-	//  - HEAD:     So we don't need to reprocess any blocks in the general case
+					recent := bc.GetBlockByNumber(number - offset)
-	//  - HEAD-1:   So we don't do large reorgs if our HEAD becomes an uncle
-	//  - HEAD-127: So we have a hard limit on the number of blocks reexecuted
-	if !bc.cacheConfig.TrieDirtyDisabled {
-		triedb := bc.triedb
 
-		for _, offset := range []uint64{0, 1, TriesInMemory - 1} {
+					log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
-			if number := bc.CurrentBlock().Number.Uint64(); number > offset {
+					if err := triedb.Commit(recent.Root(), true); err != nil {
-				recent := bc.GetBlockByNumber(number - offset)
+						log.Error("Failed to commit recent state trie", "err", err)
-
+					}
-				log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
+				}
-				if err := triedb.Commit(recent.Root(), true); err != nil {
+			}
+			if snapBase != (common.Hash{}) {
+				log.Info("Writing snapshot state to disk", "root", snapBase)
+				if err := triedb.Commit(snapBase, true); err != nil {
 					log.Error("Failed to commit recent state trie", "err", err)
 				}
 			}
-		}
+			for !bc.triegc.Empty() {
-		if snapBase != (common.Hash{}) {
+				triedb.Dereference(bc.triegc.PopItem())
-			log.Info("Writing snapshot state to disk", "root", snapBase)
+			}
-			if err := triedb.Commit(snapBase, true); err != nil {
+			if _, nodes, _ := triedb.Size(); nodes != 0 { // all memory is contained within the nodes return for hashdb
-				log.Error("Failed to commit recent state trie", "err", err)
+				log.Error("Dangling trie nodes after full cleanup")
 			}
 		}
-		for !bc.triegc.Empty() {
-			triedb.Dereference(bc.triegc.PopItem())
-		}
-		if size, _ := triedb.Size(); size != 0 {
-			log.Error("Dangling trie nodes after full cleanup")
-		}
 	}
-	// Flush the collected preimages to disk
+	// Close the trie database, release all the held resources as the last step.
-	if err := bc.stateCache.TrieDB().Close(); err != nil {
+	if err := bc.triedb.Close(); err != nil {
-		log.Error("Failed to close trie db", "err", err)
+		log.Error("Failed to close trie database", "err", err)
 	}
 	log.Info("Blockchain stopped")
 }
@@ -1028,19 +1087,30 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 		ancientReceipts, liveReceipts []types.Receipts
 	)
 	// Do a sanity check that the provided chain is actually ordered and linked
-	for i := 0; i < len(blockChain); i++ {
+	for i, block := range blockChain {
 		if i != 0 {
-			if blockChain[i].NumberU64() != blockChain[i-1].NumberU64()+1 || blockChain[i].ParentHash() != blockChain[i-1].Hash() {
+			prev := blockChain[i-1]
-				log.Error("Non contiguous receipt insert", "number", blockChain[i].Number(), "hash", blockChain[i].Hash(), "parent", blockChain[i].ParentHash(),
+			if block.NumberU64() != prev.NumberU64()+1 || block.ParentHash() != prev.Hash() {
-					"prevnumber", blockChain[i-1].Number(), "prevhash", blockChain[i-1].Hash())
+				log.Error("Non contiguous receipt insert",
-				return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])", i-1, blockChain[i-1].NumberU64(),
+					"number", block.Number(), "hash", block.Hash(), "parent", block.ParentHash(),
-					blockChain[i-1].Hash().Bytes()[:4], i, blockChain[i].NumberU64(), blockChain[i].Hash().Bytes()[:4], blockChain[i].ParentHash().Bytes()[:4])
+					"prevnumber", prev.Number(), "prevhash", prev.Hash())
+				return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])",
+					i-1, prev.NumberU64(), prev.Hash().Bytes()[:4],
+					i, block.NumberU64(), block.Hash().Bytes()[:4], block.ParentHash().Bytes()[:4])
 			}
 		}
-		if blockChain[i].NumberU64() <= ancientLimit {
+		if block.NumberU64() <= ancientLimit {
-			ancientBlocks, ancientReceipts = append(ancientBlocks, blockChain[i]), append(ancientReceipts, receiptChain[i])
+			ancientBlocks, ancientReceipts = append(ancientBlocks, block), append(ancientReceipts, receiptChain[i])
 		} else {
-			liveBlocks, liveReceipts = append(liveBlocks, blockChain[i]), append(liveReceipts, receiptChain[i])
+			liveBlocks, liveReceipts = append(liveBlocks, block), append(liveReceipts, receiptChain[i])
+		}
+
+		// Here we also validate that blob transactions in the block do not contain a sidecar.
+		// While the sidecar does not affect the block hash / tx hash, sending blobs within a block is not allowed.
+		for txIndex, tx := range block.Transactions() {
+			if tx.Type() == types.BlobTxType && tx.BlobTxSidecar() != nil {
+				return 0, fmt.Errorf("block #%d contains unexpected blob sidecar in tx at index %d", block.NumberU64(), txIndex)
+			}
 		}
 	}
 
@@ -1341,6 +1411,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 	if err != nil {
 		return err
 	}
+	// If node is running in path mode, skip explicit gc operation
+	// which is unnecessary in this mode.
+	if bc.triedb.Scheme() == rawdb.PathScheme {
+		return nil
+	}
 	// If we're running an archive node, always flush
 	if bc.cacheConfig.TrieDirtyDisabled {
 		return bc.triedb.Commit(root, false)
@@ -1349,15 +1424,15 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 	bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
 	bc.triegc.Push(root, -int64(block.NumberU64()))
 
-	current := block.NumberU64()
 	// Flush limits are not considered for the first TriesInMemory blocks.
+	current := block.NumberU64()
 	if current <= TriesInMemory {
 		return nil
 	}
 	// If we exceeded our memory allowance, flush matured singleton nodes to disk
 	var (
-		nodes, imgs = bc.triedb.Size()
+		_, nodes, imgs = bc.triedb.Size() // all memory is contained within the nodes return for hashdb
-		limit       = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024
+		limit          = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024
 	)
 	if nodes > limit || imgs > 4*1024*1024 {
 		bc.triedb.Cap(limit - ethdb.IdealBatchSize)
@@ -1793,8 +1868,12 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 		stats.processed++
 		stats.usedGas += usedGas
 
-		dirty, _ := bc.triedb.Size()
+		var snapDiffItems, snapBufItems common.StorageSize
-		stats.report(chain, it.index, dirty, setHead)
+		if bc.snaps != nil {
+			snapDiffItems, snapBufItems = bc.snaps.Size()
+		}
+		trieDiffNodes, trieBufNodes, _ := bc.triedb.Size()
+		stats.report(chain, it.index, snapDiffItems, snapBufItems, trieDiffNodes, trieBufNodes, setHead)
 
 		if !setHead {
 			// After merge we expect few side chains. Simply count
@@ -1936,6 +2015,12 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i
 	)
 	parent := it.previous()
 	for parent != nil && !bc.HasState(parent.Root) {
+		if bc.stateRecoverable(parent.Root) {
+			if err := bc.triedb.Recover(parent.Root); err != nil {
+				return 0, err
+			}
+			break
+		}
 		hashes = append(hashes, parent.Hash())
 		numbers = append(numbers, parent.Number.Uint64())
 
@@ -1992,6 +2077,12 @@ func (bc *BlockChain) recoverAncestors(block *types.Block) (common.Hash, error)
 		parent  = block
 	)
 	for parent != nil && !bc.HasState(parent.Root()) {
+		if bc.stateRecoverable(parent.Root()) {
+			if err := bc.triedb.Recover(parent.Root()); err != nil {
+				return common.Hash{}, err
+			}
+			break
+		}
 		hashes = append(hashes, parent.Hash())
 		numbers = append(numbers, parent.NumberU64())
 		parent = bc.GetBlock(parent.ParentHash(), parent.NumberU64()-1)
@@ -2336,6 +2427,12 @@ func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool {
 func (bc *BlockChain) indexBlocks(tail *uint64, head uint64, done chan struct{}) {
 	defer func() { close(done) }()
 
+	// If head is 0, it means the chain is just initialized and no blocks are inserted,
+	// so don't need to indexing anything.
+	if head == 0 {
+		return
+	}
+
 	// The tail flag is not existent, it means the node is just initialized
 	// and all blocks(may from ancient store) are not indexed yet.
 	if tail == nil {
@@ -2393,6 +2490,15 @@ func (bc *BlockChain) maintainTxIndex() {
 		return
 	}
 	defer sub.Unsubscribe()
+	log.Info("Initialized transaction indexer", "limit", bc.TxLookupLimit())
+
+	// Launch the initial processing if chain is not empty. This step is
+	// useful in these scenarios that chain has no progress and indexer
+	// is never triggered.
+	if head := rawdb.ReadHeadBlock(bc.db); head != nil {
+		done = make(chan struct{})
+		go bc.indexBlocks(rawdb.ReadTxIndexTail(bc.db), head.NumberU64(), done)
+	}
 
 	for {
 		select {

core/blockchain_insert.go

@@ -39,7 +39,7 @@ const statsReportLimit = 8 * time.Second
 
 // report prints statistics if some number of blocks have been processed
 // or more than a few seconds have passed since the last message.
-func (st *insertStats) report(chain []*types.Block, index int, dirty common.StorageSize, setHead bool) {
+func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, snapBufItems, trieDiffNodes, triebufNodes common.StorageSize, setHead bool) {
 	// Fetch the timings for the batch
 	var (
 		now     = mclock.Now()
@@ -63,7 +63,16 @@ func (st *insertStats) report(chain []*types.Block, index int, dirty common.Stor
 		if timestamp := time.Unix(int64(end.Time()), 0); time.Since(timestamp) > time.Minute {
 			context = append(context, []interface{}{"age", common.PrettyAge(timestamp)}...)
 		}
-		context = append(context, []interface{}{"dirty", dirty}...)
+		if snapDiffItems != 0 || snapBufItems != 0 { // snapshots enabled
+			context = append(context, []interface{}{"snapdiffs", snapDiffItems}...)
+			if snapBufItems != 0 { // future snapshot refactor
+				context = append(context, []interface{}{"snapdirty", snapBufItems}...)
+			}
+		}
+		if trieDiffNodes != 0 { // pathdb
+			context = append(context, []interface{}{"triediffs", trieDiffNodes}...)
+		}
+		context = append(context, []interface{}{"triedirty", triebufNodes}...)
 
 		if st.queued > 0 {
 			context = append(context, []interface{}{"queued", st.queued}...)

core/blockchain_reader.go

@@ -293,10 +293,16 @@ func (bc *BlockChain) HasBlockAndState(hash common.Hash, number uint64) bool {
 	return bc.HasState(block.Root())
 }
 
-// TrieNode retrieves a blob of data associated with a trie node
+// stateRecoverable checks if the specified state is recoverable.
-// either from ephemeral in-memory cache, or from persistent storage.
+// Note, this function assumes the state is not present, because
-func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) {
+// state is not treated as recoverable if it's available, thus
-	return bc.stateCache.TrieDB().Node(hash)
+// false will be returned in this case.
+func (bc *BlockChain) stateRecoverable(root common.Hash) bool {
+	if bc.triedb.Scheme() == rawdb.HashScheme {
+		return false
+	}
+	result, _ := bc.triedb.Recoverable(root)
+	return result
 }
 
 // ContractCodeWithPrefix retrieves a blob of data associated with a contract

core/blockchain_repair_test.go

@@ -22,6 +22,7 @@ package core
 
 import (
 	"math/big"
+	"path"
 	"testing"
 	"time"
 
@@ -1749,16 +1750,24 @@ func testLongReorgedSnapSyncingDeepRepair(t *testing.T, snapshots bool) {
 }
 
 func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
+		testRepairWithScheme(t, tt, snapshots, scheme)
+	}
+}
+
+func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) {
 	// It's hard to follow the test case, visualize the input
 	//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
 	// fmt.Println(tt.dump(true))
 
 	// Create a temporary persistent database
 	datadir := t.TempDir()
+	ancient := path.Join(datadir, "ancient")
 
 	db, err := rawdb.Open(rawdb.OpenOptions{
 		Directory:         datadir,
-		AncientsDirectory: datadir,
+		AncientsDirectory: ancient,
+		Ephemeral:         true,
 	})
 	if err != nil {
 		t.Fatalf("Failed to create persistent database: %v", err)
@@ -1777,6 +1786,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 			TrieDirtyLimit: 256,
 			TrieTimeLimit:  5 * time.Minute,
 			SnapshotLimit:  0, // Disable snapshot by default
+			StateScheme:    scheme,
 		}
 	)
 	defer engine.Close()
@@ -1806,7 +1816,9 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 		t.Fatalf("Failed to import canonical chain start: %v", err)
 	}
 	if tt.commitBlock > 0 {
-		chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false)
+		if err := chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false); err != nil {
+			t.Fatalf("Failed to flush trie state: %v", err)
+		}
 		if snapshots {
 			if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
 				t.Fatalf("Failed to flatten snapshots: %v", err)
@@ -1828,21 +1840,22 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 		rawdb.WriteLastPivotNumber(db, *tt.pivotBlock)
 	}
 	// Pull the plug on the database, simulating a hard crash
+	chain.triedb.Close()
 	db.Close()
 	chain.stopWithoutSaving()
 
 	// Start a new blockchain back up and see where the repair leads us
 	db, err = rawdb.Open(rawdb.OpenOptions{
 		Directory:         datadir,
-		AncientsDirectory: datadir,
+		AncientsDirectory: ancient,
+		Ephemeral:         true,
 	})
-
 	if err != nil {
 		t.Fatalf("Failed to reopen persistent database: %v", err)
 	}
 	defer db.Close()
 
-	newChain, err := NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil)
+	newChain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
@@ -1885,17 +1898,22 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 // In this case the snapshot layer of B3 is not created because of existent
 // state.
 func TestIssue23496(t *testing.T) {
+	testIssue23496(t, rawdb.HashScheme)
+	testIssue23496(t, rawdb.PathScheme)
+}
+
+func testIssue23496(t *testing.T, scheme string) {
 	// It's hard to follow the test case, visualize the input
 	//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
 
 	// Create a temporary persistent database
 	datadir := t.TempDir()
+	ancient := path.Join(datadir, "ancient")
 
 	db, err := rawdb.Open(rawdb.OpenOptions{
 		Directory:         datadir,
-		AncientsDirectory: datadir,
+		AncientsDirectory: ancient,
 	})
-
 	if err != nil {
 		t.Fatalf("Failed to create persistent database: %v", err)
 	}
@@ -1908,15 +1926,8 @@ func TestIssue23496(t *testing.T) {
 			BaseFee: big.NewInt(params.InitialBaseFee),
 		}
 		engine = ethash.NewFullFaker()
-		config = &CacheConfig{
-			TrieCleanLimit: 256,
-			TrieDirtyLimit: 256,
-			TrieTimeLimit:  5 * time.Minute,
-			SnapshotLimit:  256,
-			SnapshotWait:   true,
-		}
 	)
-	chain, err := NewBlockChain(db, config, gspec, nil, engine, vm.Config{}, nil, nil)
+	chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to create chain: %v", err)
 	}
@@ -1929,7 +1940,7 @@ func TestIssue23496(t *testing.T) {
 	if _, err := chain.InsertChain(blocks[:1]); err != nil {
 		t.Fatalf("Failed to import canonical chain start: %v", err)
 	}
-	chain.stateCache.TrieDB().Commit(blocks[0].Root(), false)
+	chain.triedb.Commit(blocks[0].Root(), false)
 
 	// Insert block B2 and commit the snapshot into disk
 	if _, err := chain.InsertChain(blocks[1:2]); err != nil {
@@ -1943,7 +1954,7 @@ func TestIssue23496(t *testing.T) {
 	if _, err := chain.InsertChain(blocks[2:3]); err != nil {
 		t.Fatalf("Failed to import canonical chain start: %v", err)
 	}
-	chain.stateCache.TrieDB().Commit(blocks[2].Root(), false)
+	chain.triedb.Commit(blocks[2].Root(), false)
 
 	// Insert the remaining blocks
 	if _, err := chain.InsertChain(blocks[3:]); err != nil {
@@ -1951,20 +1962,22 @@ func TestIssue23496(t *testing.T) {
 	}
 
 	// Pull the plug on the database, simulating a hard crash
+	chain.triedb.Close()
 	db.Close()
 	chain.stopWithoutSaving()
 
 	// Start a new blockchain back up and see where the repair leads us
 	db, err = rawdb.Open(rawdb.OpenOptions{
 		Directory:         datadir,
-		AncientsDirectory: datadir,
+		AncientsDirectory: ancient,
+		Ephemeral:         true,
 	})
 	if err != nil {
 		t.Fatalf("Failed to reopen persistent database: %v", err)
 	}
 	defer db.Close()
 
-	chain, err = NewBlockChain(db, nil, gspec, nil, engine, vm.Config{}, nil, nil)
+	chain, err = NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
@@ -1976,8 +1989,12 @@ func TestIssue23496(t *testing.T) {
 	if head := chain.CurrentSnapBlock(); head.Number.Uint64() != uint64(4) {
 		t.Errorf("Head fast block mismatch: have %d, want %d", head.Number, uint64(4))
 	}
-	if head := chain.CurrentBlock(); head.Number.Uint64() != uint64(1) {
+	expHead := uint64(1)
-		t.Errorf("Head block mismatch: have %d, want %d", head.Number, uint64(1))
+	if scheme == rawdb.PathScheme {
+		expHead = uint64(2)
+	}
+	if head := chain.CurrentBlock(); head.Number.Uint64() != expHead {
+		t.Errorf("Head block mismatch: have %d, want %d", head.Number, expHead)
 	}
 
 	// Reinsert B2-B4

core/blockchain_sethead_test.go

@@ -22,6 +22,7 @@ package core
 import (
 	"fmt"
 	"math/big"
+	"path"
 	"strings"
 	"testing"
 	"time"
@@ -29,9 +30,13 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
+	"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
 )
 
 // rewindTest is a test case for chain rollback upon user request.
@@ -1949,16 +1954,24 @@ func testLongReorgedSnapSyncingDeepSetHead(t *testing.T, snapshots bool) {
 }
 
 func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
+		testSetHeadWithScheme(t, tt, snapshots, scheme)
+	}
+}
+
+func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) {
 	// It's hard to follow the test case, visualize the input
 	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
 	// fmt.Println(tt.dump(false))
 
 	// Create a temporary persistent database
 	datadir := t.TempDir()
+	ancient := path.Join(datadir, "ancient")
 
 	db, err := rawdb.Open(rawdb.OpenOptions{
 		Directory:         datadir,
-		AncientsDirectory: datadir,
+		AncientsDirectory: ancient,
+		Ephemeral:         true,
 	})
 	if err != nil {
 		t.Fatalf("Failed to create persistent database: %v", err)
@@ -1977,6 +1990,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
 			TrieDirtyLimit: 256,
 			TrieTimeLimit:  5 * time.Minute,
 			SnapshotLimit:  0, // Disable snapshot
+			StateScheme:    scheme,
 		}
 	)
 	if snapshots {
@@ -2007,7 +2021,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
 		t.Fatalf("Failed to import canonical chain start: %v", err)
 	}
 	if tt.commitBlock > 0 {
-		chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false)
+		chain.triedb.Commit(canonblocks[tt.commitBlock-1].Root(), false)
 		if snapshots {
 			if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
 				t.Fatalf("Failed to flatten snapshots: %v", err)
@@ -2017,13 +2031,17 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
 	if _, err := chain.InsertChain(canonblocks[tt.commitBlock:]); err != nil {
 		t.Fatalf("Failed to import canonical chain tail: %v", err)
 	}
-	// Manually dereference anything not committed to not have to work with 128+ tries
+	// Reopen the trie database without persisting in-memory dirty nodes.
-	for _, block := range sideblocks {
+	chain.triedb.Close()
-		chain.stateCache.TrieDB().Dereference(block.Root())
+	dbconfig := &trie.Config{}
-	}
+	if scheme == rawdb.PathScheme {
-	for _, block := range canonblocks {
+		dbconfig.PathDB = pathdb.Defaults
-		chain.stateCache.TrieDB().Dereference(block.Root())
+	} else {
+		dbconfig.HashDB = hashdb.Defaults
 	}
+	chain.triedb = trie.NewDatabase(chain.db, dbconfig)
+	chain.stateCache = state.NewDatabaseWithNodeDB(chain.db, chain.triedb)
+
 	// Force run a freeze cycle
 	type freezer interface {
 		Freeze(threshold uint64) error

core/blockchain_snapshot_test.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"math/big"
 	"os"
+	"path"
 	"strings"
 	"testing"
 	"time"
@@ -39,6 +40,7 @@ import (
 
 // snapshotTestBasic wraps the common testing fields in the snapshot tests.
 type snapshotTestBasic struct {
+	scheme        string // Disk scheme used for storing trie nodes
 	chainBlocks   int    // Number of blocks to generate for the canonical chain
 	snapshotBlock uint64 // Block number of the relevant snapshot disk layer
 	commitBlock   uint64 // Block number for which to commit the state to disk
@@ -51,6 +53,7 @@ type snapshotTestBasic struct {
 
 	// share fields, set in runtime
 	datadir string
+	ancient string
 	db      ethdb.Database
 	genDb   ethdb.Database
 	engine  consensus.Engine
@@ -60,10 +63,12 @@ type snapshotTestBasic struct {
 func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Block) {
 	// Create a temporary persistent database
 	datadir := t.TempDir()
+	ancient := path.Join(datadir, "ancient")
 
 	db, err := rawdb.Open(rawdb.OpenOptions{
 		Directory:         datadir,
-		AncientsDirectory: datadir,
+		AncientsDirectory: ancient,
+		Ephemeral:         true,
 	})
 	if err != nil {
 		t.Fatalf("Failed to create persistent database: %v", err)
@@ -75,13 +80,8 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 			Config:  params.AllEthashProtocolChanges,
 		}
 		engine = ethash.NewFullFaker()
-
-		// Snapshot is enabled, the first snapshot is created from the Genesis.
-		// The snapshot memory allowance is 256MB, it means no snapshot flush
-		// will happen during the block insertion.
-		cacheConfig = defaultCacheConfig
 	)
-	chain, err := NewBlockChain(db, cacheConfig, gspec, nil, engine, vm.Config{}, nil, nil)
+	chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(basic.scheme), gspec, nil, engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to create chain: %v", err)
 	}
@@ -102,7 +102,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 		startPoint = point
 
 		if basic.commitBlock > 0 && basic.commitBlock == point {
-			chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), false)
+			chain.TrieDB().Commit(blocks[point-1].Root(), false)
 		}
 		if basic.snapshotBlock > 0 && basic.snapshotBlock == point {
 			// Flushing the entire snap tree into the disk, the
@@ -121,6 +121,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 
 	// Set runtime fields
 	basic.datadir = datadir
+	basic.ancient = ancient
 	basic.db = db
 	basic.genDb = genDb
 	basic.engine = engine
@@ -210,6 +211,7 @@ func (basic *snapshotTestBasic) teardown() {
 	basic.db.Close()
 	basic.genDb.Close()
 	os.RemoveAll(basic.datadir)
+	os.RemoveAll(basic.ancient)
 }
 
 // snapshotTest is a test case type for normal snapshot recovery.
@@ -226,7 +228,7 @@ func (snaptest *snapshotTest) test(t *testing.T) {
 
 	// Restart the chain normally
 	chain.Stop()
-	newchain, err := NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
+	newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
@@ -235,7 +237,7 @@ func (snaptest *snapshotTest) test(t *testing.T) {
 	snaptest.verify(t, newchain, blocks)
 }
 
-// crashSnapshotTest is a test case type for innormal snapshot recovery.
+// crashSnapshotTest is a test case type for irregular snapshot recovery.
 // It can be used for testing that restart Geth after the crash.
 type crashSnapshotTest struct {
 	snapshotTestBasic
@@ -251,13 +253,14 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
 	db := chain.db
 	db.Close()
 	chain.stopWithoutSaving()
+	chain.triedb.Close()
 
 	// Start a new blockchain back up and see where the repair leads us
 	newdb, err := rawdb.Open(rawdb.OpenOptions{
 		Directory:         snaptest.datadir,
-		AncientsDirectory: snaptest.datadir,
+		AncientsDirectory: snaptest.ancient,
+		Ephemeral:         true,
 	})
-
 	if err != nil {
 		t.Fatalf("Failed to reopen persistent database: %v", err)
 	}
@@ -267,13 +270,13 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
 	// the crash, we do restart twice here: one after the crash and one
 	// after the normal stop. It's used to ensure the broken snapshot
 	// can be detected all the time.
-	newchain, err := NewBlockChain(newdb, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
+	newchain, err := NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
 	newchain.Stop()
 
-	newchain, err = NewBlockChain(newdb, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
+	newchain, err = NewBlockChain(newdb, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
@@ -300,7 +303,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
 
 	// Insert blocks without enabling snapshot if gapping is required.
 	chain.Stop()
-	gappedBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, func(i int, b *BlockGen) {})
+	gappedBlocks, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.gapped, func(i int, b *BlockGen) {})
 
 	// Insert a few more blocks without enabling snapshot
 	var cacheConfig = &CacheConfig{
@@ -308,6 +311,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
 		TrieDirtyLimit: 256,
 		TrieTimeLimit:  5 * time.Minute,
 		SnapshotLimit:  0,
+		StateScheme:    snaptest.scheme,
 	}
 	newchain, err := NewBlockChain(snaptest.db, cacheConfig, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
 	if err != nil {
@@ -317,7 +321,7 @@ func (snaptest *gappedSnapshotTest) test(t *testing.T) {
 	newchain.Stop()
 
 	// Restart the chain with enabling the snapshot
-	newchain, err = NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
+	newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
@@ -345,7 +349,7 @@ func (snaptest *setHeadSnapshotTest) test(t *testing.T) {
 	chain.SetHead(snaptest.setHead)
 	chain.Stop()
 
-	newchain, err := NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
+	newchain, err := NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
@@ -379,22 +383,24 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) {
 		TrieDirtyLimit: 256,
 		TrieTimeLimit:  5 * time.Minute,
 		SnapshotLimit:  0,
+		StateScheme:    snaptest.scheme,
 	}
 	newchain, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
-	newBlocks, _ := GenerateChain(params.TestChainConfig, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, func(i int, b *BlockGen) {})
+	newBlocks, _ := GenerateChain(snaptest.gspec.Config, blocks[len(blocks)-1], snaptest.engine, snaptest.genDb, snaptest.newBlocks, func(i int, b *BlockGen) {})
 	newchain.InsertChain(newBlocks)
 	newchain.Stop()
 
-	// Restart the chain, the wiper should starts working
+	// Restart the chain, the wiper should start working
 	config = &CacheConfig{
 		TrieCleanLimit: 256,
 		TrieDirtyLimit: 256,
 		TrieTimeLimit:  5 * time.Minute,
 		SnapshotLimit:  256,
 		SnapshotWait:   false, // Don't wait rebuild
+		StateScheme:    snaptest.scheme,
 	}
 	tmp, err := NewBlockChain(snaptest.db, config, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
 	if err != nil {
@@ -402,14 +408,15 @@ func (snaptest *wipeCrashSnapshotTest) test(t *testing.T) {
 	}
 
 	// Simulate the blockchain crash.
+	tmp.triedb.Close()
 	tmp.stopWithoutSaving()
 
-	newchain, err = NewBlockChain(snaptest.db, nil, snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
+	newchain, err = NewBlockChain(snaptest.db, DefaultCacheConfigWithScheme(snaptest.scheme), snaptest.gspec, nil, snaptest.engine, vm.Config{}, nil, nil)
 	if err != nil {
 		t.Fatalf("Failed to recreate chain: %v", err)
 	}
-	defer newchain.Stop()
 	snaptest.verify(t, newchain, blocks)
+	newchain.Stop()
 }
 
 // Tests a Geth restart with valid snapshot. Before the shutdown, all snapshot
@@ -433,20 +440,23 @@ func TestRestartWithNewSnapshot(t *testing.T) {
 	// Expected head fast block: C8
 	// Expected head block     : C8
 	// Expected snapshot disk  : G
-	test := &snapshotTest{
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
-		snapshotTestBasic{
+		test := &snapshotTest{
-			chainBlocks:        8,
+			snapshotTestBasic{
-			snapshotBlock:      0,
+				scheme:             scheme,
-			commitBlock:        0,
+				chainBlocks:        8,
-			expCanonicalBlocks: 8,
+				snapshotBlock:      0,
-			expHeadHeader:      8,
+				commitBlock:        0,
-			expHeadFastBlock:   8,
+				expCanonicalBlocks: 8,
-			expHeadBlock:       8,
+				expHeadHeader:      8,
-			expSnapshotBottom:  0, // Initial disk layer built from genesis
+				expHeadFastBlock:   8,
-		},
+				expHeadBlock:       8,
+				expSnapshotBottom:  0, // Initial disk layer built from genesis
+			},
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }
 
 // Tests a Geth was crashed and restarts with a broken snapshot. In this case the
@@ -472,20 +482,23 @@ func TestNoCommitCrashWithNewSnapshot(t *testing.T) {
 	// Expected head fast block: C8
 	// Expected head block     : G
 	// Expected snapshot disk  : C4
-	test := &crashSnapshotTest{
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
-		snapshotTestBasic{
+		test := &crashSnapshotTest{
-			chainBlocks:        8,
+			snapshotTestBasic{
-			snapshotBlock:      4,
+				scheme:             scheme,
-			commitBlock:        0,
+				chainBlocks:        8,
-			expCanonicalBlocks: 8,
+				snapshotBlock:      4,
-			expHeadHeader:      8,
+				commitBlock:        0,
-			expHeadFastBlock:   8,
+				expCanonicalBlocks: 8,
-			expHeadBlock:       0,
+				expHeadHeader:      8,
-			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
+				expHeadFastBlock:   8,
-		},
+				expHeadBlock:       0,
+				expSnapshotBottom:  4, // Last committed disk layer, wait recovery
+			},
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }
 
 // Tests a Geth was crashed and restarts with a broken snapshot. In this case the
@@ -511,20 +524,23 @@ func TestLowCommitCrashWithNewSnapshot(t *testing.T) {
 	// Expected head fast block: C8
 	// Expected head block     : C2
 	// Expected snapshot disk  : C4
-	test := &crashSnapshotTest{
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
-		snapshotTestBasic{
+		test := &crashSnapshotTest{
-			chainBlocks:        8,
+			snapshotTestBasic{
-			snapshotBlock:      4,
+				scheme:             scheme,
-			commitBlock:        2,
+				chainBlocks:        8,
-			expCanonicalBlocks: 8,
+				snapshotBlock:      4,
-			expHeadHeader:      8,
+				commitBlock:        2,
-			expHeadFastBlock:   8,
+				expCanonicalBlocks: 8,
-			expHeadBlock:       2,
+				expHeadHeader:      8,
-			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
+				expHeadFastBlock:   8,
-		},
+				expHeadBlock:       2,
+				expSnapshotBottom:  4, // Last committed disk layer, wait recovery
+			},
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }
 
 // Tests a Geth was crashed and restarts with a broken snapshot. In this case
@@ -550,20 +566,27 @@ func TestHighCommitCrashWithNewSnapshot(t *testing.T) {
 	// Expected head fast block: C8
 	// Expected head block     : G
 	// Expected snapshot disk  : C4
-	test := &crashSnapshotTest{
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
-		snapshotTestBasic{
+		expHead := uint64(0)
-			chainBlocks:        8,
+		if scheme == rawdb.PathScheme {
-			snapshotBlock:      4,
+			expHead = uint64(4)
-			commitBlock:        6,
+		}
-			expCanonicalBlocks: 8,
+		test := &crashSnapshotTest{
-			expHeadHeader:      8,
+			snapshotTestBasic{
-			expHeadFastBlock:   8,
+				scheme:             scheme,
-			expHeadBlock:       0,
+				chainBlocks:        8,
-			expSnapshotBottom:  4, // Last committed disk layer, wait recovery
+				snapshotBlock:      4,
-		},
+				commitBlock:        6,
+				expCanonicalBlocks: 8,
+				expHeadHeader:      8,
+				expHeadFastBlock:   8,
+				expHeadBlock:       expHead,
+				expSnapshotBottom:  4, // Last committed disk layer, wait recovery
+			},
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }
 
 // Tests a Geth was running with snapshot enabled. Then restarts without
@@ -587,21 +610,24 @@ func TestGappedNewSnapshot(t *testing.T) {
 	// Expected head fast block: C10
 	// Expected head block     : C10
 	// Expected snapshot disk  : C10
-	test := &gappedSnapshotTest{
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
-		snapshotTestBasic: snapshotTestBasic{
+		test := &gappedSnapshotTest{
-			chainBlocks:        8,
+			snapshotTestBasic: snapshotTestBasic{
-			snapshotBlock:      0,
+				scheme:             scheme,
-			commitBlock:        0,
+				chainBlocks:        8,
-			expCanonicalBlocks: 10,
+				snapshotBlock:      0,
-			expHeadHeader:      10,
+				commitBlock:        0,
-			expHeadFastBlock:   10,
+				expCanonicalBlocks: 10,
-			expHeadBlock:       10,
+				expHeadHeader:      10,
-			expSnapshotBottom:  10, // Rebuilt snapshot from the latest HEAD
+				expHeadFastBlock:   10,
-		},
+				expHeadBlock:       10,
-		gapped: 2,
+				expSnapshotBottom:  10, // Rebuilt snapshot from the latest HEAD
+			},
+			gapped: 2,
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }
 
 // Tests the Geth was running with snapshot enabled and resetHead is applied.
@@ -625,21 +651,24 @@ func TestSetHeadWithNewSnapshot(t *testing.T) {
 	// Expected head fast block: C4
 	// Expected head block     : C4
 	// Expected snapshot disk  : G
-	test := &setHeadSnapshotTest{
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
-		snapshotTestBasic: snapshotTestBasic{
+		test := &setHeadSnapshotTest{
-			chainBlocks:        8,
+			snapshotTestBasic: snapshotTestBasic{
-			snapshotBlock:      0,
+				scheme:             scheme,
-			commitBlock:        0,
+				chainBlocks:        8,
-			expCanonicalBlocks: 4,
+				snapshotBlock:      0,
-			expHeadHeader:      4,
+				commitBlock:        0,
-			expHeadFastBlock:   4,
+				expCanonicalBlocks: 4,
-			expHeadBlock:       4,
+				expHeadHeader:      4,
-			expSnapshotBottom:  0, // The initial disk layer is built from the genesis
+				expHeadFastBlock:   4,
-		},
+				expHeadBlock:       4,
-		setHead: 4,
+				expSnapshotBottom:  0, // The initial disk layer is built from the genesis
+			},
+			setHead: 4,
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }
 
 // Tests the Geth was running with a complete snapshot and then imports a few
@@ -663,19 +692,22 @@ func TestRecoverSnapshotFromWipingCrash(t *testing.T) {
 	// Expected head fast block: C10
 	// Expected head block     : C8
 	// Expected snapshot disk  : C10
-	test := &wipeCrashSnapshotTest{
+	for _, scheme := range []string{rawdb.HashScheme, rawdb.PathScheme} {
-		snapshotTestBasic: snapshotTestBasic{
+		test := &wipeCrashSnapshotTest{
-			chainBlocks:        8,
+			snapshotTestBasic: snapshotTestBasic{
-			snapshotBlock:      4,
+				scheme:             scheme,
-			commitBlock:        0,
+				chainBlocks:        8,
-			expCanonicalBlocks: 10,
+				snapshotBlock:      4,
-			expHeadHeader:      10,
+				commitBlock:        0,
-			expHeadFastBlock:   10,
+				expCanonicalBlocks: 10,
-			expHeadBlock:       10,
+				expHeadHeader:      10,
-			expSnapshotBottom:  10,
+				expHeadFastBlock:   10,
-		},
+				expHeadBlock:       10,
-		newBlocks: 2,
+				expSnapshotBottom:  10,
+			},
+			newBlocks: 2,
+		}
+		test.test(t)
+		test.teardown()
 	}
-	test.test(t)
-	test.teardown()
 }

core/bloombits/matcher.go

@@ -630,13 +630,16 @@ func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan
 			request <- &Retrieval{Bit: bit, Sections: sections, Context: s.ctx}
 
 			result := <-request
+
+			// Deliver a result before s.Close() to avoid a deadlock
+			s.deliverSections(result.Bit, result.Sections, result.Bitsets)
+
 			if result.Error != nil {
 				s.errLock.Lock()
 				s.err = result.Error
 				s.errLock.Unlock()
 				s.Close()
 			}
-			s.deliverSections(result.Bit, result.Sections, result.Bitsets)
 		}
 	}
 }

core/chain_makers.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/misc"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
+	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -87,6 +88,11 @@ func (b *BlockGen) SetPoS() {
 	b.header.Difficulty = new(big.Int)
 }
 
+// SetBlobGas sets the data gas used by the blob in the generated block.
+func (b *BlockGen) SetBlobGas(blobGasUsed uint64) {
+	b.header.BlobGasUsed = &blobGasUsed
+}
+
 // addTx adds a transaction to the generated block. If no coinbase has
 // been set, the block's coinbase is set to the zero address.
 //
@@ -283,7 +289,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 	}
 	blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
 	chainreader := &fakeChainReader{config: config}
-	genblock := func(i int, parent *types.Block, statedb *state.StateDB) (*types.Block, types.Receipts) {
+	genblock := func(i int, parent *types.Block, triedb *trie.Database, statedb *state.StateDB) (*types.Block, types.Receipts) {
 		b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
 		b.header = makeHeader(chainreader, parent, statedb, b.engine)
 
@@ -326,19 +332,23 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 			if err != nil {
 				panic(fmt.Sprintf("state write error: %v", err))
 			}
-			if err := statedb.Database().TrieDB().Commit(root, false); err != nil {
+			if err = triedb.Commit(root, false); err != nil {
 				panic(fmt.Sprintf("trie write error: %v", err))
 			}
 			return block, b.receipts
 		}
 		return nil, nil
 	}
+	// Forcibly use hash-based state scheme for retaining all nodes in disk.
+	triedb := trie.NewDatabase(db, trie.HashDefaults)
+	defer triedb.Close()
+
 	for i := 0; i < n; i++ {
-		statedb, err := state.New(parent.Root(), state.NewDatabase(db), nil)
+		statedb, err := state.New(parent.Root(), state.NewDatabaseWithNodeDB(db, triedb), nil)
 		if err != nil {
 			panic(err)
 		}
-		block, receipt := genblock(i, parent, statedb)
+		block, receipt := genblock(i, parent, triedb, statedb)
 		blocks[i] = block
 		receipts[i] = receipt
 		parent = block
@@ -351,7 +361,9 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 // then generate chain on top.
 func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int, gen func(int, *BlockGen)) (ethdb.Database, []*types.Block, []types.Receipts) {
 	db := rawdb.NewMemoryDatabase()
-	_, err := genesis.Commit(db, trie.NewDatabase(db))
+	triedb := trie.NewDatabase(db, trie.HashDefaults)
+	defer triedb.Close()
+	_, err := genesis.Commit(db, triedb)
 	if err != nil {
 		panic(err)
 	}
@@ -387,6 +399,20 @@ func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.S
 			header.GasLimit = CalcGasLimit(parentGasLimit, parentGasLimit)
 		}
 	}
+	if chain.Config().IsCancun(header.Number, header.Time) {
+		var (
+			parentExcessBlobGas uint64
+			parentBlobGasUsed   uint64
+		)
+		if parent.ExcessBlobGas() != nil {
+			parentExcessBlobGas = *parent.ExcessBlobGas()
+			parentBlobGasUsed = *parent.BlobGasUsed()
+		}
+		excessBlobGas := eip4844.CalcExcessBlobGas(parentExcessBlobGas, parentBlobGasUsed)
+		header.ExcessBlobGas = &excessBlobGas
+		header.BlobGasUsed = new(uint64)
+		header.ParentBeaconRoot = new(common.Hash)
+	}
 	return header
 }
 

core/chain_makers_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 func TestGenerateWithdrawalChain(t *testing.T) {
@@ -74,8 +75,7 @@ func TestGenerateWithdrawalChain(t *testing.T) {
 		Storage: storage,
 		Code:    common.Hex2Bytes("600154600354"),
 	}
-
+	genesis := gspec.MustCommit(gendb, trie.NewDatabase(gendb, trie.HashDefaults))
-	genesis := gspec.MustCommit(gendb)
 
 	chain, _ := GenerateChain(gspec.Config, genesis, beacon.NewFaker(), gendb, 4, func(i int, gen *BlockGen) {
 		tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(address), address, big.NewInt(1000), params.TxGas, new(big.Int).Add(gen.BaseFee(), common.Big1), nil), signer, key)
@@ -146,6 +146,7 @@ func ExampleGenerateChain() {
 		addr2   = crypto.PubkeyToAddress(key2.PublicKey)
 		addr3   = crypto.PubkeyToAddress(key3.PublicKey)
 		db      = rawdb.NewMemoryDatabase()
+		genDb   = rawdb.NewMemoryDatabase()
 	)
 
 	// Ensure that key1 has some funds in the genesis block.
@@ -153,13 +154,13 @@ func ExampleGenerateChain() {
 		Config: &params.ChainConfig{HomesteadBlock: new(big.Int)},
 		Alloc:  GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}},
 	}
-	genesis := gspec.MustCommit(db)
+	genesis := gspec.MustCommit(genDb, trie.NewDatabase(genDb, trie.HashDefaults))
 
 	// This call generates a chain of 5 blocks. The function runs for
 	// each block and adds different features to gen based on the
 	// block index.
 	signer := types.HomesteadSigner{}
-	chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 5, func(i int, gen *BlockGen) {
+	chain, _ := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), genDb, 5, func(i int, gen *BlockGen) {
 		switch i {
 		case 0:
 			// In block 1, addr1 sends addr2 some ether.
@@ -188,7 +189,7 @@ func ExampleGenerateChain() {
 	})
 
 	// Import the chain. This runs all block validation rules.
-	blockchain, _ := NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
+	blockchain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.HashScheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
 	defer blockchain.Stop()
 
 	if i, err := blockchain.InsertChain(chain); err != nil {

core/dao_test.go

@@ -83,7 +83,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		if _, err := bc.InsertChain(blocks); err != nil {
 			t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
 		}
-		if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
+		if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
 			t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
 		}
 		bc.Stop()
@@ -106,7 +106,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		if _, err := bc.InsertChain(blocks); err != nil {
 			t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
 		}
-		if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
+		if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
 			t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
 		}
 		bc.Stop()
@@ -131,7 +131,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	if _, err := bc.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
 	}
-	if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
+	if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
 		t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
 	}
 	blocks, _ = GenerateChain(&proConf, conBc.GetBlockByHash(conBc.CurrentBlock().Hash()), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {})
@@ -149,7 +149,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	if _, err := bc.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
 	}
-	if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
+	if err := bc.triedb.Commit(bc.CurrentHeader().Root, false); err != nil {
 		t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
 	}
 	blocks, _ = GenerateChain(&conConf, proBc.GetBlockByHash(proBc.CurrentBlock().Hash()), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {})

core/forkid/forkid.go

@@ -26,7 +26,6 @@ import (
 	"reflect"
 	"strings"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
@@ -73,12 +72,12 @@ type ID struct {
 type Filter func(id ID) error
 
 // NewID calculates the Ethereum fork ID from the chain config, genesis hash, head and time.
-func NewID(config *params.ChainConfig, genesis common.Hash, head, time uint64) ID {
+func NewID(config *params.ChainConfig, genesis *types.Block, head, time uint64) ID {
 	// Calculate the starting checksum from the genesis hash
-	hash := crc32.ChecksumIEEE(genesis[:])
+	hash := crc32.ChecksumIEEE(genesis.Hash().Bytes())
 
 	// Calculate the current fork checksum and the next fork block
-	forksByBlock, forksByTime := gatherForks(config)
+	forksByBlock, forksByTime := gatherForks(config, genesis.Time())
 	for _, fork := range forksByBlock {
 		if fork <= head {
 			// Fork already passed, checksum the previous hash and the fork number
@@ -104,7 +103,7 @@ func NewIDWithChain(chain Blockchain) ID {
 
 	return NewID(
 		chain.Config(),
-		chain.Genesis().Hash(),
+		chain.Genesis(),
 		head.Number.Uint64(),
 		head.Time,
 	)
@@ -115,7 +114,7 @@ func NewIDWithChain(chain Blockchain) ID {
 func NewFilter(chain Blockchain) Filter {
 	return newFilter(
 		chain.Config(),
-		chain.Genesis().Hash(),
+		chain.Genesis(),
 		func() (uint64, uint64) {
 			head := chain.CurrentHeader()
 			return head.Number.Uint64(), head.Time
@@ -124,7 +123,7 @@ func NewFilter(chain Blockchain) Filter {
 }
 
 // NewStaticFilter creates a filter at block zero.
-func NewStaticFilter(config *params.ChainConfig, genesis common.Hash) Filter {
+func NewStaticFilter(config *params.ChainConfig, genesis *types.Block) Filter {
 	head := func() (uint64, uint64) { return 0, 0 }
 	return newFilter(config, genesis, head)
 }
@@ -132,14 +131,14 @@ func NewStaticFilter(config *params.ChainConfig, genesis common.Hash) Filter {
 // newFilter is the internal version of NewFilter, taking closures as its arguments
 // instead of a chain. The reason is to allow testing it without having to simulate
 // an entire blockchain.
-func newFilter(config *params.ChainConfig, genesis common.Hash, headfn func() (uint64, uint64)) Filter {
+func newFilter(config *params.ChainConfig, genesis *types.Block, headfn func() (uint64, uint64)) Filter {
 	// Calculate the all the valid fork hash and fork next combos
 	var (
-		forksByBlock, forksByTime = gatherForks(config)
+		forksByBlock, forksByTime = gatherForks(config, genesis.Time())
 		forks                     = append(append([]uint64{}, forksByBlock...), forksByTime...)
 		sums                      = make([][4]byte, len(forks)+1) // 0th is the genesis
 	)
-	hash := crc32.ChecksumIEEE(genesis[:])
+	hash := crc32.ChecksumIEEE(genesis.Hash().Bytes())
 	sums[0] = checksumToBytes(hash)
 	for i, fork := range forks {
 		hash = checksumUpdate(hash, fork)
@@ -240,7 +239,7 @@ func checksumToBytes(hash uint32) [4]byte {
 
 // gatherForks gathers all the known forks and creates two sorted lists out of
 // them, one for the block number based forks and the second for the timestamps.
-func gatherForks(config *params.ChainConfig) ([]uint64, []uint64) {
+func gatherForks(config *params.ChainConfig, genesis uint64) ([]uint64, []uint64) {
 	// Gather all the fork block numbers via reflection
 	kind := reflect.TypeOf(params.ChainConfig{})
 	conf := reflect.ValueOf(config).Elem()
@@ -290,7 +289,8 @@ func gatherForks(config *params.ChainConfig) ([]uint64, []uint64) {
 	if len(forksByBlock) > 0 && forksByBlock[0] == 0 {
 		forksByBlock = forksByBlock[1:]
 	}
-	if len(forksByTime) > 0 && forksByTime[0] == 0 {
+	// Skip any forks before genesis.
+	for len(forksByTime) > 0 && forksByTime[0] <= genesis {
 		forksByTime = forksByTime[1:]
 	}
 	return forksByBlock, forksByTime

core/forkid/forkid_test.go

@@ -18,10 +18,14 @@ package forkid
 
 import (
 	"bytes"
+	"hash/crc32"
 	"math"
+	"math/big"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 )
@@ -36,13 +40,13 @@ func TestCreation(t *testing.T) {
 	}
 	tests := []struct {
 		config  *params.ChainConfig
-		genesis common.Hash
+		genesis *types.Block
 		cases   []testcase
 	}{
 		// Mainnet test cases
 		{
 			params.MainnetChainConfig,
-			params.MainnetGenesisHash,
+			core.DefaultGenesisBlock().ToBlock(),
 			[]testcase{
 				{0, 0, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}},                    // Unsynced
 				{1149999, 0, ID{Hash: checksumToBytes(0xfc64ec04), Next: 1150000}},              // Last Frontier block
@@ -77,7 +81,7 @@ func TestCreation(t *testing.T) {
 		// Goerli test cases
 		{
 			params.GoerliChainConfig,
-			params.GoerliGenesisHash,
+			core.DefaultGoerliGenesisBlock().ToBlock(),
 			[]testcase{
 				{0, 0, ID{Hash: checksumToBytes(0xa3f5ab08), Next: 1561651}},                   // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople and first Petersburg block
 				{1561650, 0, ID{Hash: checksumToBytes(0xa3f5ab08), Next: 1561651}},             // Last Petersburg block
@@ -94,7 +98,7 @@ func TestCreation(t *testing.T) {
 		// Sepolia test cases
 		{
 			params.SepoliaChainConfig,
-			params.SepoliaGenesisHash,
+			core.DefaultSepoliaGenesisBlock().ToBlock(),
 			[]testcase{
 				{0, 0, ID{Hash: checksumToBytes(0xfe3366e7), Next: 1735371}},                   // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople, Petersburg, Istanbul, Berlin and first London block
 				{1735370, 0, ID{Hash: checksumToBytes(0xfe3366e7), Next: 1735371}},             // Last London block
@@ -103,6 +107,16 @@ func TestCreation(t *testing.T) {
 				{1735372, 1677557088, ID{Hash: checksumToBytes(0xf7f9bc08), Next: 0}},          // First Shanghai block
 			},
 		},
+		// Holesky test cases
+		{
+			params.HoleskyChainConfig,
+			core.DefaultHoleskyGenesisBlock().ToBlock(),
+			[]testcase{
+				{0, 0, ID{Hash: checksumToBytes(0xc61a6098), Next: 1696000704}},   // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople, Petersburg, Istanbul, Berlin, London, Paris block
+				{123, 0, ID{Hash: checksumToBytes(0xc61a6098), Next: 1696000704}}, // First MergeNetsplit block
+				{123, 1696000704, ID{Hash: checksumToBytes(0xfd4f016b), Next: 0}}, // Last MergeNetsplit block
+			},
+		},
 	}
 	for i, tt := range tests {
 		for j, ttt := range tt.cases {
@@ -353,7 +367,7 @@ func TestValidation(t *testing.T) {
 		//{params.MainnetChainConfig, 20999999, 1677999999, ID{Hash: checksumToBytes(0x71147644), Next: 1678000000}, ErrLocalIncompatibleOrStale},
 	}
 	for i, tt := range tests {
-		filter := newFilter(tt.config, params.MainnetGenesisHash, func() (uint64, uint64) { return tt.head, tt.time })
+		filter := newFilter(tt.config, core.DefaultGenesisBlock().ToBlock(), func() (uint64, uint64) { return tt.head, tt.time })
 		if err := filter(tt.id); err != tt.err {
 			t.Errorf("test %d: validation error mismatch: have %v, want %v", i, err, tt.err)
 		}
@@ -382,3 +396,55 @@ func TestEncoding(t *testing.T) {
 		}
 	}
 }
+
+// Tests that time-based forks which are active at genesis are not included in
+// forkid hash.
+func TestTimeBasedForkInGenesis(t *testing.T) {
+	var (
+		time       = uint64(1690475657)
+		genesis    = types.NewBlockWithHeader(&types.Header{Time: time})
+		forkidHash = checksumToBytes(crc32.ChecksumIEEE(genesis.Hash().Bytes()))
+		config     = func(shanghai, cancun uint64) *params.ChainConfig {
+			return &params.ChainConfig{
+				ChainID:                       big.NewInt(1337),
+				HomesteadBlock:                big.NewInt(0),
+				DAOForkBlock:                  nil,
+				DAOForkSupport:                true,
+				EIP150Block:                   big.NewInt(0),
+				EIP155Block:                   big.NewInt(0),
+				EIP158Block:                   big.NewInt(0),
+				ByzantiumBlock:                big.NewInt(0),
+				ConstantinopleBlock:           big.NewInt(0),
+				PetersburgBlock:               big.NewInt(0),
+				IstanbulBlock:                 big.NewInt(0),
+				MuirGlacierBlock:              big.NewInt(0),
+				BerlinBlock:                   big.NewInt(0),
+				LondonBlock:                   big.NewInt(0),
+				TerminalTotalDifficulty:       big.NewInt(0),
+				TerminalTotalDifficultyPassed: true,
+				MergeNetsplitBlock:            big.NewInt(0),
+				ShanghaiTime:                  &shanghai,
+				CancunTime:                    &cancun,
+				Ethash:                        new(params.EthashConfig),
+			}
+		}
+	)
+	tests := []struct {
+		config *params.ChainConfig
+		want   ID
+	}{
+		// Shanghai active before genesis, skip
+		{config(time-1, time+1), ID{Hash: forkidHash, Next: time + 1}},
+
+		// Shanghai active at genesis, skip
+		{config(time, time+1), ID{Hash: forkidHash, Next: time + 1}},
+
+		// Shanghai not active, skip
+		{config(time+1, time+2), ID{Hash: forkidHash, Next: time + 1}},
+	}
+	for _, tt := range tests {
+		if have := NewID(tt.config, genesis, 0, time); have != tt.want {
+			t.Fatalf("incorrect forkid hash: have %x, want %x", have, tt.want)
+		}
+	}
+}

core/genesis.go

@@ -130,7 +130,9 @@ func (ga *GenesisAlloc) deriveHash() (common.Hash, error) {
 		return common.Hash{}, err
 	}
 	for addr, account := range *ga {
-		statedb.AddBalance(addr, account.Balance)
+		if account.Balance != nil {
+			statedb.AddBalance(addr, account.Balance)
+		}
 		statedb.SetCode(addr, account.Code)
 		statedb.SetNonce(addr, account.Nonce)
 		for key, value := range account.Storage {
@@ -149,7 +151,9 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhas
 		return err
 	}
 	for addr, account := range *ga {
-		statedb.AddBalance(addr, account.Balance)
+		if account.Balance != nil {
+			statedb.AddBalance(addr, account.Balance)
+		}
 		statedb.SetCode(addr, account.Code)
 		statedb.SetNonce(addr, account.Nonce)
 		for key, value := range account.Storage {
@@ -323,10 +327,12 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen
 		applyOverrides(genesis.Config)
 		return genesis.Config, block.Hash(), nil
 	}
-	// We have the genesis block in database(perhaps in ancient database)
+	// The genesis block is present(perhaps in ancient database) while the
-	// but the corresponding state is missing.
+	// state database is not initialized yet. It can happen that the node
+	// is initialized with an external ancient store. Commit genesis state
+	// in this case.
 	header := rawdb.ReadHeader(db, stored, 0)
-	if header.Root != types.EmptyRootHash && !rawdb.HasLegacyTrieNode(db, header.Root) {
+	if header.Root != types.EmptyRootHash && !triedb.Initialized(header.Root) {
 		if genesis == nil {
 			genesis = DefaultGenesisBlock()
 		}
@@ -477,6 +483,11 @@ func (g *Genesis) ToBlock() *types.Block {
 			withdrawals = make([]*types.Withdrawal, 0)
 		}
 		if conf.IsCancun(num, g.Timestamp) {
+			// EIP-4788: The parentBeaconBlockRoot of the genesis block is always
+			// the zero hash. This is because the genesis block does not have a parent
+			// by definition.
+			head.ParentBeaconRoot = new(common.Hash)
+			// EIP-4844 fields
 			head.ExcessBlobGas = g.ExcessBlobGas
 			head.BlobGasUsed = g.BlobGasUsed
 			if head.ExcessBlobGas == nil {
@@ -526,10 +537,8 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *trie.Database) (*types.Block
 
 // MustCommit writes the genesis block and state to db, panicking on error.
 // The block is committed as the canonical head block.
-// Note the state changes will be committed in hash-based scheme, use Commit
+func (g *Genesis) MustCommit(db ethdb.Database, triedb *trie.Database) *types.Block {
-// if path-scheme is preferred.
+	block, err := g.Commit(db, triedb)
-func (g *Genesis) MustCommit(db ethdb.Database) *types.Block {
-	block, err := g.Commit(db, trie.NewDatabase(db))
 	if err != nil {
 		panic(err)
 	}
@@ -573,6 +582,18 @@ func DefaultSepoliaGenesisBlock() *Genesis {
 	}
 }
 
+// DefaultHoleskyGenesisBlock returns the Holesky network genesis block.
+func DefaultHoleskyGenesisBlock() *Genesis {
+	return &Genesis{
+		Config:     params.HoleskyChainConfig,
+		Nonce:      0x1234,
+		GasLimit:   0x17d7840,
+		Difficulty: big.NewInt(0x01),
+		Timestamp:  1695902100,
+		Alloc:      decodePrealloc(holeskyAllocData),
+	}
+}
+
 // DeveloperGenesisBlock returns the 'geth --dev' genesis block.
 func DeveloperGenesisBlock(gasLimit uint64, faucet common.Address) *Genesis {
 	// Override the default period to the user requested one
@@ -600,13 +621,34 @@ func DeveloperGenesisBlock(gasLimit uint64, faucet common.Address) *Genesis {
 }
 
 func decodePrealloc(data string) GenesisAlloc {
-	var p []struct{ Addr, Balance *big.Int }
+	var p []struct {
+		Addr    *big.Int
+		Balance *big.Int
+		Misc    *struct {
+			Nonce uint64
+			Code  []byte
+			Slots []struct {
+				Key common.Hash
+				Val common.Hash
+			}
+		} `rlp:"optional"`
+	}
 	if err := rlp.NewStream(strings.NewReader(data), 0).Decode(&p); err != nil {
 		panic(err)
 	}
 	ga := make(GenesisAlloc, len(p))
 	for _, account := range p {
-		ga[common.BigToAddress(account.Addr)] = GenesisAccount{Balance: account.Balance}
+		acc := GenesisAccount{Balance: account.Balance}
+		if account.Misc != nil {
+			acc.Nonce = account.Misc.Nonce
+			acc.Code = account.Misc.Code
+
+			acc.Storage = make(map[common.Hash]common.Hash)
+			for _, slot := range account.Misc.Slots {
+				acc.Storage[slot.Key] = slot.Val
+			}
+		}
+		ga[common.BigToAddress(account.Addr)] = acc
 	}
 	return ga
 }

core/genesis_test.go

@@ -30,18 +30,24 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
 )
 
 func TestInvalidCliqueConfig(t *testing.T) {
 	block := DefaultGoerliGenesisBlock()
 	block.ExtraData = []byte{}
 	db := rawdb.NewMemoryDatabase()
-	if _, err := block.Commit(db, trie.NewDatabase(db)); err == nil {
+	if _, err := block.Commit(db, trie.NewDatabase(db, nil)); err == nil {
 		t.Fatal("Expected error on invalid clique config")
 	}
 }
 
 func TestSetupGenesis(t *testing.T) {
+	testSetupGenesis(t, rawdb.HashScheme)
+	testSetupGenesis(t, rawdb.PathScheme)
+}
+
+func testSetupGenesis(t *testing.T, scheme string) {
 	var (
 		customghash = common.HexToHash("0x89c99d90b79719238d2645c7642f2c9295246e80775b38cfd162b696817fbd50")
 		customg     = Genesis{
@@ -53,6 +59,7 @@ func TestSetupGenesis(t *testing.T) {
 		oldcustomg = customg
 	)
 	oldcustomg.Config = &params.ChainConfig{HomesteadBlock: big.NewInt(2)}
+
 	tests := []struct {
 		name       string
 		fn         func(ethdb.Database) (*params.ChainConfig, common.Hash, error)
@@ -63,7 +70,7 @@ func TestSetupGenesis(t *testing.T) {
 		{
 			name: "genesis without ChainConfig",
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
-				return SetupGenesisBlock(db, trie.NewDatabase(db), new(Genesis))
+				return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), new(Genesis))
 			},
 			wantErr:    errGenesisNoConfig,
 			wantConfig: params.AllEthashProtocolChanges,
@@ -71,7 +78,7 @@ func TestSetupGenesis(t *testing.T) {
 		{
 			name: "no block in DB, genesis == nil",
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
-				return SetupGenesisBlock(db, trie.NewDatabase(db), nil)
+				return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil)
 			},
 			wantHash:   params.MainnetGenesisHash,
 			wantConfig: params.MainnetChainConfig,
@@ -79,8 +86,8 @@ func TestSetupGenesis(t *testing.T) {
 		{
 			name: "mainnet block in DB, genesis == nil",
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
-				DefaultGenesisBlock().MustCommit(db)
+				DefaultGenesisBlock().MustCommit(db, trie.NewDatabase(db, newDbConfig(scheme)))
-				return SetupGenesisBlock(db, trie.NewDatabase(db), nil)
+				return SetupGenesisBlock(db, trie.NewDatabase(db, newDbConfig(scheme)), nil)
 			},
 			wantHash:   params.MainnetGenesisHash,
 			wantConfig: params.MainnetChainConfig,
@@ -88,8 +95,9 @@ func TestSetupGenesis(t *testing.T) {
 		{
 			name: "custom block in DB, genesis == nil",
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
-				customg.MustCommit(db)
+				tdb := trie.NewDatabase(db, newDbConfig(scheme))
-				return SetupGenesisBlock(db, trie.NewDatabase(db), nil)
+				customg.Commit(db, tdb)
+				return SetupGenesisBlock(db, tdb, nil)
 			},
 			wantHash:   customghash,
 			wantConfig: customg.Config,
@@ -97,8 +105,9 @@ func TestSetupGenesis(t *testing.T) {
 		{
 			name: "custom block in DB, genesis == goerli",
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
-				customg.MustCommit(db)
+				tdb := trie.NewDatabase(db, newDbConfig(scheme))
-				return SetupGenesisBlock(db, trie.NewDatabase(db), DefaultGoerliGenesisBlock())
+				customg.Commit(db, tdb)
+				return SetupGenesisBlock(db, tdb, DefaultGoerliGenesisBlock())
 			},
 			wantErr:    &GenesisMismatchError{Stored: customghash, New: params.GoerliGenesisHash},
 			wantHash:   params.GoerliGenesisHash,
@@ -107,8 +116,9 @@ func TestSetupGenesis(t *testing.T) {
 		{
 			name: "compatible config in DB",
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
-				oldcustomg.MustCommit(db)
+				tdb := trie.NewDatabase(db, newDbConfig(scheme))
-				return SetupGenesisBlock(db, trie.NewDatabase(db), &customg)
+				oldcustomg.Commit(db, tdb)
+				return SetupGenesisBlock(db, tdb, &customg)
 			},
 			wantHash:   customghash,
 			wantConfig: customg.Config,
@@ -118,16 +128,17 @@ func TestSetupGenesis(t *testing.T) {
 			fn: func(db ethdb.Database) (*params.ChainConfig, common.Hash, error) {
 				// Commit the 'old' genesis block with Homestead transition at #2.
 				// Advance to block #4, past the homestead transition block of customg.
-				genesis := oldcustomg.MustCommit(db)
+				tdb := trie.NewDatabase(db, newDbConfig(scheme))
+				oldcustomg.Commit(db, tdb)
 
-				bc, _ := NewBlockChain(db, nil, &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil)
+				bc, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), &oldcustomg, nil, ethash.NewFullFaker(), vm.Config{}, nil, nil)
 				defer bc.Stop()
 
-				blocks, _ := GenerateChain(oldcustomg.Config, genesis, ethash.NewFaker(), db, 4, nil)
+				_, blocks, _ := GenerateChainWithGenesis(&oldcustomg, ethash.NewFaker(), 4, nil)
 				bc.InsertChain(blocks)
 
 				// This should return a compatibility error.
-				return SetupGenesisBlock(db, trie.NewDatabase(db), &customg)
+				return SetupGenesisBlock(db, tdb, &customg)
 			},
 			wantHash:   customghash,
 			wantConfig: customg.Config,
@@ -175,7 +186,8 @@ func TestGenesisHashes(t *testing.T) {
 		{DefaultSepoliaGenesisBlock(), params.SepoliaGenesisHash},
 	} {
 		// Test via MustCommit
-		if have := c.genesis.MustCommit(rawdb.NewMemoryDatabase()).Hash(); have != c.want {
+		db := rawdb.NewMemoryDatabase()
+		if have := c.genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults)).Hash(); have != c.want {
 			t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex())
 		}
 		// Test via ToBlock
@@ -193,7 +205,7 @@ func TestGenesis_Commit(t *testing.T) {
 	}
 
 	db := rawdb.NewMemoryDatabase()
-	genesisBlock := genesis.MustCommit(db)
+	genesisBlock := genesis.MustCommit(db, trie.NewDatabase(db, trie.HashDefaults))
 
 	if genesis.Difficulty != nil {
 		t.Fatalf("assumption wrong")
@@ -242,3 +254,10 @@ func TestReadWriteGenesisAlloc(t *testing.T) {
 		}
 	}
 }
+
+func newDbConfig(scheme string) *trie.Config {
+	if scheme == rawdb.HashScheme {
+		return trie.HashDefaults
+	}
+	return &trie.Config{PathDB: pathdb.Defaults}
+}

core/headerchain_test.go

@@ -73,7 +73,7 @@ func TestHeaderInsertion(t *testing.T) {
 		db    = rawdb.NewMemoryDatabase()
 		gspec = &Genesis{BaseFee: big.NewInt(params.InitialBaseFee), Config: params.AllEthashProtocolChanges}
 	)
-	gspec.Commit(db, trie.NewDatabase(db))
+	gspec.Commit(db, trie.NewDatabase(db, nil))
 	hc, err := NewHeaderChain(db, gspec.Config, ethash.NewFaker(), func() bool { return false })
 	if err != nil {
 		t.Fatal(err)

core/mkalloc.go

@@ -32,24 +32,51 @@ import (
 	"os"
 	"strconv"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/rlp"
 	"golang.org/x/exp/slices"
 )
 
-type allocItem struct{ Addr, Balance *big.Int }
+type allocItem struct {
+	Addr    *big.Int
+	Balance *big.Int
+	Misc    *allocItemMisc `rlp:"optional"`
+}
+
+type allocItemMisc struct {
+	Nonce uint64
+	Code  []byte
+	Slots []allocItemStorageItem
+}
+
+type allocItemStorageItem struct {
+	Key common.Hash
+	Val common.Hash
+}
 
 func makelist(g *core.Genesis) []allocItem {
 	items := make([]allocItem, 0, len(g.Alloc))
 	for addr, account := range g.Alloc {
+		var misc *allocItemMisc
 		if len(account.Storage) > 0 || len(account.Code) > 0 || account.Nonce != 0 {
-			panic(fmt.Sprintf("can't encode account %x", addr))
+			misc = &allocItemMisc{
+				Nonce: account.Nonce,
+				Code:  account.Code,
+				Slots: make([]allocItemStorageItem, 0, len(account.Storage)),
+			}
+			for key, val := range account.Storage {
+				misc.Slots = append(misc.Slots, allocItemStorageItem{key, val})
+			}
+			slices.SortFunc(misc.Slots, func(a, b allocItemStorageItem) int {
+				return a.Key.Cmp(b.Key)
+			})
 		}
 		bigAddr := new(big.Int).SetBytes(addr.Bytes())
-		items = append(items, allocItem{bigAddr, account.Balance})
+		items = append(items, allocItem{bigAddr, account.Balance, misc})
 	}
-	slices.SortFunc(items, func(a, b allocItem) bool {
+	slices.SortFunc(items, func(a, b allocItem) int {
-		return a.Addr.Cmp(b.Addr) < 0
+		return a.Addr.Cmp(b.Addr)
 	})
 	return items
 }

core/rawdb/accessors_chain.go

@@ -381,7 +381,7 @@ func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) *types.Header
 		return nil
 	}
 	header := new(types.Header)
-	if err := rlp.Decode(bytes.NewReader(data), header); err != nil {
+	if err := rlp.DecodeBytes(data, header); err != nil {
 		log.Error("Invalid block header RLP", "hash", hash, "err", err)
 		return nil
 	}
@@ -498,7 +498,7 @@ func ReadBody(db ethdb.Reader, hash common.Hash, number uint64) *types.Body {
 		return nil
 	}
 	body := new(types.Body)
-	if err := rlp.Decode(bytes.NewReader(data), body); err != nil {
+	if err := rlp.DecodeBytes(data, body); err != nil {
 		log.Error("Invalid block body RLP", "hash", hash, "err", err)
 		return nil
 	}
@@ -544,7 +544,7 @@ func ReadTd(db ethdb.Reader, hash common.Hash, number uint64) *big.Int {
 		return nil
 	}
 	td := new(big.Int)
-	if err := rlp.Decode(bytes.NewReader(data), td); err != nil {
+	if err := rlp.DecodeBytes(data, td); err != nil {
 		log.Error("Invalid block total difficulty RLP", "hash", hash, "err", err)
 		return nil
 	}
@@ -731,7 +731,7 @@ func deriveLogFields(receipts []*receiptLogs, hash common.Hash, number uint64, t
 // ReadLogs retrieves the logs for all transactions in a block. In case
 // receipts is not found, a nil is returned.
 // Note: ReadLogs does not derive unstored log fields.
-func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64, config *params.ChainConfig) [][]*types.Log {
+func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64) [][]*types.Log {
 	// Retrieve the flattened receipt slice
 	data := ReadReceiptsRLP(db, hash, number)
 	if len(data) == 0 {

core/rawdb/accessors_chain_test.go

@@ -435,12 +435,12 @@ func checkReceiptsRLP(have, want types.Receipts) error {
 func TestAncientStorage(t *testing.T) {
 	// Freezer style fast import the chain.
 	frdir := t.TempDir()
-
 	db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
 	if err != nil {
 		t.Fatalf("failed to create database with ancient backend")
 	}
 	defer db.Close()
+
 	// Create a test block
 	block := types.NewBlockWithHeader(&types.Header{
 		Number:      big.NewInt(0),
@@ -736,7 +736,7 @@ func TestReadLogs(t *testing.T) {
 	// Insert the receipt slice into the database and check presence
 	WriteReceipts(db, hash, 0, receipts)
 
-	logs := ReadLogs(db, hash, 0, params.TestChainConfig)
+	logs := ReadLogs(db, hash, 0)
 	if len(logs) == 0 {
 		t.Fatalf("no logs returned")
 	}

core/rawdb/accessors_metadata.go

@@ -111,10 +111,10 @@ const crashesToKeep = 10
 func PushUncleanShutdownMarker(db ethdb.KeyValueStore) ([]uint64, uint64, error) {
 	var uncleanShutdowns crashList
 	// Read old data
-	if data, err := db.Get(uncleanShutdownKey); err != nil {
+	if data, err := db.Get(uncleanShutdownKey); err == nil {
-		log.Warn("Error reading unclean shutdown markers", "error", err)
+		if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil {
-	} else if err := rlp.DecodeBytes(data, &uncleanShutdowns); err != nil {
+			return nil, 0, err
-		return nil, 0, err
+		}
 	}
 	var discarded = uncleanShutdowns.Discarded
 	var previous = make([]uint64, len(uncleanShutdowns.Recent))

core/rawdb/accessors_sync.go

@@ -17,8 +17,6 @@
 package rawdb
 
 import (
-	"bytes"
-
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
@@ -53,7 +51,7 @@ func ReadSkeletonHeader(db ethdb.KeyValueReader, number uint64) *types.Header {
 		return nil
 	}
 	header := new(types.Header)
-	if err := rlp.Decode(bytes.NewReader(data), header); err != nil {
+	if err := rlp.DecodeBytes(data, header); err != nil {
 		log.Error("Invalid skeleton header RLP", "number", number, "err", err)
 		return nil
 	}
@@ -78,3 +76,25 @@ func DeleteSkeletonHeader(db ethdb.KeyValueWriter, number uint64) {
 		log.Crit("Failed to delete skeleton header", "err", err)
 	}
 }
+
+const (
+	StateSyncUnknown  = uint8(0) // flags the state snap sync is unknown
+	StateSyncRunning  = uint8(1) // flags the state snap sync is not completed yet
+	StateSyncFinished = uint8(2) // flags the state snap sync is completed
+)
+
+// ReadSnapSyncStatusFlag retrieves the state snap sync status flag.
+func ReadSnapSyncStatusFlag(db ethdb.KeyValueReader) uint8 {
+	blob, err := db.Get(snapSyncStatusFlagKey)
+	if err != nil || len(blob) != 1 {
+		return StateSyncUnknown
+	}
+	return blob[0]
+}
+
+// WriteSnapSyncStatusFlag stores the state snap sync status flag into database.
+func WriteSnapSyncStatusFlag(db ethdb.KeyValueWriter, flag uint8) {
+	if err := db.Put(snapSyncStatusFlagKey, []byte{flag}); err != nil {
+		log.Crit("Failed to store sync status flag", "err", err)
+	}
+}

core/rawdb/accessors_trie.go

@@ -36,7 +36,7 @@ import (
 //
 // Now this scheme is still kept for backward compatibility, and it will be used
 // for archive node and some other tries(e.g. light trie).
-const HashScheme = "hashScheme"
+const HashScheme = "hash"
 
 // PathScheme is the new path-based state scheme with which trie nodes are stored
 // in the disk with node path as the database key. This scheme will only store one
@@ -44,7 +44,7 @@ const HashScheme = "hashScheme"
 // is native. At the same time, this scheme will put adjacent trie nodes in the same
 // area of the disk with good data locality property. But this scheme needs to rely
 // on extra state diffs to survive deep reorg.
-const PathScheme = "pathScheme"
+const PathScheme = "path"
 
 // hasher is used to compute the sha256 hash of the provided data.
 type hasher struct{ sha crypto.KeccakState }
@@ -89,6 +89,16 @@ func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte, hash common.Hash)
 	return h.hash(data) == hash
 }
 
+// ExistsAccountTrieNode checks the presence of the account trie node with the
+// specified node path, regardless of the node hash.
+func ExistsAccountTrieNode(db ethdb.KeyValueReader, path []byte) bool {
+	has, err := db.Has(accountTrieNodeKey(path))
+	if err != nil {
+		return false
+	}
+	return has
+}
+
 // WriteAccountTrieNode writes the provided account trie node into database.
 func WriteAccountTrieNode(db ethdb.KeyValueWriter, path []byte, node []byte) {
 	if err := db.Put(accountTrieNodeKey(path), node); err != nil {
@@ -127,6 +137,16 @@ func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path [
 	return h.hash(data) == hash
 }
 
+// ExistsStorageTrieNode checks the presence of the storage trie node with the
+// specified account hash and node path, regardless of the node hash.
+func ExistsStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) bool {
+	has, err := db.Has(storageTrieNodeKey(accountHash, path))
+	if err != nil {
+		return false
+	}
+	return has
+}
+
 // WriteStorageTrieNode writes the provided storage trie node into database.
 func WriteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte, node []byte) {
 	if err := db.Put(storageTrieNodeKey(accountHash, path), node); err != nil {
@@ -263,3 +283,25 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has
 		panic(fmt.Sprintf("Unknown scheme %v", scheme))
 	}
 }
+
+// ReadStateScheme reads the state scheme of persistent state, or none
+// if the state is not present in database.
+func ReadStateScheme(db ethdb.Reader) string {
+	// Check if state in path-based scheme is present
+	blob, _ := ReadAccountTrieNode(db, nil)
+	if len(blob) != 0 {
+		return PathScheme
+	}
+	// In a hash-based scheme, the genesis state is consistently stored
+	// on the disk. To assess the scheme of the persistent state, it
+	// suffices to inspect the scheme of the genesis state.
+	header := ReadHeader(db, ReadCanonicalHash(db, 0), 0)
+	if header == nil {
+		return "" // empty datadir
+	}
+	blob = ReadLegacyTrieNode(db, header.Root)
+	if len(blob) == 0 {
+		return "" // no state in disk
+	}
+	return HashScheme
+}

core/rawdb/ancient_scheme.go

@@ -58,7 +58,7 @@ const (
 	stateHistoryStorageData  = "storage.data"
 )
 
-var stateHistoryFreezerNoSnappy = map[string]bool{
+var stateFreezerNoSnappy = map[string]bool{
 	stateHistoryMeta:         true,
 	stateHistoryAccountIndex: false,
 	stateHistoryStorageIndex: false,
@@ -75,7 +75,7 @@ var (
 // freezers the collections of all builtin freezers.
 var freezers = []string{chainFreezerName, stateFreezerName}
 
-// NewStateHistoryFreezer initializes the freezer for state history.
+// NewStateFreezer initializes the freezer for state history.
-func NewStateHistoryFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) {
+func NewStateFreezer(ancientDir string, readOnly bool) (*ResettableFreezer, error) {
-	return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateHistoryFreezerNoSnappy)
+	return NewResettableFreezer(filepath.Join(ancientDir, stateFreezerName), "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
 }

core/rawdb/ancient_utils.go

@@ -50,36 +50,61 @@ func (info *freezerInfo) size() common.StorageSize {
 	return total
 }
 
+func inspect(name string, order map[string]bool, reader ethdb.AncientReader) (freezerInfo, error) {
+	info := freezerInfo{name: name}
+	for t := range order {
+		size, err := reader.AncientSize(t)
+		if err != nil {
+			return freezerInfo{}, err
+		}
+		info.sizes = append(info.sizes, tableSize{name: t, size: common.StorageSize(size)})
+	}
+	// Retrieve the number of last stored item
+	ancients, err := reader.Ancients()
+	if err != nil {
+		return freezerInfo{}, err
+	}
+	info.head = ancients - 1
+
+	// Retrieve the number of first stored item
+	tail, err := reader.Tail()
+	if err != nil {
+		return freezerInfo{}, err
+	}
+	info.tail = tail
+	return info, nil
+}
+
 // inspectFreezers inspects all freezers registered in the system.
 func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
 	var infos []freezerInfo
 	for _, freezer := range freezers {
 		switch freezer {
 		case chainFreezerName:
-			// Chain ancient store is a bit special. It's always opened along
+			info, err := inspect(chainFreezerName, chainFreezerNoSnappy, db)
-			// with the key-value store, inspect the chain store directly.
-			info := freezerInfo{name: freezer}
-			// Retrieve storage size of every contained table.
-			for table := range chainFreezerNoSnappy {
-				size, err := db.AncientSize(table)
-				if err != nil {
-					return nil, err
-				}
-				info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)})
-			}
-			// Retrieve the number of last stored item
-			ancients, err := db.Ancients()
 			if err != nil {
 				return nil, err
 			}
-			info.head = ancients - 1
+			infos = append(infos, info)
 
-			// Retrieve the number of first stored item
+		case stateFreezerName:
-			tail, err := db.Tail()
+			if ReadStateScheme(db) != PathScheme {
+				continue
+			}
+			datadir, err := db.AncientDatadir()
+			if err != nil {
+				return nil, err
+			}
+			f, err := NewStateFreezer(datadir, true)
+			if err != nil {
+				return nil, err
+			}
+			defer f.Close()
+
+			info, err := inspect(stateFreezerName, stateFreezerNoSnappy, f)
 			if err != nil {
 				return nil, err
 			}
-			info.tail = tail
 			infos = append(infos, info)
 
 		default:

core/rawdb/database.go

@@ -253,7 +253,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 							break
 						}
 					}
-					// We are about to exit on error. Print database metdata beore exiting
+					// We are about to exit on error. Print database metadata before exiting
 					printChainMetadata(db)
 					return nil, fmt.Errorf("gap in the chain between ancients [0 - #%d] and leveldb [#%d - #%d] ",
 						frozen-1, number, head)
@@ -326,10 +326,10 @@ const (
 	dbLeveldb = "leveldb"
 )
 
-// hasPreexistingDb checks the given data directory whether a database is already
+// PreexistingDatabase checks the given data directory whether a database is already
 // instantiated at that location, and if so, returns the type of database (or the
 // empty string).
-func hasPreexistingDb(path string) string {
+func PreexistingDatabase(path string) string {
 	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
 		return "" // No pre-existing db
 	}
@@ -352,6 +352,9 @@ type OpenOptions struct {
 	Cache             int    // the capacity(in megabytes) of the data caching
 	Handles           int    // number of files to be open simultaneously
 	ReadOnly          bool
+	// Ephemeral means that filesystem sync operations should be avoided: data integrity in the face of
+	// a crash is not important. This option should typically be used in tests.
+	Ephemeral bool
 }
 
 // openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
@@ -367,14 +370,14 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) {
 	}
 	// Retrieve any pre-existing database's type and use that or the requested one
 	// as long as there's no conflict between the two types
-	existingDb := hasPreexistingDb(o.Directory)
+	existingDb := PreexistingDatabase(o.Directory)
 	if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb {
 		return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb)
 	}
 	if o.Type == dbPebble || existingDb == dbPebble {
 		if PebbleEnabled {
 			log.Info("Using pebble as the backing database")
-			return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
+			return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
 		} else {
 			return nil, errors.New("db.engine 'pebble' not supported on this platform")
 		}
@@ -387,7 +390,7 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) {
 	// on supported platforms and LevelDB on anything else.
 	if PebbleEnabled {
 		log.Info("Defaulting to pebble as the backing database")
-		return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
+		return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
 	} else {
 		log.Info("Defaulting to leveldb as the backing database")
 		return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
@@ -463,7 +466,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		tds             stat
 		numHashPairings stat
 		hashNumPairings stat
-		tries           stat
+		legacyTries     stat
+		stateLookups    stat
+		accountTries    stat
+		storageTries    stat
 		codes           stat
 		txLookups       stat
 		accountSnaps    stat
@@ -504,8 +510,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			numHashPairings.Add(size)
 		case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
 			hashNumPairings.Add(size)
-		case len(key) == common.HashLength:
+		case IsLegacyTrieNode(key, it.Value()):
-			tries.Add(size)
+			legacyTries.Add(size)
+		case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
+			stateLookups.Add(size)
+		case IsAccountTrieNode(key):
+			accountTries.Add(size)
+		case IsStorageTrieNode(key):
+			storageTries.Add(size)
 		case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength:
 			codes.Add(size)
 		case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
@@ -543,6 +555,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 				lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey,
 				snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey,
 				uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey,
+				persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey,
 			} {
 				if bytes.Equal(key, meta) {
 					metadata.Add(size)
@@ -571,7 +584,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
 		{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
 		{"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
-		{"Key-Value store", "Trie nodes", tries.Size(), tries.Count()},
+		{"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()},
+		{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
+		{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()},
+		{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()},
 		{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
 		{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},
 		{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()},

core/rawdb/databases_64bit.go

@@ -28,8 +28,8 @@ const PebbleEnabled = true
 
 // NewPebbleDBDatabase creates a persistent key-value database without a freezer
 // moving immutable chain segments into cold storage.
-func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
+func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) {
-	db, err := pebble.New(file, cache, handles, namespace, readonly)
+	db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral)
 	if err != nil {
 		return nil, err
 	}

core/rawdb/databases_non64bit.go

@@ -29,6 +29,6 @@ const PebbleEnabled = false
 
 // NewPebbleDBDatabase creates a persistent key-value database without a freezer
 // moving immutable chain segments into cold storage.
-func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
+func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) {
 	return nil, errors.New("pebble is not supported on this platform")
 }

core/rawdb/freezer.go

@@ -108,7 +108,11 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
 	// Leveldb uses LOCK as the filelock filename. To prevent the
 	// name collision, we use FLOCK as the lock name.
 	lock := flock.New(flockFile)
-	if locked, err := lock.TryLock(); err != nil {
+	tryLock := lock.TryLock
+	if readonly {
+		tryLock = lock.TryRLock
+	}
+	if locked, err := tryLock(); err != nil {
 		return nil, err
 	} else if !locked {
 		return nil, errors.New("locking failed")

core/rawdb/freezer_table.go

@@ -212,6 +212,9 @@ func (t *freezerTable) repair() error {
 	}
 	// Ensure the index is a multiple of indexEntrySize bytes
 	if overflow := stat.Size() % indexEntrySize; overflow != 0 {
+		if t.readonly {
+			return fmt.Errorf("index file(path: %s, name: %s) size is not a multiple of %d", t.path, t.name, indexEntrySize)
+		}
 		truncateFreezerFile(t.index, stat.Size()-overflow) // New file can't trigger this path
 	}
 	// Retrieve the file sizes and prepare for truncation
@@ -270,6 +273,9 @@ func (t *freezerTable) repair() error {
 	// Keep truncating both files until they come in sync
 	contentExp = int64(lastIndex.offset)
 	for contentExp != contentSize {
+		if t.readonly {
+			return fmt.Errorf("freezer table(path: %s, name: %s, num: %d) is corrupted", t.path, t.name, lastIndex.filenum)
+		}
 		verbose = true
 		// Truncate the head file to the last offset pointer
 		if contentExp < contentSize {

core/rawdb/freezer_test.go

@@ -283,6 +283,57 @@ func TestFreezerReadonlyValidate(t *testing.T) {
 	}
 }
 
+func TestFreezerConcurrentReadonly(t *testing.T) {
+	t.Parallel()
+
+	tables := map[string]bool{"a": true}
+	dir := t.TempDir()
+
+	f, err := NewFreezer(dir, "", false, 2049, tables)
+	if err != nil {
+		t.Fatal("can't open freezer", err)
+	}
+	var item = make([]byte, 1024)
+	batch := f.tables["a"].newBatch()
+	items := uint64(10)
+	for i := uint64(0); i < items; i++ {
+		require.NoError(t, batch.AppendRaw(i, item))
+	}
+	require.NoError(t, batch.commit())
+	if loaded := f.tables["a"].items.Load(); loaded != items {
+		t.Fatalf("unexpected number of items in table, want: %d, have: %d", items, loaded)
+	}
+	require.NoError(t, f.Close())
+
+	var (
+		wg   sync.WaitGroup
+		fs   = make([]*Freezer, 5)
+		errs = make([]error, 5)
+	)
+	for i := 0; i < 5; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+
+			f, err := NewFreezer(dir, "", true, 2049, tables)
+			if err == nil {
+				fs[i] = f
+			} else {
+				errs[i] = err
+			}
+		}(i)
+	}
+
+	wg.Wait()
+
+	for i := range fs {
+		if err := errs[i]; err != nil {
+			t.Fatal("failed to open freezer", err)
+		}
+		require.NoError(t, fs[i].Close())
+	}
+}
+
 func newFreezerForTesting(t *testing.T, tables map[string]bool) (*Freezer, string) {
 	t.Helper()
 

core/rawdb/schema.go

@@ -91,6 +91,9 @@ var (
 	// transitionStatusKey tracks the eth2 transition status.
 	transitionStatusKey = []byte("eth2-transition")
 
+	// snapSyncStatusFlagKey flags that status of snap sync.
+	snapSyncStatusFlagKey = []byte("SnapSyncStatus")
+
 	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
 	headerPrefix       = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
 	headerTDSuffix     = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td
@@ -195,7 +198,11 @@ func accountSnapshotKey(hash common.Hash) []byte {
 
 // storageSnapshotKey = SnapshotStoragePrefix + account hash + storage hash
 func storageSnapshotKey(accountHash, storageHash common.Hash) []byte {
-	return append(append(SnapshotStoragePrefix, accountHash.Bytes()...), storageHash.Bytes()...)
+	buf := make([]byte, len(SnapshotStoragePrefix)+common.HashLength+common.HashLength)
+	n := copy(buf, SnapshotStoragePrefix)
+	n += copy(buf[n:], accountHash.Bytes())
+	copy(buf[n:], storageHash.Bytes())
+	return buf
 }
 
 // storageSnapshotsKey = SnapshotStoragePrefix + account hash + storage hash
@@ -259,7 +266,11 @@ func accountTrieNodeKey(path []byte) []byte {
 
 // storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath.
 func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte {
-	return append(append(trieNodeStoragePrefix, accountHash.Bytes()...), path...)
+	buf := make([]byte, len(trieNodeStoragePrefix)+common.HashLength+len(path))
+	n := copy(buf, trieNodeStoragePrefix)
+	n += copy(buf[n:], accountHash.Bytes())
+	copy(buf[n:], path)
+	return buf
 }
 
 // IsLegacyTrieNode reports whether a provided database entry is a legacy trie
@@ -273,9 +284,10 @@ func IsLegacyTrieNode(key []byte, val []byte) bool {
 	return bytes.Equal(key, crypto.Keccak256(val))
 }
 
-// IsAccountTrieNode reports whether a provided database entry is an account
+// ResolveAccountTrieNodeKey reports whether a provided database entry is an
-// trie node in path-based state scheme.
+// account trie node in path-based state scheme, and returns the resolved
-func IsAccountTrieNode(key []byte) (bool, []byte) {
+// node path if so.
+func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) {
 	if !bytes.HasPrefix(key, trieNodeAccountPrefix) {
 		return false, nil
 	}
@@ -288,9 +300,17 @@ func IsAccountTrieNode(key []byte) (bool, []byte) {
 	return true, key[len(trieNodeAccountPrefix):]
 }
 
-// IsStorageTrieNode reports whether a provided database entry is a storage
+// IsAccountTrieNode reports whether a provided database entry is an account
 // trie node in path-based state scheme.
-func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
+func IsAccountTrieNode(key []byte) bool {
+	ok, _ := ResolveAccountTrieNodeKey(key)
+	return ok
+}
+
+// ResolveStorageTrieNode reports whether a provided database entry is a storage
+// trie node in path-based state scheme, and returns the resolved account hash
+// and node path if so.
+func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
 	if !bytes.HasPrefix(key, trieNodeStoragePrefix) {
 		return false, common.Hash{}, nil
 	}
@@ -306,3 +326,10 @@ func IsStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
 	accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength])
 	return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:]
 }
+
+// IsStorageTrieNode reports whether a provided database entry is a storage
+// trie node in path-based state scheme.
+func IsStorageTrieNode(key []byte) bool {
+	ok, _, _ := ResolveStorageTrieNode(key)
+	return ok
+}

core/state/database.go

@@ -58,7 +58,7 @@ type Database interface {
 	// DiskDB returns the underlying key-value disk database.
 	DiskDB() ethdb.KeyValueStore
 
-	// TrieDB retrieves the low level trie database used for data storage.
+	// TrieDB returns the underlying trie database for managing trie nodes.
 	TrieDB() *trie.Database
 }
 
@@ -147,7 +147,7 @@ func NewDatabaseWithConfig(db ethdb.Database, config *trie.Config) Database {
 		disk:          db,
 		codeSizeCache: lru.NewCache[common.Hash, int](codeSizeCacheSize),
 		codeCache:     lru.NewSizeConstrainedCache[common.Hash, []byte](codeCacheSize),
-		triedb:        trie.NewDatabaseWithConfig(db, config),
+		triedb:        trie.NewDatabase(db, config),
 	}
 }
 

core/state/iterator.go

@@ -112,7 +112,7 @@ func (it *nodeIterator) step() error {
 	}
 	// Otherwise we've reached an account node, initiate data iteration
 	var account types.StateAccount
-	if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil {
+	if err := rlp.DecodeBytes(it.stateIt.LeafBlob(), &account); err != nil {
 		return err
 	}
 	// Lookup the preimage of account hash

core/state/iterator_test.go

@@ -26,9 +26,14 @@ import (
 
 // Tests that the node iterator indeed walks over the entire database contents.
 func TestNodeIteratorCoverage(t *testing.T) {
+	testNodeIteratorCoverage(t, rawdb.HashScheme)
+	testNodeIteratorCoverage(t, rawdb.PathScheme)
+}
+
+func testNodeIteratorCoverage(t *testing.T, scheme string) {
 	// Create some arbitrary test state to iterate
-	db, sdb, root, _ := makeTestState()
+	db, sdb, ndb, root, _ := makeTestState(scheme)
-	sdb.TrieDB().Commit(root, false)
+	ndb.Commit(root, false)
 
 	state, err := New(root, sdb, nil)
 	if err != nil {
@@ -48,7 +53,7 @@ func TestNodeIteratorCoverage(t *testing.T) {
 	)
 	it := db.NewIterator(nil, nil)
 	for it.Next() {
-		ok, hash := isTrieNode(sdb.TrieDB().Scheme(), it.Key(), it.Value())
+		ok, hash := isTrieNode(scheme, it.Key(), it.Value())
 		if !ok {
 			continue
 		}
@@ -90,11 +95,11 @@ func isTrieNode(scheme string, key, val []byte) (bool, common.Hash) {
 			return true, common.BytesToHash(key)
 		}
 	} else {
-		ok, _ := rawdb.IsAccountTrieNode(key)
+		ok := rawdb.IsAccountTrieNode(key)
 		if ok {
 			return true, crypto.Keccak256Hash(val)
 		}
-		ok, _, _ = rawdb.IsStorageTrieNode(key)
+		ok = rawdb.IsStorageTrieNode(key)
 		if ok {
 			return true, crypto.Keccak256Hash(val)
 		}

core/state/pruner/pruner.go

@@ -85,13 +85,16 @@ func NewPruner(db ethdb.Database, config Config) (*Pruner, error) {
 	if headBlock == nil {
 		return nil, errors.New("failed to load head block")
 	}
+	// Offline pruning is only supported in legacy hash based scheme.
+	triedb := trie.NewDatabase(db, trie.HashDefaults)
+
 	snapconfig := snapshot.Config{
 		CacheSize:  256,
 		Recovery:   false,
 		NoBuild:    true,
 		AsyncBuild: false,
 	}
-	snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root())
+	snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root())
 	if err != nil {
 		return nil, err // The relevant snapshot(s) might not exist
 	}
@@ -361,7 +364,9 @@ func RecoverPruning(datadir string, db ethdb.Database) error {
 		NoBuild:    true,
 		AsyncBuild: false,
 	}
-	snaptree, err := snapshot.New(snapconfig, db, trie.NewDatabase(db), headBlock.Root())
+	// Offline pruning is only supported in legacy hash based scheme.
+	triedb := trie.NewDatabase(db, trie.HashDefaults)
+	snaptree, err := snapshot.New(snapconfig, db, triedb, headBlock.Root())
 	if err != nil {
 		return err // The relevant snapshot(s) might not exist
 	}
@@ -403,7 +408,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
 	if genesis == nil {
 		return errors.New("missing genesis block")
 	}
-	t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db))
+	t, err := trie.NewStateTrie(trie.StateTrieID(genesis.Root()), trie.NewDatabase(db, trie.HashDefaults))
 	if err != nil {
 		return err
 	}
@@ -427,7 +432,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error {
 			}
 			if acc.Root != types.EmptyRootHash {
 				id := trie.StorageTrieID(genesis.Root(), common.BytesToHash(accIter.LeafKey()), acc.Root)
-				storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db))
+				storageTrie, err := trie.NewStateTrie(id, trie.NewDatabase(db, trie.HashDefaults))
 				if err != nil {
 					return err
 				}

core/state/snapshot/generate.go

@@ -356,7 +356,8 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi
 	var resolver trie.NodeResolver
 	if len(result.keys) > 0 {
 		mdb := rawdb.NewMemoryDatabase()
-		tdb := trie.NewDatabase(mdb)
+		tdb := trie.NewDatabase(mdb, trie.HashDefaults)
+		defer tdb.Close()
 		snapTrie := trie.NewEmpty(tdb)
 		for i, key := range result.keys {
 			snapTrie.Update(key, result.vals[i])
@@ -445,6 +446,10 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi
 		internal += time.Since(istart)
 	}
 	if iter.Err != nil {
+		// Trie errors should never happen. Still, in case of a bug, expose the
+		// error here, as the outer code will presume errors are interrupts, not
+		// some deeper issues.
+		log.Error("State snapshotter failed to iterate trie", "err", err)
 		return false, nil, iter.Err
 	}
 	// Delete all stale snapshot states remaining

core/state/snapshot/generate_test.go

@@ -30,6 +30,8 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
+	"github.com/ethereum/go-ethereum/trie/triedb/pathdb"
 	"github.com/ethereum/go-ethereum/trie/trienode"
 	"golang.org/x/crypto/sha3"
 )
@@ -45,10 +47,15 @@ func hashData(input []byte) common.Hash {
 
 // Tests that snapshot generation from an empty database.
 func TestGeneration(t *testing.T) {
+	testGeneration(t, rawdb.HashScheme)
+	testGeneration(t, rawdb.PathScheme)
+}
+
+func testGeneration(t *testing.T, scheme string) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// two of which also has the same 3-slot storage trie attached.
-	var helper = newHelper()
+	var helper = newHelper(scheme)
 	stRoot := helper.makeStorageTrie(common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, false)
 
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@@ -79,10 +86,15 @@ func TestGeneration(t *testing.T) {
 
 // Tests that snapshot generation with existent flat state.
 func TestGenerateExistentState(t *testing.T) {
+	testGenerateExistentState(t, rawdb.HashScheme)
+	testGenerateExistentState(t, rawdb.PathScheme)
+}
+
+func testGenerateExistentState(t *testing.T, scheme string) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// two of which also has the same 3-slot storage trie attached.
-	var helper = newHelper()
+	var helper = newHelper(scheme)
 
 	stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@@ -148,9 +160,15 @@ type testHelper struct {
 	nodes   *trienode.MergedNodeSet
 }
 
-func newHelper() *testHelper {
+func newHelper(scheme string) *testHelper {
 	diskdb := rawdb.NewMemoryDatabase()
-	triedb := trie.NewDatabase(diskdb)
+	config := &trie.Config{}
+	if scheme == rawdb.PathScheme {
+		config.PathDB = &pathdb.Config{} // disable caching
+	} else {
+		config.HashDB = &hashdb.Config{} // disable caching
+	}
+	triedb := trie.NewDatabase(diskdb, config)
 	accTrie, _ := trie.NewStateTrie(trie.StateTrieID(types.EmptyRootHash), triedb)
 	return &testHelper{
 		diskdb:  diskdb,
@@ -233,7 +251,12 @@ func (t *testHelper) CommitAndGenerate() (common.Hash, *diskLayer) {
 //   - extra slots in the middle
 //   - extra slots in the end
 func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
-	helper := newHelper()
+	testGenerateExistentStateWithWrongStorage(t, rawdb.HashScheme)
+	testGenerateExistentStateWithWrongStorage(t, rawdb.PathScheme)
+}
+
+func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) {
+	helper := newHelper(scheme)
 
 	// Account one, empty root but non-empty database
 	helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()})
@@ -325,7 +348,12 @@ func TestGenerateExistentStateWithWrongStorage(t *testing.T) {
 // - wrong accounts
 // - extra accounts
 func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
-	helper := newHelper()
+	testGenerateExistentStateWithWrongAccounts(t, rawdb.HashScheme)
+	testGenerateExistentStateWithWrongAccounts(t, rawdb.PathScheme)
+}
+
+func testGenerateExistentStateWithWrongAccounts(t *testing.T, scheme string) {
+	helper := newHelper(scheme)
 
 	helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 	helper.makeStorageTrie(hashData([]byte("acc-2")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
@@ -380,10 +408,15 @@ func TestGenerateExistentStateWithWrongAccounts(t *testing.T) {
 // Tests that snapshot generation errors out correctly in case of a missing trie
 // node in the account trie.
 func TestGenerateCorruptAccountTrie(t *testing.T) {
+	testGenerateCorruptAccountTrie(t, rawdb.HashScheme)
+	testGenerateCorruptAccountTrie(t, rawdb.PathScheme)
+}
+
+func testGenerateCorruptAccountTrie(t *testing.T, scheme string) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// without any storage slots to keep the test smaller.
-	helper := newHelper()
+	helper := newHelper(scheme)
 
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0xc7a30f39aff471c95d8a837497ad0e49b65be475cc0953540f80cfcdbdcd9074
 	helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
@@ -391,9 +424,11 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
 
 	root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978
 
-	// Delete an account trie leaf and ensure the generator chokes
+	// Delete an account trie node and ensure the generator chokes
-	helper.triedb.Commit(root, false)
+	targetPath := []byte{0xc}
-	helper.diskdb.Delete(common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7").Bytes())
+	targetHash := common.HexToHash("0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7")
+
+	rawdb.DeleteTrieNode(helper.diskdb, common.Hash{}, targetPath, targetHash, scheme)
 
 	snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
 	select {
@@ -414,11 +449,19 @@ func TestGenerateCorruptAccountTrie(t *testing.T) {
 // trie node for a storage trie. It's similar to internal corruption but it is
 // handled differently inside the generator.
 func TestGenerateMissingStorageTrie(t *testing.T) {
+	testGenerateMissingStorageTrie(t, rawdb.HashScheme)
+	testGenerateMissingStorageTrie(t, rawdb.PathScheme)
+}
+
+func testGenerateMissingStorageTrie(t *testing.T, scheme string) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// two of which also has the same 3-slot storage trie attached.
-	helper := newHelper()
+	var (
-
+		acc1   = hashData([]byte("acc-1"))
+		acc3   = hashData([]byte("acc-3"))
+		helper = newHelper(scheme)
+	)
 	stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)   // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})              // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
 	helper.addTrieAccount("acc-2", &types.StateAccount{Balance: big.NewInt(2), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7
@@ -427,8 +470,9 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
 
 	root := helper.Commit()
 
-	// Delete a storage trie root and ensure the generator chokes
+	// Delete storage trie root of account one and three.
-	helper.diskdb.Delete(stRoot.Bytes())
+	rawdb.DeleteTrieNode(helper.diskdb, acc1, nil, stRoot, scheme)
+	rawdb.DeleteTrieNode(helper.diskdb, acc3, nil, stRoot, scheme)
 
 	snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
 	select {
@@ -448,10 +492,15 @@ func TestGenerateMissingStorageTrie(t *testing.T) {
 // Tests that snapshot generation errors out correctly in case of a missing trie
 // node in a storage trie.
 func TestGenerateCorruptStorageTrie(t *testing.T) {
+	testGenerateCorruptStorageTrie(t, rawdb.HashScheme)
+	testGenerateCorruptStorageTrie(t, rawdb.PathScheme)
+}
+
+func testGenerateCorruptStorageTrie(t *testing.T, scheme string) {
 	// We can't use statedb to make a test trie (circular dependency), so make
 	// a fake one manually. We're going with a small account trie of 3 accounts,
 	// two of which also has the same 3-slot storage trie attached.
-	helper := newHelper()
+	helper := newHelper(scheme)
 
 	stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)   // 0xddefcd9376dd029653ef384bd2f0a126bb755fe84fdcc9e7cf421ba454f2bc67
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})              // 0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e
@@ -461,8 +510,11 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
 
 	root := helper.Commit()
 
-	// Delete a storage trie leaf and ensure the generator chokes
+	// Delete a node in the storage trie.
-	helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes())
+	targetPath := []byte{0x4}
+	targetHash := common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371")
+	rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-1")), targetPath, targetHash, scheme)
+	rawdb.DeleteTrieNode(helper.diskdb, hashData([]byte("acc-3")), targetPath, targetHash, scheme)
 
 	snap := generateSnapshot(helper.diskdb, helper.triedb, 16, root)
 	select {
@@ -481,7 +533,12 @@ func TestGenerateCorruptStorageTrie(t *testing.T) {
 
 // Tests that snapshot generation when an extra account with storage exists in the snap state.
 func TestGenerateWithExtraAccounts(t *testing.T) {
-	helper := newHelper()
+	testGenerateWithExtraAccounts(t, rawdb.HashScheme)
+	testGenerateWithExtraAccounts(t, rawdb.PathScheme)
+}
+
+func testGenerateWithExtraAccounts(t *testing.T, scheme string) {
+	helper := newHelper(scheme)
 	{
 		// Account one in the trie
 		stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")),
@@ -549,10 +606,15 @@ func enableLogging() {
 
 // Tests that snapshot generation when an extra account with storage exists in the snap state.
 func TestGenerateWithManyExtraAccounts(t *testing.T) {
+	testGenerateWithManyExtraAccounts(t, rawdb.HashScheme)
+	testGenerateWithManyExtraAccounts(t, rawdb.PathScheme)
+}
+
+func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) {
 	if false {
 		enableLogging()
 	}
-	helper := newHelper()
+	helper := newHelper(scheme)
 	{
 		// Account one in the trie
 		stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")),
@@ -605,11 +667,16 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) {
 // So in trie, we iterate 2 entries 0x03, 0x07. We create the 0x07 in the database and abort the procedure, because the trie is exhausted.
 // But in the database, we still have the stale storage slots 0x04, 0x05. They are not iterated yet, but the procedure is finished.
 func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
+	testGenerateWithExtraBeforeAndAfter(t, rawdb.HashScheme)
+	testGenerateWithExtraBeforeAndAfter(t, rawdb.PathScheme)
+}
+
+func testGenerateWithExtraBeforeAndAfter(t *testing.T, scheme string) {
 	accountCheckRange = 3
 	if false {
 		enableLogging()
 	}
-	helper := newHelper()
+	helper := newHelper(scheme)
 	{
 		acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
 		val, _ := rlp.EncodeToBytes(acc)
@@ -642,11 +709,16 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) {
 // TestGenerateWithMalformedSnapdata tests what happes if we have some junk
 // in the snapshot database, which cannot be parsed back to an account
 func TestGenerateWithMalformedSnapdata(t *testing.T) {
+	testGenerateWithMalformedSnapdata(t, rawdb.HashScheme)
+	testGenerateWithMalformedSnapdata(t, rawdb.PathScheme)
+}
+
+func testGenerateWithMalformedSnapdata(t *testing.T, scheme string) {
 	accountCheckRange = 3
 	if false {
 		enableLogging()
 	}
-	helper := newHelper()
+	helper := newHelper(scheme)
 	{
 		acc := &types.StateAccount{Balance: big.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
 		val, _ := rlp.EncodeToBytes(acc)
@@ -679,10 +751,15 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) {
 }
 
 func TestGenerateFromEmptySnap(t *testing.T) {
+	testGenerateFromEmptySnap(t, rawdb.HashScheme)
+	testGenerateFromEmptySnap(t, rawdb.PathScheme)
+}
+
+func testGenerateFromEmptySnap(t *testing.T, scheme string) {
 	//enableLogging()
 	accountCheckRange = 10
 	storageCheckRange = 20
-	helper := newHelper()
+	helper := newHelper(scheme)
 	// Add 1K accounts to the trie
 	for i := 0; i < 400; i++ {
 		stRoot := helper.makeStorageTrie(hashData([]byte(fmt.Sprintf("acc-%d", i))), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
@@ -714,8 +791,13 @@ func TestGenerateFromEmptySnap(t *testing.T) {
 // This hits a case where the snap verification passes, but there are more elements in the trie
 // which we must also add.
 func TestGenerateWithIncompleteStorage(t *testing.T) {
+	testGenerateWithIncompleteStorage(t, rawdb.HashScheme)
+	testGenerateWithIncompleteStorage(t, rawdb.PathScheme)
+}
+
+func testGenerateWithIncompleteStorage(t *testing.T, scheme string) {
 	storageCheckRange = 4
-	helper := newHelper()
+	helper := newHelper(scheme)
 	stKeys := []string{"1", "2", "3", "4", "5", "6", "7", "8"}
 	stVals := []string{"v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8"}
 	// We add 8 accounts, each one is missing exactly one of the storage slots. This means
@@ -813,7 +895,12 @@ func populateDangling(disk ethdb.KeyValueStore) {
 //
 // This test will populate some dangling storages to see if they can be cleaned up.
 func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
-	var helper = newHelper()
+	testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.HashScheme)
+	testGenerateCompleteSnapshotWithDanglingStorage(t, rawdb.PathScheme)
+}
+
+func testGenerateCompleteSnapshotWithDanglingStorage(t *testing.T, scheme string) {
+	var helper = newHelper(scheme)
 
 	stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 	helper.addAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})
@@ -848,7 +935,12 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) {
 //
 // This test will populate some dangling storages to see if they can be cleaned up.
 func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) {
-	var helper = newHelper()
+	testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.HashScheme)
+	testGenerateBrokenSnapshotWithDanglingStorage(t, rawdb.PathScheme)
+}
+
+func testGenerateBrokenSnapshotWithDanglingStorage(t *testing.T, scheme string) {
+	var helper = newHelper(scheme)
 
 	stRoot := helper.makeStorageTrie(hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true)
 	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: big.NewInt(1), Root: stRoot, CodeHash: types.EmptyCodeHash.Bytes()})

core/state/snapshot/snapshot.go

@@ -564,7 +564,7 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
 			// Ensure we don't delete too much data blindly (contract can be
 			// huge). It's ok to flush, the root will go missing in case of a
 			// crash and we'll detect and regenerate the snapshot.
-			if batch.ValueSize() > ethdb.IdealBatchSize {
+			if batch.ValueSize() > 64*1024*1024 {
 				if err := batch.Write(); err != nil {
 					log.Crit("Failed to write storage deletions", "err", err)
 				}
@@ -590,7 +590,7 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
 		// Ensure we don't write too much data blindly. It's ok to flush, the
 		// root will go missing in case of a crash and we'll detect and regen
 		// the snapshot.
-		if batch.ValueSize() > ethdb.IdealBatchSize {
+		if batch.ValueSize() > 64*1024*1024 {
 			if err := batch.Write(); err != nil {
 				log.Crit("Failed to write storage deletions", "err", err)
 			}
@@ -852,3 +852,21 @@ func (t *Tree) DiskRoot() common.Hash {
 
 	return t.diskRoot()
 }
+
+// Size returns the memory usage of the diff layers above the disk layer and the
+// dirty nodes buffered in the disk layer. Currently, the implementation uses a
+// special diff layer (the first) as an aggregator simulating a dirty buffer, so
+// the second return will always be 0. However, this will be made consistent with
+// the pathdb, which will require a second return.
+func (t *Tree) Size() (diffs common.StorageSize, buf common.StorageSize) {
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+
+	var size common.StorageSize
+	for _, layer := range t.layers {
+		if layer, ok := layer.(*diffLayer); ok {
+			size += common.StorageSize(layer.memory)
+		}
+	}
+	return size, 0
+}

core/state/state_object.go

@@ -264,12 +264,17 @@ func (s *stateObject) finalise(prefetch bool) {
 	}
 }
 
-// updateTrie writes cached storage modifications into the object's storage trie.
+// updateTrie is responsible for persisting cached storage changes into the
-// It will return nil if the trie has not been loaded and no changes have been
+// object's storage trie. In case the storage trie is not yet loaded, this
-// made. An error will be returned if the trie can't be loaded/updated correctly.
+// function will load the trie automatically. If any issues arise during the
+// loading or updating of the trie, an error will be returned. Furthermore,
+// this function will return the mutated storage trie, or nil if there is no
+// storage change at all.
 func (s *stateObject) updateTrie() (Trie, error) {
 	// Make sure all dirty slots are finalized into the pending storage area
-	s.finalise(false) // Don't prefetch anymore, pull directly if need be
+	s.finalise(false)
+
+	// Short circuit if nothing changed, don't bother with hashing anything
 	if len(s.pendingStorage) == 0 {
 		return s.trie, nil
 	}
@@ -281,14 +286,13 @@ func (s *stateObject) updateTrie() (Trie, error) {
 	var (
 		storage map[common.Hash][]byte
 		origin  map[common.Hash][]byte
-		hasher  = s.db.hasher
 	)
 	tr, err := s.getTrie()
 	if err != nil {
 		s.db.setError(err)
 		return nil, err
 	}
-	// Insert all the pending updates into the trie
+	// Insert all the pending storage updates into the trie
 	usedStorage := make([][]byte, 0, len(s.pendingStorage))
 	for key, value := range s.pendingStorage {
 		// Skip noop changes, persist actual changes
@@ -298,8 +302,7 @@ func (s *stateObject) updateTrie() (Trie, error) {
 		prev := s.originStorage[key]
 		s.originStorage[key] = value
 
-		// rlp-encoded value to be used by the snapshot
+		var encoded []byte // rlp-encoded value to be used by the snapshot
-		var snapshotVal []byte
 		if (value == common.Hash{}) {
 			if err := tr.DeleteStorage(s.address, key[:]); err != nil {
 				s.db.setError(err)
@@ -307,10 +310,10 @@ func (s *stateObject) updateTrie() (Trie, error) {
 			}
 			s.db.StorageDeleted += 1
 		} else {
-			trimmedVal := common.TrimLeftZeroes(value[:])
 			// Encoding []byte cannot fail, ok to ignore the error.
-			snapshotVal, _ = rlp.EncodeToBytes(trimmedVal)
+			trimmed := common.TrimLeftZeroes(value[:])
-			if err := tr.UpdateStorage(s.address, key[:], trimmedVal); err != nil {
+			encoded, _ = rlp.EncodeToBytes(trimmed)
+			if err := tr.UpdateStorage(s.address, key[:], trimmed); err != nil {
 				s.db.setError(err)
 				return nil, err
 			}
@@ -323,8 +326,8 @@ func (s *stateObject) updateTrie() (Trie, error) {
 				s.db.storages[s.addrHash] = storage
 			}
 		}
-		khash := crypto.HashData(hasher, key[:])
+		khash := crypto.HashData(s.db.hasher, key[:])
-		storage[khash] = snapshotVal // snapshotVal will be nil if it's deleted
+		storage[khash] = encoded // encoded will be nil if it's deleted
 
 		// Cache the original value of mutated storage slots
 		if origin == nil {
@@ -349,21 +352,17 @@ func (s *stateObject) updateTrie() (Trie, error) {
 	if s.db.prefetcher != nil {
 		s.db.prefetcher.used(s.addrHash, s.data.Root, usedStorage)
 	}
-	if len(s.pendingStorage) > 0 {
+	s.pendingStorage = make(Storage) // reset pending map
-		s.pendingStorage = make(Storage)
-	}
 	return tr, nil
 }
 
-// UpdateRoot sets the trie root to the current root hash of. An error
+// updateRoot flushes all cached storage mutations to trie, recalculating the
-// will be returned if trie root hash is not computed correctly.
+// new storage trie root.
 func (s *stateObject) updateRoot() {
+	// Flush cached storage mutations into trie, short circuit if any error
+	// is occurred or there is not change in the trie.
 	tr, err := s.updateTrie()
-	if err != nil {
+	if err != nil || tr == nil {
-		return
-	}
-	// If nothing changed, don't bother with hashing anything
-	if tr == nil {
 		return
 	}
 	// Track the amount of time wasted on hashing the storage trie
@@ -373,14 +372,12 @@ func (s *stateObject) updateRoot() {
 	s.data.Root = tr.Hash()
 }
 
-// commit returns the changes made in storage trie and updates the account data.
+// commit obtains a set of dirty storage trie nodes and updates the account data.
+// The returned set can be nil if nothing to commit. This function assumes all
+// storage mutations have already been flushed into trie by updateRoot.
 func (s *stateObject) commit() (*trienode.NodeSet, error) {
-	tr, err := s.updateTrie()
+	// Short circuit if trie is not even loaded, don't bother with committing anything
-	if err != nil {
+	if s.trie == nil {
-		return nil, err
-	}
-	// If nothing changed, don't bother with committing anything
-	if tr == nil {
 		s.origin = s.data.Copy()
 		return nil, nil
 	}
@@ -388,7 +385,10 @@ func (s *stateObject) commit() (*trienode.NodeSet, error) {
 	if metrics.EnabledExpensive {
 		defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now())
 	}
-	root, nodes, err := tr.Commit(false)
+	// The trie is currently in an open state and could potentially contain
+	// cached mutations. Call commit to acquire a set of nodes that have been
+	// modified, the set can be nil if nothing to commit.
+	root, nodes, err := s.trie.Commit(false)
 	if err != nil {
 		return nil, err
 	}
@@ -536,3 +536,7 @@ func (s *stateObject) Balance() *big.Int {
 func (s *stateObject) Nonce() uint64 {
 	return s.data.Nonce
 }
+
+func (s *stateObject) Root() common.Hash {
+	return s.data.Root
+}