fix merge conflict

.gitignore

@@ -47,4 +47,6 @@ profile.cov
 /dashboard/assets/package-lock.json
 
 **/yarn-error.log
-logs/
+logs/
+
+tests/spec-tests/

accounts/abi/bind/backend.go

@@ -36,6 +36,10 @@ var (
 	// on a backend that doesn't implement PendingContractCaller.
 	ErrNoPendingState = errors.New("backend does not support pending state")
 
+	// ErrNoBlockHashState is raised when attempting to perform a block hash action
+	// on a backend that doesn't implement BlockHashContractCaller.
+	ErrNoBlockHashState = errors.New("backend does not support block hash state")
+
 	// ErrNoCodeAfterDeploy is returned by WaitDeployed if contract creation leaves
 	// an empty contract behind.
 	ErrNoCodeAfterDeploy = errors.New("no contract code after deployment")
@@ -64,6 +68,17 @@ type PendingContractCaller interface {
 	PendingCallContract(ctx context.Context, call ethereum.CallMsg) ([]byte, error)
 }
 
+// BlockHashContractCaller defines methods to perform contract calls on a specific block hash.
+// Call will try to discover this interface when access to a block by hash is requested.
+// If the backend does not support the block hash state, Call returns ErrNoBlockHashState.
+type BlockHashContractCaller interface {
+	// CodeAtHash returns the code of the given account in the state at the specified block hash.
+	CodeAtHash(ctx context.Context, contract common.Address, blockHash common.Hash) ([]byte, error)
+
+	// CallContractAtHash executes an Ethereum contract all against the state at the specified block hash.
+	CallContractAtHash(ctx context.Context, call ethereum.CallMsg, blockHash common.Hash) ([]byte, error)
+}
+
 // ContractTransactor defines the methods needed to allow operating with a contract
 // on a write only basis. Besides the transacting method, the remainder are helpers
 // used when the user does not provide some needed values, but rather leaves it up

accounts/abi/bind/backends/simulated.go

@@ -50,6 +50,7 @@ var _ bind.ContractBackend = (*SimulatedBackend)(nil)
 
 var (
 	errBlockNumberUnsupported  = errors.New("simulatedBackend cannot access blocks other than the latest block")
+	errBlockHashUnsupported    = errors.New("simulatedBackend cannot access blocks by hash other than the latest block")
 	errBlockDoesNotExist       = errors.New("block does not exist in blockchain")
 	errTransactionDoesNotExist = errors.New("transaction does not exist")
 )
@@ -199,6 +200,23 @@ func (b *SimulatedBackend) CodeAt(ctx context.Context, contract common.Address,
 	if err != nil {
 		return nil, err
 	}
+	return stateDB.GetCode(contract), nil
+}
+
+// CodeAtHash returns the code associated with a certain account in the blockchain.
+func (b *SimulatedBackend) CodeAtHash(ctx context.Context, contract common.Address, blockHash common.Hash) ([]byte, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	header, err := b.headerByHash(blockHash)
+	if err != nil {
+		return nil, err
+	}
+
+	stateDB, err := b.blockchain.StateAt(header.Root)
+	if err != nil {
+		return nil, err
+	}
 
 	return stateDB.GetCode(contract), nil
 }
@@ -212,7 +230,6 @@ func (b *SimulatedBackend) BalanceAt(ctx context.Context, contract common.Addres
 	if err != nil {
 		return nil, err
 	}
-
 	return stateDB.GetBalance(contract), nil
 }
 
@@ -225,7 +242,6 @@ func (b *SimulatedBackend) NonceAt(ctx context.Context, contract common.Address,
 	if err != nil {
 		return 0, err
 	}
-
 	return stateDB.GetNonce(contract), nil
 }
 
@@ -238,7 +254,6 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres
 	if err != nil {
 		return nil, err
 	}
-
 	val := stateDB.GetState(contract, key)
 	return val[:], nil
 }
@@ -324,7 +339,11 @@ func (b *SimulatedBackend) blockByNumber(ctx context.Context, number *big.Int) (
 func (b *SimulatedBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
+	return b.headerByHash(hash)
+}
 
+// headerByHash retrieves a header from the database by hash without Lock.
+func (b *SimulatedBackend) headerByHash(hash common.Hash) (*types.Header, error) {
 	if hash == b.pendingBlock.Hash() {
 		return b.pendingBlock.Header(), nil
 	}
@@ -440,6 +459,22 @@ func (b *SimulatedBackend) CallContract(ctx context.Context, call ethereum.CallM
 	if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number) != 0 {
 		return nil, errBlockNumberUnsupported
 	}
+	return b.callContractAtHead(ctx, call)
+}
+
+// CallContractAtHash executes a contract call on a specific block hash.
+func (b *SimulatedBackend) CallContractAtHash(ctx context.Context, call ethereum.CallMsg, blockHash common.Hash) ([]byte, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	if blockHash != b.blockchain.CurrentBlock().Hash() {
+		return nil, errBlockHashUnsupported
+	}
+	return b.callContractAtHead(ctx, call)
+}
+
+// callContractAtHead executes a contract call against the latest block state.
+func (b *SimulatedBackend) callContractAtHead(ctx context.Context, call ethereum.CallMsg) ([]byte, error) {
 	stateDB, err := b.blockchain.State()
 	if err != nil {
 		return nil, err
@@ -590,7 +625,7 @@ func (b *SimulatedBackend) EstimateGas(ctx context.Context, call ethereum.CallMs
 			return 0, err
 		}
 		if failed {
-			if result != nil && result.Err != vm.ErrOutOfGas {
+			if result != nil && !errors.Is(result.Err, vm.ErrOutOfGas) {
 				if len(result.Revert()) > 0 {
 					return 0, newRevertError(result)
 				}
@@ -610,8 +645,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 	if call.GasPrice != nil && (call.GasFeeCap != nil || call.GasTipCap != nil) {
 		return nil, errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified")
 	}
-	head := b.blockchain.CurrentHeader()
+	if !b.blockchain.Config().IsLondon(header.Number) {
-	if !b.blockchain.Config().IsLondon(head.Number) {
 		// If there's no basefee, then it must be a non-1559 execution
 		if call.GasPrice == nil {
 			call.GasPrice = new(big.Int)
@@ -633,13 +667,13 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM
 			// Backfill the legacy gasPrice for EVM execution, unless we're all zeroes
 			call.GasPrice = new(big.Int)
 			if call.GasFeeCap.BitLen() > 0 || call.GasTipCap.BitLen() > 0 {
-				call.GasPrice = math.BigMin(new(big.Int).Add(call.GasTipCap, head.BaseFee), call.GasFeeCap)
+				call.GasPrice = math.BigMin(new(big.Int).Add(call.GasTipCap, header.BaseFee), call.GasFeeCap)
 			}
 		}
 	}
 	// Ensure message is initialized properly.
 	if call.Gas == 0 {
-		call.Gas = 50000000
+		call.Gas = 10 * header.GasLimit
 	}
 	if call.Value == nil {
 		call.Value = new(big.Int)
@@ -700,8 +734,10 @@ func (b *SimulatedBackend) SendTransaction(ctx context.Context, tx *types.Transa
 		}
 		block.AddTxWithChain(b.blockchain, tx)
 	})
-	stateDB, _ := b.blockchain.State()
+	stateDB, err := b.blockchain.State()
-
+	if err != nil {
+		return err
+	}
 	b.pendingBlock = blocks[0]
 	b.pendingState, _ = state.New(b.pendingBlock.Root(), stateDB.Database(), nil)
 	b.pendingReceipts = receipts[0]
@@ -821,11 +857,12 @@ func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error {
 	blocks, _ := core.GenerateChain(b.config, block, ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) {
 		block.OffsetTime(int64(adjustment.Seconds()))
 	})
-	stateDB, _ := b.blockchain.State()
+	stateDB, err := b.blockchain.State()
-
+	if err != nil {
+		return err
+	}
 	b.pendingBlock = blocks[0]
 	b.pendingState, _ = state.New(b.pendingBlock.Root(), stateDB.Database(), nil)
-
 	return nil
 }
 

accounts/abi/bind/backends/simulated_test.go

@@ -996,6 +996,43 @@ func TestCodeAt(t *testing.T) {
 	}
 }
 
+func TestCodeAtHash(t *testing.T) {
+	testAddr := crypto.PubkeyToAddress(testKey.PublicKey)
+	sim := simTestBackend(testAddr)
+	defer sim.Close()
+	bgCtx := context.Background()
+	code, err := sim.CodeAtHash(bgCtx, testAddr, sim.Blockchain().CurrentHeader().Hash())
+	if err != nil {
+		t.Errorf("could not get code at test addr: %v", err)
+	}
+	if len(code) != 0 {
+		t.Errorf("got code for account that does not have contract code")
+	}
+
+	parsed, err := abi.JSON(strings.NewReader(abiJSON))
+	if err != nil {
+		t.Errorf("could not get code at test addr: %v", err)
+	}
+	auth, _ := bind.NewKeyedTransactorWithChainID(testKey, big.NewInt(1337))
+	contractAddr, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(abiBin), sim)
+	if err != nil {
+		t.Errorf("could not deploy contract: %v tx: %v contract: %v", err, tx, contract)
+	}
+
+	blockHash := sim.Commit()
+	code, err = sim.CodeAtHash(bgCtx, contractAddr, blockHash)
+	if err != nil {
+		t.Errorf("could not get code at test addr: %v", err)
+	}
+	if len(code) == 0 {
+		t.Errorf("did not get code for account that has contract code")
+	}
+	// ensure code received equals code deployed
+	if !bytes.Equal(code, common.FromHex(deployedCode)) {
+		t.Errorf("code received did not match expected deployed code:\n expected %v\n actual %v", common.FromHex(deployedCode), code)
+	}
+}
+
 // When receive("X") is called with sender 0x00... and value 1, it produces this tx receipt:
 //
 //	receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]}
@@ -1038,7 +1075,7 @@ func TestPendingAndCallContract(t *testing.T) {
 		t.Errorf("response from calling contract was expected to be 'hello world' instead received %v", string(res))
 	}
 
-	sim.Commit()
+	blockHash := sim.Commit()
 
 	// make sure you can call the contract
 	res, err = sim.CallContract(bgCtx, ethereum.CallMsg{
@@ -1056,6 +1093,23 @@ func TestPendingAndCallContract(t *testing.T) {
 	if !bytes.Equal(res, expectedReturn) || !strings.Contains(string(res), "hello world") {
 		t.Errorf("response from calling contract was expected to be 'hello world' instead received %v", string(res))
 	}
+
+	// make sure you can call the contract by hash
+	res, err = sim.CallContractAtHash(bgCtx, ethereum.CallMsg{
+		From: testAddr,
+		To:   &addr,
+		Data: input,
+	}, blockHash)
+	if err != nil {
+		t.Errorf("could not call receive method on contract: %v", err)
+	}
+	if len(res) == 0 {
+		t.Errorf("result of contract call was empty: %v", res)
+	}
+
+	if !bytes.Equal(res, expectedReturn) || !strings.Contains(string(res), "hello world") {
+		t.Errorf("response from calling contract was expected to be 'hello world' instead received %v", string(res))
+	}
 }
 
 // This test is based on the following contract:

accounts/abi/bind/base.go

@@ -48,6 +48,7 @@ type CallOpts struct {
 	Pending     bool            // Whether to operate on the pending state or the last known one
 	From        common.Address  // Optional the sender address, otherwise the first account is used
 	BlockNumber *big.Int        // Optional the block number on which the call should be performed
+	BlockHash   common.Hash     // Optional the block hash on which the call should be performed
 	Context     context.Context // Network context to support cancellation and timeouts (nil = no timeout)
 }
 
@@ -189,6 +190,23 @@ func (c *BoundContract) Call(opts *CallOpts, results *[]interface{}, method stri
 				return ErrNoCode
 			}
 		}
+	} else if opts.BlockHash != (common.Hash{}) {
+		bh, ok := c.caller.(BlockHashContractCaller)
+		if !ok {
+			return ErrNoBlockHashState
+		}
+		output, err = bh.CallContractAtHash(ctx, msg, opts.BlockHash)
+		if err != nil {
+			return err
+		}
+		if len(output) == 0 {
+			// Make sure we have a contract to operate on, and bail out otherwise.
+			if code, err = bh.CodeAtHash(ctx, c.address, opts.BlockHash); err != nil {
+				return err
+			} else if len(code) == 0 {
+				return ErrNoCode
+			}
+		}
 	} else {
 		output, err = c.caller.CallContract(ctx, msg, opts.BlockNumber)
 		if err != nil {

accounts/abi/bind/base_test.go

@@ -114,6 +114,26 @@ func (mc *mockPendingCaller) PendingCallContract(ctx context.Context, call ether
 	return mc.pendingCallContractBytes, mc.pendingCallContractErr
 }
 
+type mockBlockHashCaller struct {
+	*mockCaller
+	codeAtHashBytes          []byte
+	codeAtHashErr            error
+	codeAtHashCalled         bool
+	callContractAtHashCalled bool
+	callContractAtHashBytes  []byte
+	callContractAtHashErr    error
+}
+
+func (mc *mockBlockHashCaller) CodeAtHash(ctx context.Context, contract common.Address, hash common.Hash) ([]byte, error) {
+	mc.codeAtHashCalled = true
+	return mc.codeAtHashBytes, mc.codeAtHashErr
+}
+
+func (mc *mockBlockHashCaller) CallContractAtHash(ctx context.Context, call ethereum.CallMsg, hash common.Hash) ([]byte, error) {
+	mc.callContractAtHashCalled = true
+	return mc.callContractAtHashBytes, mc.callContractAtHashErr
+}
+
 func TestPassingBlockNumber(t *testing.T) {
 	mc := &mockPendingCaller{
 		mockCaller: &mockCaller{
@@ -400,6 +420,15 @@ func TestCall(t *testing.T) {
 			Pending: true,
 		},
 		method: method,
+	}, {
+		name: "ok hash",
+		mc: &mockBlockHashCaller{
+			codeAtHashBytes: []byte{0},
+		},
+		opts: &bind.CallOpts{
+			BlockHash: common.Hash{0xaa},
+		},
+		method: method,
 	}, {
 		name:    "pack error, no method",
 		mc:      new(mockCaller),
@@ -413,6 +442,14 @@ func TestCall(t *testing.T) {
 		},
 		method:       method,
 		wantErrExact: bind.ErrNoPendingState,
+	}, {
+		name: "interface error, blockHash but not a BlockHashContractCaller",
+		mc:   new(mockCaller),
+		opts: &bind.CallOpts{
+			BlockHash: common.Hash{0xaa},
+		},
+		method:       method,
+		wantErrExact: bind.ErrNoBlockHashState,
 	}, {
 		name: "pending call canceled",
 		mc: &mockPendingCaller{
@@ -460,6 +497,34 @@ func TestCall(t *testing.T) {
 		mc:           new(mockCaller),
 		method:       method,
 		wantErrExact: bind.ErrNoCode,
+	}, {
+		name: "call contract at hash error",
+		mc: &mockBlockHashCaller{
+			callContractAtHashErr: context.DeadlineExceeded,
+		},
+		opts: &bind.CallOpts{
+			BlockHash: common.Hash{0xaa},
+		},
+		method:       method,
+		wantErrExact: context.DeadlineExceeded,
+	}, {
+		name: "code at error",
+		mc: &mockBlockHashCaller{
+			codeAtHashErr: errors.New(""),
+		},
+		opts: &bind.CallOpts{
+			BlockHash: common.Hash{0xaa},
+		},
+		method:  method,
+		wantErr: true,
+	}, {
+		name: "no code at hash",
+		mc:   new(mockBlockHashCaller),
+		opts: &bind.CallOpts{
+			BlockHash: common.Hash{0xaa},
+		},
+		method:       method,
+		wantErrExact: bind.ErrNoCode,
 	}, {
 		name: "unpack error missing arg",
 		mc: &mockCaller{

accounts/keystore/watch.go

@@ -20,6 +20,7 @@
 package keystore
 
 import (
+	"os"
 	"time"
 
 	"github.com/ethereum/go-ethereum/log"
@@ -77,7 +78,9 @@ func (w *watcher) loop() {
 	}
 	defer watcher.Close()
 	if err := watcher.Add(w.ac.keydir); err != nil {
-		logger.Warn("Failed to watch keystore folder", "err", err)
+		if !os.IsNotExist(err) {
+			logger.Warn("Failed to watch keystore folder", "err", err)
+		}
 		return
 	}
 

accounts/scwallet/README.md

@@ -8,7 +8,7 @@
 
 ## Preparing the smartcard
 
-  **WARNING: FOILLOWING THESE INSTRUCTIONS WILL DESTROY THE MASTER KEY ON YOUR CARD. ONLY PROCEED IF NO FUNDS ARE ASSOCIATED WITH THESE ACCOUNTS**
+  **WARNING: FOLLOWING THESE INSTRUCTIONS WILL DESTROY THE MASTER KEY ON YOUR CARD. ONLY PROCEED IF NO FUNDS ARE ASSOCIATED WITH THESE ACCOUNTS**
 
   You can use status' [keycard-cli](https://github.com/status-im/keycard-cli) and you should get _at least_ version 2.1.1 of their [smartcard application](https://github.com/status-im/status-keycard/releases/download/2.2.1/keycard_v2.2.1.cap)
 

appveyor.yml

@@ -54,4 +54,4 @@ for:
       - go run build/ci.go archive -arch %GETH_ARCH% -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds
       - go run build/ci.go nsis -arch %GETH_ARCH% -signer WINDOWS_SIGNING_KEY -upload gethstore/builds
     test_script:
-      - go run build/ci.go test -dlgo -arch %GETH_ARCH% -cc %GETH_CC%
+      - go run build/ci.go test -dlgo -arch %GETH_ARCH% -cc %GETH_CC% -short

build/checksums.txt

@@ -1,25 +1,30 @@
 # This file contains sha256 checksums of optional build dependencies.
 
+# version:spec-tests 1.0.6
 # https://github.com/ethereum/execution-spec-tests/releases
-24bac679f3a2d8240d8e08e7f6a70b70c2dabf673317d924cf1d1887b9fe1f81  fixtures.tar.gz
+# https://github.com/ethereum/execution-spec-tests/releases/download/v1.0.6/
+485af7b66cf41eb3a8c1bd46632913b8eb95995df867cf665617bbc9b4beedd1  fixtures_develop.tar.gz
 
+# version:golang 1.21.3
 # https://go.dev/dl/
-818d46ede85682dd551ad378ef37a4d247006f12ec59b5b755601d2ce114369a  go1.21.0.src.tar.gz
+186f2b6f8c8b704e696821b09ab2041a5c1ee13dcbc3156a13adcf75931ee488  go1.21.3.src.tar.gz
-b314de9f704ab122c077d2ec8e67e3670affe8865479d1f01991e7ac55d65e70  go1.21.0.darwin-amd64.tar.gz
+27014fc69e301d7588a169ca239b3cc609f0aa1abf38528bf0d20d3b259211eb  go1.21.3.darwin-amd64.tar.gz
-3aca44de55c5e098de2f406e98aba328898b05d509a2e2a356416faacf2c4566  go1.21.0.darwin-arm64.tar.gz
+65302a7a9f7a4834932b3a7a14cb8be51beddda757b567a2f9e0cbd0d7b5a6ab  go1.21.3.darwin-arm64.tar.gz
-312a0065714a50862af714e7a5b3fbbd70fe68f905ffb9bcc56d42eadf6bffab  go1.21.0.freebsd-386.tar.gz
+8e0cd2f66cf1bde9d07b4aee01e3d7c3cfdd14e20650488e1683da4b8492594a  go1.21.3.freebsd-386.tar.gz
-b8eaa36654625df799654f77f4af0ea7bd9e5e760ebe86e68fe7c484748ae995  go1.21.0.freebsd-amd64.tar.gz
+6e74f65f586e93d1f3947894766f69e9b2ebda488592a09df61f36f06bfe58a8  go1.21.3.freebsd-amd64.tar.gz
-0e6f378d9b072fab0a3d9ff4d5e990d98487d47252dba8160015a61e6bd0bcba  go1.21.0.linux-386.tar.gz
+fb209fd070db500a84291c5a95251cceeb1723e8f6142de9baca5af70a927c0e  go1.21.3.linux-386.tar.gz
-d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742  go1.21.0.linux-amd64.tar.gz
+1241381b2843fae5a9707eec1f8fb2ef94d827990582c7c7c32f5bdfbfd420c8  go1.21.3.linux-amd64.tar.gz
-f3d4548edf9b22f26bbd49720350bbfe59d75b7090a1a2bff1afad8214febaf3  go1.21.0.linux-arm64.tar.gz
+fc90fa48ae97ba6368eecb914343590bbb61b388089510d0c56c2dde52987ef3  go1.21.3.linux-arm64.tar.gz
-e377a0004957c8c560a3ff99601bce612330a3d95ba3b0a2ae144165fc87deb1  go1.21.0.linux-armv6l.tar.gz
+a1ddcaaf0821a12a800884c14cb4268ce1c1f5a0301e9060646f1e15e611c6c7  go1.21.3.linux-armv6l.tar.gz
-e938ffc81d8ebe5efc179240960ba22da6a841ff05d5cab7ce2547112b14a47f  go1.21.0.linux-ppc64le.tar.gz
+3b0e10a3704f164a6e85e0377728ec5fd21524fabe4c925610e34076586d5826  go1.21.3.linux-ppc64le.tar.gz
-be7338df8e5d5472dfa307b0df2b446d85d001b0a2a3cdb1a14048d751b70481  go1.21.0.linux-s390x.tar.gz
+4c78e2e6f4c684a3d5a9bdc97202729053f44eb7be188206f0627ef3e18716b6  go1.21.3.linux-s390x.tar.gz
-af920fbb74fc3d173118dc3cc35f02a709c1de642700e92a91a7d16981df3fec  go1.21.0.windows-386.zip
+e36737f4f2fadb4d2f919ec4ce517133a56e06064cca6e82fc883bb000c4d56c  go1.21.3.windows-386.zip
-732121e64e0ecb07c77fdf6cc1bc5ce7b242c2d40d4ac29021ad4c64a08731f6  go1.21.0.windows-amd64.zip
+27c8daf157493f288d42a6f38debc6a2cb391f6543139eba9152fceca0be2a10  go1.21.3.windows-amd64.zip
-41342f5a0f8c083b14c68bde738ddcd313a4f53a5854bfdfab47f0e88247de12  go1.21.0.windows-arm64.zip
+bfb7a5c56f9ded07d8ae0e0b3702ac07b65e68fa8f33da24ed6df4ce01fe2c5c  go1.21.3.windows-arm64.zip
 
-# https://github.com/golangci/golangci-lint/releases
+# version:golangci 1.51.1
+# https://github.com/golangci/golangci-lint/releases/
+# https://github.com/golangci/golangci-lint/releases/download/v1.51.1/
 fba08acc4027f69f07cef48fbff70b8a7ecdfaa1c2aba9ad3fb31d60d9f5d4bc  golangci-lint-1.51.1-darwin-amd64.tar.gz
 75b8f0ff3a4e68147156be4161a49d4576f1be37a0b506473f8c482140c1e7f2  golangci-lint-1.51.1-darwin-arm64.tar.gz
 e06b3459aaed356e1667580be00b05f41f3b2e29685d12cdee571c23e1edb414  golangci-lint-1.51.1-freebsd-386.tar.gz
@@ -48,4 +53,12 @@ bce02f7232723cb727755ee11f168a700a00896a25d37f87c4b173bce55596b4  golangci-lint-
 cf6403f84707ce8c98664736772271bc8874f2e760c2fd0f00cf3e85963507e9  golangci-lint-1.51.1-windows-armv7.zip
 
 # This is the builder on PPA that will build Go itself (inception-y), don't modify!
+#
+# This version is fine to be old and full of security holes, we just use it
+# to build the latest Go. Don't change it. If it ever becomes insufficient,
+# we need to switch over to a recursive builder to jump across supported
+# versions.
+#
+# version:ppa-builder 1.19.6
+# https://go.dev/dl/
 d7f0013f82e6d7f862cc6cb5c8cdb48eef5f2e239b35baa97e2f1a7466043767  go1.19.6.src.tar.gz

build/ci.go

@@ -136,23 +136,6 @@ var (
 		"golang-go":   "/usr/lib/go",
 	}
 
-	// This is the version of Go that will be downloaded by
-	//
-	//     go run ci.go install -dlgo
-	dlgoVersion = "1.21.0"
-
-	// This is the version of Go that will be used to bootstrap the PPA builder.
-	//
-	// This version is fine to be old and full of security holes, we just use it
-	// to build the latest Go. Don't change it. If it ever becomes insufficient,
-	// we need to switch over to a recursive builder to jumpt across supported
-	// versions.
-	gobootVersion = "1.19.6"
-
-	// This is the version of execution-spec-tests that we are using.
-	// When updating, you must also update build/checksums.txt.
-	executionSpecTestsVersion = "1.0.2"
-
 	// This is where the tests should be unpacked.
 	executionSpecTestsDir = "tests/spec-tests"
 )
@@ -192,6 +175,8 @@ func main() {
 		doWindowsInstaller(os.Args[2:])
 	case "purge":
 		doPurge(os.Args[2:])
+	case "sanitycheck":
+		doSanityCheck()
 	default:
 		log.Fatal("unknown command ", os.Args[1])
 	}
@@ -213,9 +198,8 @@ func doInstall(cmdline []string) {
 	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
 	if *dlgo {
 		csdb := build.MustLoadChecksums("build/checksums.txt")
-		tc.Root = build.DownloadGo(csdb, dlgoVersion)
+		tc.Root = build.DownloadGo(csdb)
 	}
-
 	// Disable CLI markdown doc generation in release builds.
 	buildTags := []string{"urfave_cli_no_docs"}
 
@@ -301,6 +285,7 @@ func doTest(cmdline []string) {
 		coverage = flag.Bool("coverage", false, "Whether to record code coverage")
 		verbose  = flag.Bool("v", false, "Whether to log verbosely")
 		race     = flag.Bool("race", false, "Execute the race detector")
+		short     = flag.Bool("short", false, "Pass the 'short'-flag to go test")
 		cachedir = flag.String("cachedir", "./build/cache", "directory for caching downloads")
 	)
 	flag.CommandLine.Parse(cmdline)
@@ -312,7 +297,7 @@ func doTest(cmdline []string) {
 	// Configure the toolchain.
 	tc := build.GoToolchain{GOARCH: *arch, CC: *cc}
 	if *dlgo {
-		tc.Root = build.DownloadGo(csdb, dlgoVersion)
+		tc.Root = build.DownloadGo(csdb)
 	}
 	gotest := tc.Go("test")
 
@@ -322,6 +307,9 @@ func doTest(cmdline []string) {
 	// Enable CKZG backend in CI.
 	gotest.Args = append(gotest.Args, "-tags=ckzg")
 
+	// Enable integration-tests
+	gotest.Args = append(gotest.Args, "-tags=integrationtests")
+
 	// Test a single package at a time. CI builders are slow
 	// and some tests run into timeouts under load.
 	gotest.Args = append(gotest.Args, "-p", "1")
@@ -334,6 +322,9 @@ func doTest(cmdline []string) {
 	if *race {
 		gotest.Args = append(gotest.Args, "-race")
 	}
+	if *short {
+		gotest.Args = append(gotest.Args, "-short")
+	}
 
 	packages := []string{"./..."}
 	if len(flag.CommandLine.Args()) > 0 {
@@ -345,8 +336,12 @@ func doTest(cmdline []string) {
 
 // downloadSpecTestFixtures downloads and extracts the execution-spec-tests fixtures.
 func downloadSpecTestFixtures(csdb *build.ChecksumDB, cachedir string) string {
+	executionSpecTestsVersion, err := build.Version(csdb, "spec-tests")
+	if err != nil {
+		log.Fatal(err)
+	}
 	ext := ".tar.gz"
-	base := "fixtures" // TODO(MariusVanDerWijden) rename once the version becomes part of the filename
+	base := "fixtures_develop" // TODO(MariusVanDerWijden) rename once the version becomes part of the filename
 	url := fmt.Sprintf("https://github.com/ethereum/execution-spec-tests/releases/download/v%s/%s%s", executionSpecTestsVersion, base, ext)
 	archivePath := filepath.Join(cachedir, base+ext)
 	if err := csdb.DownloadFile(url, archivePath); err != nil {
@@ -377,9 +372,11 @@ func doLint(cmdline []string) {
 
 // downloadLinter downloads and unpacks golangci-lint.
 func downloadLinter(cachedir string) string {
-	const version = "1.51.1"
-
 	csdb := build.MustLoadChecksums("build/checksums.txt")
+	version, err := build.Version(csdb, "golangci")
+	if err != nil {
+		log.Fatal(err)
+	}
 	arch := runtime.GOARCH
 	ext := ".tar.gz"
 
@@ -761,6 +758,10 @@ func doDebianSource(cmdline []string) {
 // to bootstrap the builder Go.
 func downloadGoBootstrapSources(cachedir string) string {
 	csdb := build.MustLoadChecksums("build/checksums.txt")
+	gobootVersion, err := build.Version(csdb, "ppa-builder")
+	if err != nil {
+		log.Fatal(err)
+	}
 	file := fmt.Sprintf("go%s.src.tar.gz", gobootVersion)
 	url := "https://dl.google.com/go/" + file
 	dst := filepath.Join(cachedir, file)
@@ -773,6 +774,10 @@ func downloadGoBootstrapSources(cachedir string) string {
 // downloadGoSources downloads the Go source tarball.
 func downloadGoSources(cachedir string) string {
 	csdb := build.MustLoadChecksums("build/checksums.txt")
+	dlgoVersion, err := build.Version(csdb, "golang")
+	if err != nil {
+		log.Fatal(err)
+	}
 	file := fmt.Sprintf("go%s.src.tar.gz", dlgoVersion)
 	url := "https://dl.google.com/go/" + file
 	dst := filepath.Join(cachedir, file)
@@ -1099,3 +1104,7 @@ func doPurge(cmdline []string) {
 		log.Fatal(err)
 	}
 }
+
+func doSanityCheck() {
+	build.DownloadAndVerifyChecksums(build.MustLoadChecksums("build/checksums.txt"))
+}

build/update-license.go

@@ -65,10 +65,8 @@ var (
 		"vendor/", "tests/testdata/", "build/",
 
 		// don't relicense vendored sources
-		"cmd/internal/browser",
 		"common/bitutil/bitutil",
 		"common/prque/",
-		"consensus/ethash/xor.go",
 		"crypto/blake2b/",
 		"crypto/bn256/",
 		"crypto/bls12381/",
@@ -78,6 +76,7 @@ var (
 		"log/",
 		"metrics/",
 		"signer/rules/deps",
+		"internal/reexec",
 
 		// skip special licenses
 		"crypto/secp256k1", // Relicensed to BSD-3 via https://github.com/ethereum/go-ethereum/pull/17225

cmd/clef/main.go

@@ -1206,7 +1206,7 @@ func GenDoc(ctx *cli.Context) error {
 						URL:     accounts.URL{Path: ".. ignored .."},
 					},
 					{
-						Address: common.HexToAddress("0xffffffffffffffffffffffffffffffffffffffff"),
+						Address: common.MaxAddress,
 					},
 				}})
 	}

cmd/clef/run_test.go

@@ -21,8 +21,8 @@ import (
 	"os"
 	"testing"
 
-	"github.com/docker/docker/pkg/reexec"
 	"github.com/ethereum/go-ethereum/internal/cmdtest"
+	"github.com/ethereum/go-ethereum/internal/reexec"
 )
 
 const registeredName = "clef-test"

cmd/devp2p/crawl.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"errors"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -51,7 +52,14 @@ type resolver interface {
 	RequestENR(*enode.Node) (*enode.Node, error)
 }
 
-func newCrawler(input nodeSet, disc resolver, iters ...enode.Iterator) *crawler {
+func newCrawler(input nodeSet, bootnodes []*enode.Node, disc resolver, iters ...enode.Iterator) (*crawler, error) {
+	if len(input) == 0 {
+		input.add(bootnodes...)
+	}
+	if len(input) == 0 {
+		return nil, errors.New("no input nodes to start crawling")
+	}
+
 	c := &crawler{
 		input:     input,
 		output:    make(nodeSet, len(input)),
@@ -67,7 +75,7 @@ func newCrawler(input nodeSet, disc resolver, iters ...enode.Iterator) *crawler
 	for id, n := range input {
 		c.output[id] = n
 	}
-	return c
+	return c, nil
 }
 
 func (c *crawler) run(timeout time.Duration, nthreads int) nodeSet {

cmd/devp2p/discv4cmd.go

@@ -143,7 +143,7 @@ var discoveryNodeFlags = []cli.Flag{
 
 func discv4Ping(ctx *cli.Context) error {
 	n := getNodeArg(ctx)
-	disc := startV4(ctx)
+	disc, _ := startV4(ctx)
 	defer disc.Close()
 
 	start := time.Now()
@@ -156,7 +156,7 @@ func discv4Ping(ctx *cli.Context) error {
 
 func discv4RequestRecord(ctx *cli.Context) error {
 	n := getNodeArg(ctx)
-	disc := startV4(ctx)
+	disc, _ := startV4(ctx)
 	defer disc.Close()
 
 	respN, err := disc.RequestENR(n)
@@ -169,7 +169,7 @@ func discv4RequestRecord(ctx *cli.Context) error {
 
 func discv4Resolve(ctx *cli.Context) error {
 	n := getNodeArg(ctx)
-	disc := startV4(ctx)
+	disc, _ := startV4(ctx)
 	defer disc.Close()
 
 	fmt.Println(disc.Resolve(n).String())
@@ -196,10 +196,13 @@ func discv4ResolveJSON(ctx *cli.Context) error {
 		nodeargs = append(nodeargs, n)
 	}
 
-	// Run the crawler.
+	disc, config := startV4(ctx)
-	disc := startV4(ctx)
 	defer disc.Close()
-	c := newCrawler(inputSet, disc, enode.IterNodes(nodeargs))
+
+	c, err := newCrawler(inputSet, config.Bootnodes, disc, enode.IterNodes(nodeargs))
+	if err != nil {
+		return err
+	}
 	c.revalidateInterval = 0
 	output := c.run(0, 1)
 	writeNodesJSON(nodesFile, output)
@@ -211,14 +214,18 @@ func discv4Crawl(ctx *cli.Context) error {
 		return errors.New("need nodes file as argument")
 	}
 	nodesFile := ctx.Args().First()
-	var inputSet nodeSet
+	inputSet := make(nodeSet)
 	if common.FileExist(nodesFile) {
 		inputSet = loadNodesJSON(nodesFile)
 	}
 
-	disc := startV4(ctx)
+	disc, config := startV4(ctx)
 	defer disc.Close()
-	c := newCrawler(inputSet, disc, disc.RandomNodes())
+
+	c, err := newCrawler(inputSet, config.Bootnodes, disc, disc.RandomNodes())
+	if err != nil {
+		return err
+	}
 	c.revalidateInterval = 10 * time.Minute
 	output := c.run(ctx.Duration(crawlTimeoutFlag.Name), ctx.Int(crawlParallelismFlag.Name))
 	writeNodesJSON(nodesFile, output)
@@ -238,14 +245,14 @@ func discv4Test(ctx *cli.Context) error {
 }
 
 // startV4 starts an ephemeral discovery V4 node.
-func startV4(ctx *cli.Context) *discover.UDPv4 {
+func startV4(ctx *cli.Context) (*discover.UDPv4, discover.Config) {
 	ln, config := makeDiscoveryConfig(ctx)
 	socket := listen(ctx, ln)
 	disc, err := discover.ListenV4(socket, ln, config)
 	if err != nil {
 		exit(err)
 	}
-	return disc
+	return disc, config
 }
 
 func makeDiscoveryConfig(ctx *cli.Context) (*enode.LocalNode, discover.Config) {

cmd/devp2p/discv5cmd.go

@@ -81,7 +81,7 @@ var (
 
 func discv5Ping(ctx *cli.Context) error {
 	n := getNodeArg(ctx)
-	disc := startV5(ctx)
+	disc, _ := startV5(ctx)
 	defer disc.Close()
 
 	fmt.Println(disc.Ping(n))
@@ -90,7 +90,7 @@ func discv5Ping(ctx *cli.Context) error {
 
 func discv5Resolve(ctx *cli.Context) error {
 	n := getNodeArg(ctx)
-	disc := startV5(ctx)
+	disc, _ := startV5(ctx)
 	defer disc.Close()
 
 	fmt.Println(disc.Resolve(n))
@@ -102,14 +102,18 @@ func discv5Crawl(ctx *cli.Context) error {
 		return errors.New("need nodes file as argument")
 	}
 	nodesFile := ctx.Args().First()
-	var inputSet nodeSet
+	inputSet := make(nodeSet)
 	if common.FileExist(nodesFile) {
 		inputSet = loadNodesJSON(nodesFile)
 	}
 
-	disc := startV5(ctx)
+	disc, config := startV5(ctx)
 	defer disc.Close()
-	c := newCrawler(inputSet, disc, disc.RandomNodes())
+
+	c, err := newCrawler(inputSet, config.Bootnodes, disc, disc.RandomNodes())
+	if err != nil {
+		return err
+	}
 	c.revalidateInterval = 10 * time.Minute
 	output := c.run(ctx.Duration(crawlTimeoutFlag.Name), ctx.Int(crawlParallelismFlag.Name))
 	writeNodesJSON(nodesFile, output)
@@ -127,7 +131,7 @@ func discv5Test(ctx *cli.Context) error {
 }
 
 func discv5Listen(ctx *cli.Context) error {
-	disc := startV5(ctx)
+	disc, _ := startV5(ctx)
 	defer disc.Close()
 
 	fmt.Println(disc.Self())
@@ -135,12 +139,12 @@ func discv5Listen(ctx *cli.Context) error {
 }
 
 // startV5 starts an ephemeral discovery v5 node.
-func startV5(ctx *cli.Context) *discover.UDPv5 {
+func startV5(ctx *cli.Context) (*discover.UDPv5, discover.Config) {
 	ln, config := makeDiscoveryConfig(ctx)
 	socket := listen(ctx, ln)
 	disc, err := discover.ListenV5(socket, ln, config)
 	if err != nil {
 		exit(err)
 	}
-	return disc
+	return disc, config
 }

cmd/devp2p/dns_cloudflare.go

@@ -114,7 +114,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 	records = lrecords
 
 	log.Info(fmt.Sprintf("Retrieving existing TXT records on %s", name))
-	entries, err := c.DNSRecords(context.Background(), c.zoneID, cloudflare.DNSRecord{Type: "TXT"})
+	entries, _, err := c.ListDNSRecords(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), cloudflare.ListDNSRecordsParams{Type: "TXT"})
 	if err != nil {
 		return err
 	}
@@ -141,14 +141,25 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 			if path != name {
 				ttl = treeNodeTTLCloudflare // Max TTL permitted by Cloudflare
 			}
-			record := cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl}
+			record := cloudflare.CreateDNSRecordParams{Type: "TXT", Name: path, Content: val, TTL: ttl}
-			_, err = c.CreateDNSRecord(context.Background(), c.zoneID, record)
+			_, err = c.CreateDNSRecord(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), record)
 		} else if old.Content != val {
 			// Entry already exists, only change its content.
 			log.Info(fmt.Sprintf("Updating %s from %q to %q", path, old.Content, val))
 			updated++
-			old.Content = val
+
-			err = c.UpdateDNSRecord(context.Background(), c.zoneID, old.ID, old)
+			record := cloudflare.UpdateDNSRecordParams{
+				Type:     old.Type,
+				Name:     old.Name,
+				Content:  val,
+				Data:     old.Data,
+				ID:       old.ID,
+				Priority: old.Priority,
+				TTL:      old.TTL,
+				Proxied:  old.Proxied,
+				Tags:     old.Tags,
+			}
+			_, err = c.UpdateDNSRecord(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), record)
 		} else {
 			skipped++
 			log.Debug(fmt.Sprintf("Skipping %s = %q", path, val))
@@ -168,7 +179,7 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string)
 		// Stale entry, nuke it.
 		log.Debug(fmt.Sprintf("Deleting %s = %q", path, entry.Content))
 		deleted++
-		if err := c.DeleteDNSRecord(context.Background(), c.zoneID, entry.ID); err != nil {
+		if err := c.DeleteDNSRecord(context.Background(), cloudflare.ZoneIdentifier(c.zoneID), entry.ID); err != nil {
 			return fmt.Errorf("failed to delete %s: %v", path, err)
 		}
 	}

cmd/devp2p/internal/ethtest/chain_test.go

@@ -145,7 +145,7 @@ func TestChain_GetHeaders(t *testing.T) {
 	}{
 		{
 			req: GetBlockHeaders{
-				GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+				GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
 					Origin:  eth.HashOrNumber{Number: uint64(2)},
 					Amount:  uint64(5),
 					Skip:    1,
@@ -162,7 +162,7 @@ func TestChain_GetHeaders(t *testing.T) {
 		},
 		{
 			req: GetBlockHeaders{
-				GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+				GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
 					Origin:  eth.HashOrNumber{Number: uint64(chain.Len() - 1)},
 					Amount:  uint64(3),
 					Skip:    0,
@@ -177,7 +177,7 @@ func TestChain_GetHeaders(t *testing.T) {
 		},
 		{
 			req: GetBlockHeaders{
-				GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+				GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
 					Origin:  eth.HashOrNumber{Hash: chain.Head().Hash()},
 					Amount:  uint64(1),
 					Skip:    0,

cmd/devp2p/internal/ethtest/helpers.go

@@ -62,7 +62,6 @@ func (s *Suite) dial() (*Conn, error) {
 	}
 	// set default p2p capabilities
 	conn.caps = []p2p.Cap{
-		{Name: "eth", Version: 66},
 		{Name: "eth", Version: 67},
 		{Name: "eth", Version: 68},
 	}
@@ -237,8 +236,8 @@ func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message {
 				return errorf("could not get headers for inbound header request: %v", err)
 			}
 			resp := &BlockHeaders{
-				RequestId:          msg.ReqID(),
+				RequestId:           msg.ReqID(),
-				BlockHeadersPacket: eth.BlockHeadersPacket(headers),
+				BlockHeadersRequest: eth.BlockHeadersRequest(headers),
 			}
 			if err := c.Write(resp); err != nil {
 				return errorf("could not write to connection: %v", err)
@@ -267,7 +266,7 @@ func (c *Conn) headersRequest(request *GetBlockHeaders, chain *Chain, reqID uint
 	if !ok {
 		return nil, fmt.Errorf("unexpected message received: %s", pretty.Sdump(msg))
 	}
-	headers := []*types.Header(resp.BlockHeadersPacket)
+	headers := []*types.Header(resp.BlockHeadersRequest)
 	return headers, nil
 }
 
@@ -379,7 +378,7 @@ func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block) error {
 	conn.SetReadDeadline(time.Now().Add(20 * time.Second))
 	// create request
 	req := &GetBlockHeaders{
-		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+		GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
 			Origin: eth.HashOrNumber{Hash: block.Hash()},
 			Amount: 1,
 		},
@@ -604,8 +603,8 @@ func (s *Suite) hashAnnounce() error {
 			pretty.Sdump(blockHeaderReq))
 	}
 	err = sendConn.Write(&BlockHeaders{
-		RequestId:          blockHeaderReq.ReqID(),
+		RequestId:           blockHeaderReq.ReqID(),
-		BlockHeadersPacket: eth.BlockHeadersPacket{nextBlock.Header()},
+		BlockHeadersRequest: eth.BlockHeadersRequest{nextBlock.Header()},
 	})
 	if err != nil {
 		return fmt.Errorf("failed to write to connection: %v", err)

cmd/devp2p/internal/ethtest/snap.go

@@ -27,8 +27,8 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/protocols/snap"
 	"github.com/ethereum/go-ethereum/internal/utesting"
-	"github.com/ethereum/go-ethereum/light"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ethereum/go-ethereum/trie/trienode"
 	"golang.org/x/crypto/sha3"
 )
 
@@ -58,7 +58,7 @@ type accRangeTest struct {
 func (s *Suite) TestSnapGetAccountRange(t *utesting.T) {
 	var (
 		root           = s.chain.RootAt(999)
-		ffHash         = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
+		ffHash         = common.MaxHash
 		zero           = common.Hash{}
 		firstKeyMinus1 = common.HexToHash("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf29")
 		firstKey       = common.HexToHash("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a")
@@ -125,7 +125,7 @@ type stRangesTest struct {
 // TestSnapGetStorageRanges various forms of GetStorageRanges requests.
 func (s *Suite) TestSnapGetStorageRanges(t *utesting.T) {
 	var (
-		ffHash    = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
+		ffHash    = common.MaxHash
 		zero      = common.Hash{}
 		firstKey  = common.HexToHash("0x00bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a")
 		secondKey = common.HexToHash("0x09e47cd5056a689e708f22fe1f932709a320518e444f5f7d8d46a3da523d6606")
@@ -530,17 +530,13 @@ func (s *Suite) snapGetAccountRange(t *utesting.T, tc *accRangeTest) error {
 	for i, key := range hashes {
 		keys[i] = common.CopyBytes(key[:])
 	}
-	nodes := make(light.NodeList, len(proof))
+	nodes := make(trienode.ProofList, len(proof))
 	for i, node := range proof {
 		nodes[i] = node
 	}
-	proofdb := nodes.NodeSet()
+	proofdb := nodes.Set()
 
-	var end []byte
+	_, err = trie.VerifyRangeProof(tc.root, tc.origin[:], keys, accounts, proofdb)
-	if len(keys) > 0 {
-		end = keys[len(keys)-1]
-	}
-	_, err = trie.VerifyRangeProof(tc.root, tc.origin[:], end, keys, accounts, proofdb)
 	return err
 }
 

cmd/devp2p/internal/ethtest/suite.go

@@ -112,7 +112,7 @@ func (s *Suite) TestGetBlockHeaders(t *utesting.T) {
 	}
 	// write request
 	req := &GetBlockHeaders{
-		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+		GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
 			Origin:  eth.HashOrNumber{Hash: s.chain.blocks[1].Hash()},
 			Amount:  2,
 			Skip:    1,
@@ -150,7 +150,7 @@ func (s *Suite) TestSimultaneousRequests(t *utesting.T) {
 	// create two requests
 	req1 := &GetBlockHeaders{
 		RequestId: uint64(111),
-		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+		GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
 			Origin: eth.HashOrNumber{
 				Hash: s.chain.blocks[1].Hash(),
 			},
@@ -161,7 +161,7 @@ func (s *Suite) TestSimultaneousRequests(t *utesting.T) {
 	}
 	req2 := &GetBlockHeaders{
 		RequestId: uint64(222),
-		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+		GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
 			Origin: eth.HashOrNumber{
 				Hash: s.chain.blocks[1].Hash(),
 			},
@@ -201,10 +201,10 @@ func (s *Suite) TestSimultaneousRequests(t *utesting.T) {
 	if err != nil {
 		t.Fatalf("failed to get expected headers for request 2: %v", err)
 	}
-	if !headersMatch(expected1, headers1.BlockHeadersPacket) {
+	if !headersMatch(expected1, headers1.BlockHeadersRequest) {
 		t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1)
 	}
-	if !headersMatch(expected2, headers2.BlockHeadersPacket) {
+	if !headersMatch(expected2, headers2.BlockHeadersRequest) {
 		t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2)
 	}
 }
@@ -224,7 +224,7 @@ func (s *Suite) TestSameRequestID(t *utesting.T) {
 	reqID := uint64(1234)
 	request1 := &GetBlockHeaders{
 		RequestId: reqID,
-		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+		GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
 			Origin: eth.HashOrNumber{
 				Number: 1,
 			},
@@ -233,7 +233,7 @@ func (s *Suite) TestSameRequestID(t *utesting.T) {
 	}
 	request2 := &GetBlockHeaders{
 		RequestId: reqID,
-		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+		GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
 			Origin: eth.HashOrNumber{
 				Number: 33,
 			},
@@ -270,10 +270,10 @@ func (s *Suite) TestSameRequestID(t *utesting.T) {
 	if err != nil {
 		t.Fatalf("failed to get expected block headers: %v", err)
 	}
-	if !headersMatch(expected1, headers1.BlockHeadersPacket) {
+	if !headersMatch(expected1, headers1.BlockHeadersRequest) {
 		t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1)
 	}
-	if !headersMatch(expected2, headers2.BlockHeadersPacket) {
+	if !headersMatch(expected2, headers2.BlockHeadersRequest) {
 		t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2)
 	}
 }
@@ -290,7 +290,7 @@ func (s *Suite) TestZeroRequestID(t *utesting.T) {
 		t.Fatalf("peering failed: %v", err)
 	}
 	req := &GetBlockHeaders{
-		GetBlockHeadersPacket: &eth.GetBlockHeadersPacket{
+		GetBlockHeadersRequest: &eth.GetBlockHeadersRequest{
 			Origin: eth.HashOrNumber{Number: 0},
 			Amount: 2,
 		},
@@ -322,7 +322,7 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) {
 	// create block bodies request
 	req := &GetBlockBodies{
 		RequestId: uint64(55),
-		GetBlockBodiesPacket: eth.GetBlockBodiesPacket{
+		GetBlockBodiesRequest: eth.GetBlockBodiesRequest{
 			s.chain.blocks[54].Hash(),
 			s.chain.blocks[75].Hash(),
 		},
@@ -336,11 +336,11 @@ func (s *Suite) TestGetBlockBodies(t *utesting.T) {
 	if !ok {
 		t.Fatalf("unexpected: %s", pretty.Sdump(msg))
 	}
-	bodies := resp.BlockBodiesPacket
+	bodies := resp.BlockBodiesResponse
 	t.Logf("received %d block bodies", len(bodies))
-	if len(bodies) != len(req.GetBlockBodiesPacket) {
+	if len(bodies) != len(req.GetBlockBodiesRequest) {
 		t.Fatalf("wrong bodies in response: expected %d bodies, "+
-			"got %d", len(req.GetBlockBodiesPacket), len(bodies))
+			"got %d", len(req.GetBlockBodiesRequest), len(bodies))
 	}
 }
 
@@ -481,8 +481,8 @@ func (s *Suite) TestLargeTxRequest(t *utesting.T) {
 		hashes = append(hashes, hash)
 	}
 	getTxReq := &GetPooledTransactions{
-		RequestId:                   1234,
+		RequestId:                    1234,
-		GetPooledTransactionsPacket: hashes,
+		GetPooledTransactionsRequest: hashes,
 	}
 	if err = conn.Write(getTxReq); err != nil {
 		t.Fatalf("could not write to conn: %v", err)
@@ -490,7 +490,7 @@ func (s *Suite) TestLargeTxRequest(t *utesting.T) {
 	// check that all received transactions match those that were sent to node
 	switch msg := conn.waitForResponse(s.chain, timeout, getTxReq.RequestId).(type) {
 	case *PooledTransactions:
-		for _, gotTx := range msg.PooledTransactionsPacket {
+		for _, gotTx := range msg.PooledTransactionsResponse {
 			if _, exists := hashMap[gotTx.Hash()]; !exists {
 				t.Fatalf("unexpected tx received: %v", gotTx.Hash())
 			}
@@ -547,8 +547,8 @@ func (s *Suite) TestNewPooledTxs(t *utesting.T) {
 		msg := conn.readAndServe(s.chain, timeout)
 		switch msg := msg.(type) {
 		case *GetPooledTransactions:
-			if len(msg.GetPooledTransactionsPacket) != len(hashes) {
+			if len(msg.GetPooledTransactionsRequest) != len(hashes) {
-				t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsPacket))
+				t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsRequest))
 			}
 			return
 

cmd/devp2p/internal/ethtest/suite_test.go

@@ -120,6 +120,7 @@ func setupGeth(stack *node.Node) error {
 	if err != nil {
 		return err
 	}
+	backend.SetSynced()
 
 	_, err = backend.BlockChain().InsertChain(chain.blocks[1:])
 	return err

cmd/devp2p/internal/ethtest/types.go

@@ -99,24 +99,24 @@ func (msg Transactions) Code() int     { return 18 }
 func (msg Transactions) ReqID() uint64 { return 18 }
 
 // GetBlockHeaders represents a block header query.
-type GetBlockHeaders eth.GetBlockHeadersPacket66
+type GetBlockHeaders eth.GetBlockHeadersPacket
 
 func (msg GetBlockHeaders) Code() int     { return 19 }
 func (msg GetBlockHeaders) ReqID() uint64 { return msg.RequestId }
 
-type BlockHeaders eth.BlockHeadersPacket66
+type BlockHeaders eth.BlockHeadersPacket
 
 func (msg BlockHeaders) Code() int     { return 20 }
 func (msg BlockHeaders) ReqID() uint64 { return msg.RequestId }
 
 // GetBlockBodies represents a GetBlockBodies request
-type GetBlockBodies eth.GetBlockBodiesPacket66
+type GetBlockBodies eth.GetBlockBodiesPacket
 
 func (msg GetBlockBodies) Code() int     { return 21 }
 func (msg GetBlockBodies) ReqID() uint64 { return msg.RequestId }
 
 // BlockBodies is the network packet for block content distribution.
-type BlockBodies eth.BlockBodiesPacket66
+type BlockBodies eth.BlockBodiesPacket
 
 func (msg BlockBodies) Code() int     { return 22 }
 func (msg BlockBodies) ReqID() uint64 { return msg.RequestId }
@@ -128,7 +128,7 @@ func (msg NewBlock) Code() int     { return 23 }
 func (msg NewBlock) ReqID() uint64 { return 0 }
 
 // NewPooledTransactionHashes66 is the network packet for the tx hash propagation message.
-type NewPooledTransactionHashes66 eth.NewPooledTransactionHashesPacket66
+type NewPooledTransactionHashes66 eth.NewPooledTransactionHashesPacket67
 
 func (msg NewPooledTransactionHashes66) Code() int     { return 24 }
 func (msg NewPooledTransactionHashes66) ReqID() uint64 { return 0 }
@@ -139,12 +139,12 @@ type NewPooledTransactionHashes eth.NewPooledTransactionHashesPacket68
 func (msg NewPooledTransactionHashes) Code() int     { return 24 }
 func (msg NewPooledTransactionHashes) ReqID() uint64 { return 0 }
 
-type GetPooledTransactions eth.GetPooledTransactionsPacket66
+type GetPooledTransactions eth.GetPooledTransactionsPacket
 
 func (msg GetPooledTransactions) Code() int     { return 25 }
 func (msg GetPooledTransactions) ReqID() uint64 { return msg.RequestId }
 
-type PooledTransactions eth.PooledTransactionsPacket66
+type PooledTransactions eth.PooledTransactionsPacket
 
 func (msg PooledTransactions) Code() int     { return 26 }
 func (msg PooledTransactions) ReqID() uint64 { return msg.RequestId }
@@ -180,25 +180,25 @@ func (c *Conn) Read() Message {
 	case (Status{}).Code():
 		msg = new(Status)
 	case (GetBlockHeaders{}).Code():
-		ethMsg := new(eth.GetBlockHeadersPacket66)
+		ethMsg := new(eth.GetBlockHeadersPacket)
 		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
 			return errorf("could not rlp decode message: %v", err)
 		}
 		return (*GetBlockHeaders)(ethMsg)
 	case (BlockHeaders{}).Code():
-		ethMsg := new(eth.BlockHeadersPacket66)
+		ethMsg := new(eth.BlockHeadersPacket)
 		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
 			return errorf("could not rlp decode message: %v", err)
 		}
 		return (*BlockHeaders)(ethMsg)
 	case (GetBlockBodies{}).Code():
-		ethMsg := new(eth.GetBlockBodiesPacket66)
+		ethMsg := new(eth.GetBlockBodiesPacket)
 		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
 			return errorf("could not rlp decode message: %v", err)
 		}
 		return (*GetBlockBodies)(ethMsg)
 	case (BlockBodies{}).Code():
-		ethMsg := new(eth.BlockBodiesPacket66)
+		ethMsg := new(eth.BlockBodiesPacket)
 		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
 			return errorf("could not rlp decode message: %v", err)
 		}
@@ -217,13 +217,13 @@ func (c *Conn) Read() Message {
 		}
 		msg = new(NewPooledTransactionHashes66)
 	case (GetPooledTransactions{}.Code()):
-		ethMsg := new(eth.GetPooledTransactionsPacket66)
+		ethMsg := new(eth.GetPooledTransactionsPacket)
 		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
 			return errorf("could not rlp decode message: %v", err)
 		}
 		return (*GetPooledTransactions)(ethMsg)
 	case (PooledTransactions{}.Code()):
-		ethMsg := new(eth.PooledTransactionsPacket66)
+		ethMsg := new(eth.PooledTransactionsPacket)
 		if err := rlp.DecodeBytes(rawData, ethMsg); err != nil {
 			return errorf("could not rlp decode message: %v", err)
 		}

cmd/ethkey/run_test.go

@@ -21,8 +21,8 @@ import (
 	"os"
 	"testing"
 
-	"github.com/docker/docker/pkg/reexec"
 	"github.com/ethereum/go-ethereum/internal/cmdtest"
+	"github.com/ethereum/go-ethereum/internal/reexec"
 )
 
 type testEthkey struct {

cmd/evm/blockrunner.go

@@ -21,30 +21,35 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"regexp"
+	"sort"
 
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/tests"
 	"github.com/urfave/cli/v2"
 )
 
+var RunFlag = &cli.StringFlag{
+	Name:  "run",
+	Value: ".*",
+	Usage: "Run only those tests matching the regular expression.",
+}
+
 var blockTestCommand = &cli.Command{
 	Action:    blockTestCmd,
 	Name:      "blocktest",
 	Usage:     "executes the given blockchain tests",
 	ArgsUsage: "<file>",
+	Flags:     []cli.Flag{RunFlag},
 }
 
 func blockTestCmd(ctx *cli.Context) error {
 	if len(ctx.Args().First()) == 0 {
 		return errors.New("path-to-test argument required")
 	}
-	// Configure the go-ethereum logger
+
-	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
-	glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name)))
-	log.Root().SetHandler(glogger)
 	var tracer vm.EVMLogger
 	// Configure the EVM logger
 	if ctx.Bool(MachineFlag.Name) {
@@ -64,9 +69,24 @@ func blockTestCmd(ctx *cli.Context) error {
 	if err = json.Unmarshal(src, &tests); err != nil {
 		return err
 	}
-	for i, test := range tests {
+	re, err := regexp.Compile(ctx.String(RunFlag.Name))
+	if err != nil {
+		return fmt.Errorf("invalid regex -%s: %v", RunFlag.Name, err)
+	}
+
+	// Run them in order
+	var keys []string
+	for key := range tests {
+		keys = append(keys, key)
+	}
+	sort.Strings(keys)
+	for _, name := range keys {
+		if !re.MatchString(name) {
+			continue
+		}
+		test := tests[name]
 		if err := test.Run(false, rawdb.HashScheme, tracer); err != nil {
-			return fmt.Errorf("test %v: %w", i, err)
+			return fmt.Errorf("test %v: %w", name, err)
 		}
 	}
 	return nil

cmd/evm/internal/t8ntool/block.go

@@ -37,33 +37,38 @@ import (
 
 //go:generate go run github.com/fjl/gencodec -type header -field-override headerMarshaling -out gen_header.go
 type header struct {
-	ParentHash      common.Hash       `json:"parentHash"`
+	ParentHash            common.Hash       `json:"parentHash"`
-	OmmerHash       *common.Hash      `json:"sha3Uncles"`
+	OmmerHash             *common.Hash      `json:"sha3Uncles"`
-	Coinbase        *common.Address   `json:"miner"`
+	Coinbase              *common.Address   `json:"miner"`
-	Root            common.Hash       `json:"stateRoot"        gencodec:"required"`
+	Root                  common.Hash       `json:"stateRoot"        gencodec:"required"`
-	TxHash          *common.Hash      `json:"transactionsRoot"`
+	TxHash                *common.Hash      `json:"transactionsRoot"`
-	ReceiptHash     *common.Hash      `json:"receiptsRoot"`
+	ReceiptHash           *common.Hash      `json:"receiptsRoot"`
-	Bloom           types.Bloom       `json:"logsBloom"`
+	Bloom                 types.Bloom       `json:"logsBloom"`
-	Difficulty      *big.Int          `json:"difficulty"`
+	Difficulty            *big.Int          `json:"difficulty"`
-	Number          *big.Int          `json:"number"           gencodec:"required"`
+	Number                *big.Int          `json:"number"           gencodec:"required"`
-	GasLimit        uint64            `json:"gasLimit"         gencodec:"required"`
+	GasLimit              uint64            `json:"gasLimit"         gencodec:"required"`
-	GasUsed         uint64            `json:"gasUsed"`
+	GasUsed               uint64            `json:"gasUsed"`
-	Time            uint64            `json:"timestamp"        gencodec:"required"`
+	Time                  uint64            `json:"timestamp"        gencodec:"required"`
-	Extra           []byte            `json:"extraData"`
+	Extra                 []byte            `json:"extraData"`
-	MixDigest       common.Hash       `json:"mixHash"`
+	MixDigest             common.Hash       `json:"mixHash"`
-	Nonce           *types.BlockNonce `json:"nonce"`
+	Nonce                 *types.BlockNonce `json:"nonce"`
-	BaseFee         *big.Int          `json:"baseFeePerGas" rlp:"optional"`
+	BaseFee               *big.Int          `json:"baseFeePerGas" rlp:"optional"`
-	WithdrawalsHash *common.Hash      `json:"withdrawalsRoot" rlp:"optional"`
+	WithdrawalsHash       *common.Hash      `json:"withdrawalsRoot" rlp:"optional"`
+	BlobGasUsed           *uint64           `json:"blobGasUsed"   rlp:"optional"`
+	ExcessBlobGas         *uint64           `json:"excessBlobGas"   rlp:"optional"`
+	ParentBeaconBlockRoot *common.Hash      `json:"parentBeaconBlockRoot" rlp:"optional"`
 }
 
 type headerMarshaling struct {
-	Difficulty *math.HexOrDecimal256
+	Difficulty    *math.HexOrDecimal256
-	Number     *math.HexOrDecimal256
+	Number        *math.HexOrDecimal256
-	GasLimit   math.HexOrDecimal64
+	GasLimit      math.HexOrDecimal64
-	GasUsed    math.HexOrDecimal64
+	GasUsed       math.HexOrDecimal64
-	Time       math.HexOrDecimal64
+	Time          math.HexOrDecimal64
-	Extra      hexutil.Bytes
+	Extra         hexutil.Bytes
-	BaseFee    *math.HexOrDecimal256
+	BaseFee       *math.HexOrDecimal256
+	BlobGasUsed   *math.HexOrDecimal64
+	ExcessBlobGas *math.HexOrDecimal64
 }
 
 type bbInput struct {
@@ -113,22 +118,25 @@ func (c *cliqueInput) UnmarshalJSON(input []byte) error {
 // ToBlock converts i into a *types.Block
 func (i *bbInput) ToBlock() *types.Block {
 	header := &types.Header{
-		ParentHash:      i.Header.ParentHash,
+		ParentHash:       i.Header.ParentHash,
-		UncleHash:       types.EmptyUncleHash,
+		UncleHash:        types.EmptyUncleHash,
-		Coinbase:        common.Address{},
+		Coinbase:         common.Address{},
-		Root:            i.Header.Root,
+		Root:             i.Header.Root,
-		TxHash:          types.EmptyTxsHash,
+		TxHash:           types.EmptyTxsHash,
-		ReceiptHash:     types.EmptyReceiptsHash,
+		ReceiptHash:      types.EmptyReceiptsHash,
-		Bloom:           i.Header.Bloom,
+		Bloom:            i.Header.Bloom,
-		Difficulty:      common.Big0,
+		Difficulty:       common.Big0,
-		Number:          i.Header.Number,
+		Number:           i.Header.Number,
-		GasLimit:        i.Header.GasLimit,
+		GasLimit:         i.Header.GasLimit,
-		GasUsed:         i.Header.GasUsed,
+		GasUsed:          i.Header.GasUsed,
-		Time:            i.Header.Time,
+		Time:             i.Header.Time,
-		Extra:           i.Header.Extra,
+		Extra:            i.Header.Extra,
-		MixDigest:       i.Header.MixDigest,
+		MixDigest:        i.Header.MixDigest,
-		BaseFee:         i.Header.BaseFee,
+		BaseFee:          i.Header.BaseFee,
-		WithdrawalsHash: i.Header.WithdrawalsHash,
+		WithdrawalsHash:  i.Header.WithdrawalsHash,
+		BlobGasUsed:      i.Header.BlobGasUsed,
+		ExcessBlobGas:    i.Header.ExcessBlobGas,
+		ParentBeaconRoot: i.Header.ParentBeaconBlockRoot,
 	}
 
 	// Fill optional values.
@@ -150,7 +158,7 @@ func (i *bbInput) ToBlock() *types.Block {
 	if i.Header.Nonce != nil {
 		header.Nonce = *i.Header.Nonce
 	}
-	if header.Difficulty != nil {
+	if i.Header.Difficulty != nil {
 		header.Difficulty = i.Header.Difficulty
 	}
 	return types.NewBlockWithHeader(header).WithBody(i.Txs, i.Ommers).WithWithdrawals(i.Withdrawals)

cmd/evm/internal/t8ntool/execution.go

@@ -59,7 +59,7 @@ type ExecutionResult struct {
 	BaseFee              *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"`
 	WithdrawalsRoot      *common.Hash          `json:"withdrawalsRoot,omitempty"`
 	CurrentExcessBlobGas *math.HexOrDecimal64  `json:"currentExcessBlobGas,omitempty"`
-	CurrentBlobGasUsed   *math.HexOrDecimal64  `json:"currentBlobGasUsed,omitempty"`
+	CurrentBlobGasUsed   *math.HexOrDecimal64  `json:"blobGasUsed,omitempty"`
 }
 
 type ommer struct {
@@ -85,7 +85,7 @@ type stEnv struct {
 	Withdrawals           []*types.Withdrawal                 `json:"withdrawals,omitempty"`
 	BaseFee               *big.Int                            `json:"currentBaseFee,omitempty"`
 	ParentUncleHash       common.Hash                         `json:"parentUncleHash"`
-	ExcessBlobGas         *uint64                             `json:"excessBlobGas,omitempty"`
+	ExcessBlobGas         *uint64                             `json:"currentExcessBlobGas,omitempty"`
 	ParentExcessBlobGas   *uint64                             `json:"parentExcessBlobGas,omitempty"`
 	ParentBlobGasUsed     *uint64                             `json:"parentBlobGasUsed,omitempty"`
 	ParentBeaconBlockRoot *common.Hash                        `json:"parentBeaconBlockRoot"`
@@ -116,8 +116,8 @@ type rejectedTx struct {
 
 // Apply applies a set of transactions to a pre-state
 func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
-	txs types.Transactions, miningReward int64,
+	txIt txIterator, miningReward int64,
-	getTracerFn func(txIndex int, txHash common.Hash) (tracer vm.EVMLogger, err error)) (*state.StateDB, *ExecutionResult, error) {
+	getTracerFn func(txIndex int, txHash common.Hash) (tracer vm.EVMLogger, err error)) (*state.StateDB, *ExecutionResult, []byte, error) {
 	// Capture errors for BLOCKHASH operation, if we haven't been supplied the
 	// required blockhashes
 	var hashError error
@@ -163,17 +163,19 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		rnd := common.BigToHash(pre.Env.Random)
 		vmContext.Random = &rnd
 	}
-	// If excessBlobGas is defined, add it to the vmContext.
+	// Calculate the BlobBaseFee
+	var excessBlobGas uint64
 	if pre.Env.ExcessBlobGas != nil {
-		vmContext.ExcessBlobGas = pre.Env.ExcessBlobGas
+		excessBlobGas := *pre.Env.ExcessBlobGas
+		vmContext.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas)
 	} else {
 		// If it is not explicitly defined, but we have the parent values, we try
 		// to calculate it ourselves.
 		parentExcessBlobGas := pre.Env.ParentExcessBlobGas
 		parentBlobGasUsed := pre.Env.ParentBlobGasUsed
 		if parentExcessBlobGas != nil && parentBlobGasUsed != nil {
-			excessBlobGas := eip4844.CalcExcessBlobGas(*parentExcessBlobGas, *parentBlobGasUsed)
+			excessBlobGas = eip4844.CalcExcessBlobGas(*parentExcessBlobGas, *parentBlobGasUsed)
-			vmContext.ExcessBlobGas = &excessBlobGas
+			vmContext.BlobBaseFee = eip4844.CalcBlobFee(excessBlobGas)
 		}
 	}
 	// If DAO is supported/enabled, we need to handle it here. In geth 'proper', it's
@@ -188,8 +190,15 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		core.ProcessBeaconBlockRoot(*beaconRoot, evm, statedb)
 	}
 	var blobGasUsed uint64
-	for i, tx := range txs {
+
-		if tx.Type() == types.BlobTxType && vmContext.ExcessBlobGas == nil {
+	for i := 0; txIt.Next(); i++ {
+		tx, err := txIt.Tx()
+		if err != nil {
+			log.Warn("rejected tx", "index", i, "error", err)
+			rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()})
+			continue
+		}
+		if tx.Type() == types.BlobTxType && vmContext.BlobBaseFee == nil {
 			errMsg := "blob tx used but field env.ExcessBlobGas missing"
 			log.Warn("rejected tx", "index", i, "hash", tx.Hash(), "error", errMsg)
 			rejectedTxs = append(rejectedTxs, &rejectedTx{i, errMsg})
@@ -201,9 +210,19 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 			rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()})
 			continue
 		}
+		if tx.Type() == types.BlobTxType {
+			txBlobGas := uint64(params.BlobTxBlobGasPerBlob * len(tx.BlobHashes()))
+			if used, max := blobGasUsed+txBlobGas, uint64(params.MaxBlobGasPerBlock); used > max {
+				err := fmt.Errorf("blob gas (%d) would exceed maximum allowance %d", used, max)
+				log.Warn("rejected tx", "index", i, "err", err)
+				rejectedTxs = append(rejectedTxs, &rejectedTx{i, err.Error()})
+				continue
+			}
+			blobGasUsed += txBlobGas
+		}
 		tracer, err := getTracerFn(txIndex, tx.Hash())
 		if err != nil {
-			return nil, nil, err
+			return nil, nil, nil, err
 		}
 		vmConfig.Tracer = tracer
 		statedb.SetTxContext(tx.Hash(), txIndex)
@@ -226,12 +245,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 			tracer.CaptureTxEnd(nil, err)
 			continue
 		}
-		if tx.Type() == types.BlobTxType {
-			blobGasUsed += params.BlobTxBlobGasPerBlob
-		}
 		includedTxs = append(includedTxs, tx)
 		if hashError != nil {
-			return nil, nil, NewError(ErrorMissingBlockhash, hashError)
+			return nil, nil, nil, NewError(ErrorMissingBlockhash, hashError)
 		}
 		gasUsed += msgResult.UsedGas
 
@@ -307,7 +323,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 	// Commit block
 	root, err := statedb.Commit(vmContext.BlockNumber.Uint64(), chainConfig.IsEIP158(vmContext.BlockNumber))
 	if err != nil {
-		return nil, nil, NewError(ErrorEVM, fmt.Errorf("could not commit state: %v", err))
+		return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not commit state: %v", err))
 	}
 	execRs := &ExecutionResult{
 		StateRoot:   root,
@@ -325,17 +341,18 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		h := types.DeriveSha(types.Withdrawals(pre.Env.Withdrawals), trie.NewStackTrie(nil))
 		execRs.WithdrawalsRoot = &h
 	}
-	if vmContext.ExcessBlobGas != nil {
+	if vmContext.BlobBaseFee != nil {
-		execRs.CurrentExcessBlobGas = (*math.HexOrDecimal64)(vmContext.ExcessBlobGas)
+		execRs.CurrentExcessBlobGas = (*math.HexOrDecimal64)(&excessBlobGas)
 		execRs.CurrentBlobGasUsed = (*math.HexOrDecimal64)(&blobGasUsed)
 	}
 	// Re-create statedb instance with new root upon the updated database
 	// for accessing latest states.
 	statedb, err = state.New(root, statedb.Database(), nil)
 	if err != nil {
-		return nil, nil, NewError(ErrorEVM, fmt.Errorf("could not reopen state: %v", err))
+		return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not reopen state: %v", err))
 	}
-	return statedb, execRs, nil
+	body, _ := rlp.EncodeToBytes(includedTxs)
+	return statedb, execRs, body, nil
 }
 
 func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB {

cmd/evm/internal/t8ntool/gen_header.go

@@ -18,23 +18,26 @@ var _ = (*headerMarshaling)(nil)
 // MarshalJSON marshals as JSON.
 func (h header) MarshalJSON() ([]byte, error) {
 	type header struct {
-		ParentHash      common.Hash           `json:"parentHash"`
+		ParentHash            common.Hash           `json:"parentHash"`
-		OmmerHash       *common.Hash          `json:"sha3Uncles"`
+		OmmerHash             *common.Hash          `json:"sha3Uncles"`
-		Coinbase        *common.Address       `json:"miner"`
+		Coinbase              *common.Address       `json:"miner"`
-		Root            common.Hash           `json:"stateRoot"        gencodec:"required"`
+		Root                  common.Hash           `json:"stateRoot"        gencodec:"required"`
-		TxHash          *common.Hash          `json:"transactionsRoot"`
+		TxHash                *common.Hash          `json:"transactionsRoot"`
-		ReceiptHash     *common.Hash          `json:"receiptsRoot"`
+		ReceiptHash           *common.Hash          `json:"receiptsRoot"`
-		Bloom           types.Bloom           `json:"logsBloom"`
+		Bloom                 types.Bloom           `json:"logsBloom"`
-		Difficulty      *math.HexOrDecimal256 `json:"difficulty"`
+		Difficulty            *math.HexOrDecimal256 `json:"difficulty"`
-		Number          *math.HexOrDecimal256 `json:"number"           gencodec:"required"`
+		Number                *math.HexOrDecimal256 `json:"number"           gencodec:"required"`
-		GasLimit        math.HexOrDecimal64   `json:"gasLimit"         gencodec:"required"`
+		GasLimit              math.HexOrDecimal64   `json:"gasLimit"         gencodec:"required"`
-		GasUsed         math.HexOrDecimal64   `json:"gasUsed"`
+		GasUsed               math.HexOrDecimal64   `json:"gasUsed"`
-		Time            math.HexOrDecimal64   `json:"timestamp"        gencodec:"required"`
+		Time                  math.HexOrDecimal64   `json:"timestamp"        gencodec:"required"`
-		Extra           hexutil.Bytes         `json:"extraData"`
+		Extra                 hexutil.Bytes         `json:"extraData"`
-		MixDigest       common.Hash           `json:"mixHash"`
+		MixDigest             common.Hash           `json:"mixHash"`
-		Nonce           *types.BlockNonce     `json:"nonce"`
+		Nonce                 *types.BlockNonce     `json:"nonce"`
-		BaseFee         *math.HexOrDecimal256 `json:"baseFeePerGas" rlp:"optional"`
+		BaseFee               *math.HexOrDecimal256 `json:"baseFeePerGas" rlp:"optional"`
-		WithdrawalsHash *common.Hash          `json:"withdrawalsRoot" rlp:"optional"`
+		WithdrawalsHash       *common.Hash          `json:"withdrawalsRoot" rlp:"optional"`
+		BlobGasUsed           *math.HexOrDecimal64  `json:"blobGasUsed"   rlp:"optional"`
+		ExcessBlobGas         *math.HexOrDecimal64  `json:"excessBlobGas"   rlp:"optional"`
+		ParentBeaconBlockRoot *common.Hash          `json:"parentBeaconBlockRoot" rlp:"optional"`
 	}
 	var enc header
 	enc.ParentHash = h.ParentHash
@@ -54,29 +57,35 @@ func (h header) MarshalJSON() ([]byte, error) {
 	enc.Nonce = h.Nonce
 	enc.BaseFee = (*math.HexOrDecimal256)(h.BaseFee)
 	enc.WithdrawalsHash = h.WithdrawalsHash
+	enc.BlobGasUsed = (*math.HexOrDecimal64)(h.BlobGasUsed)
+	enc.ExcessBlobGas = (*math.HexOrDecimal64)(h.ExcessBlobGas)
+	enc.ParentBeaconBlockRoot = h.ParentBeaconBlockRoot
 	return json.Marshal(&enc)
 }
 
 // UnmarshalJSON unmarshals from JSON.
 func (h *header) UnmarshalJSON(input []byte) error {
 	type header struct {
-		ParentHash      *common.Hash          `json:"parentHash"`
+		ParentHash            *common.Hash          `json:"parentHash"`
-		OmmerHash       *common.Hash          `json:"sha3Uncles"`
+		OmmerHash             *common.Hash          `json:"sha3Uncles"`
-		Coinbase        *common.Address       `json:"miner"`
+		Coinbase              *common.Address       `json:"miner"`
-		Root            *common.Hash          `json:"stateRoot"        gencodec:"required"`
+		Root                  *common.Hash          `json:"stateRoot"        gencodec:"required"`
-		TxHash          *common.Hash          `json:"transactionsRoot"`
+		TxHash                *common.Hash          `json:"transactionsRoot"`
-		ReceiptHash     *common.Hash          `json:"receiptsRoot"`
+		ReceiptHash           *common.Hash          `json:"receiptsRoot"`
-		Bloom           *types.Bloom          `json:"logsBloom"`
+		Bloom                 *types.Bloom          `json:"logsBloom"`
-		Difficulty      *math.HexOrDecimal256 `json:"difficulty"`
+		Difficulty            *math.HexOrDecimal256 `json:"difficulty"`
-		Number          *math.HexOrDecimal256 `json:"number"           gencodec:"required"`
+		Number                *math.HexOrDecimal256 `json:"number"           gencodec:"required"`
-		GasLimit        *math.HexOrDecimal64  `json:"gasLimit"         gencodec:"required"`
+		GasLimit              *math.HexOrDecimal64  `json:"gasLimit"         gencodec:"required"`
-		GasUsed         *math.HexOrDecimal64  `json:"gasUsed"`
+		GasUsed               *math.HexOrDecimal64  `json:"gasUsed"`
-		Time            *math.HexOrDecimal64  `json:"timestamp"        gencodec:"required"`
+		Time                  *math.HexOrDecimal64  `json:"timestamp"        gencodec:"required"`
-		Extra           *hexutil.Bytes        `json:"extraData"`
+		Extra                 *hexutil.Bytes        `json:"extraData"`
-		MixDigest       *common.Hash          `json:"mixHash"`
+		MixDigest             *common.Hash          `json:"mixHash"`
-		Nonce           *types.BlockNonce     `json:"nonce"`
+		Nonce                 *types.BlockNonce     `json:"nonce"`
-		BaseFee         *math.HexOrDecimal256 `json:"baseFeePerGas" rlp:"optional"`
+		BaseFee               *math.HexOrDecimal256 `json:"baseFeePerGas" rlp:"optional"`
-		WithdrawalsHash *common.Hash          `json:"withdrawalsRoot" rlp:"optional"`
+		WithdrawalsHash       *common.Hash          `json:"withdrawalsRoot" rlp:"optional"`
+		BlobGasUsed           *math.HexOrDecimal64  `json:"blobGasUsed"   rlp:"optional"`
+		ExcessBlobGas         *math.HexOrDecimal64  `json:"excessBlobGas"   rlp:"optional"`
+		ParentBeaconBlockRoot *common.Hash          `json:"parentBeaconBlockRoot" rlp:"optional"`
 	}
 	var dec header
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -137,5 +146,14 @@ func (h *header) UnmarshalJSON(input []byte) error {
 	if dec.WithdrawalsHash != nil {
 		h.WithdrawalsHash = dec.WithdrawalsHash
 	}
+	if dec.BlobGasUsed != nil {
+		h.BlobGasUsed = (*uint64)(dec.BlobGasUsed)
+	}
+	if dec.ExcessBlobGas != nil {
+		h.ExcessBlobGas = (*uint64)(dec.ExcessBlobGas)
+	}
+	if dec.ParentBeaconBlockRoot != nil {
+		h.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot
+	}
 	return nil
 }

cmd/evm/internal/t8ntool/gen_stenv.go

@@ -33,7 +33,7 @@ func (s stEnv) MarshalJSON() ([]byte, error) {
 		Withdrawals           []*types.Withdrawal                 `json:"withdrawals,omitempty"`
 		BaseFee               *math.HexOrDecimal256               `json:"currentBaseFee,omitempty"`
 		ParentUncleHash       common.Hash                         `json:"parentUncleHash"`
-		ExcessBlobGas         *math.HexOrDecimal64                `json:"excessBlobGas,omitempty"`
+		ExcessBlobGas         *math.HexOrDecimal64                `json:"currentExcessBlobGas,omitempty"`
 		ParentExcessBlobGas   *math.HexOrDecimal64                `json:"parentExcessBlobGas,omitempty"`
 		ParentBlobGasUsed     *math.HexOrDecimal64                `json:"parentBlobGasUsed,omitempty"`
 		ParentBeaconBlockRoot *common.Hash                        `json:"parentBeaconBlockRoot"`
@@ -81,7 +81,7 @@ func (s *stEnv) UnmarshalJSON(input []byte) error {
 		Withdrawals           []*types.Withdrawal                 `json:"withdrawals,omitempty"`
 		BaseFee               *math.HexOrDecimal256               `json:"currentBaseFee,omitempty"`
 		ParentUncleHash       *common.Hash                        `json:"parentUncleHash"`
-		ExcessBlobGas         *math.HexOrDecimal64                `json:"excessBlobGas,omitempty"`
+		ExcessBlobGas         *math.HexOrDecimal64                `json:"currentExcessBlobGas,omitempty"`
 		ParentExcessBlobGas   *math.HexOrDecimal64                `json:"parentExcessBlobGas,omitempty"`
 		ParentBlobGasUsed     *math.HexOrDecimal64                `json:"parentBlobGasUsed,omitempty"`
 		ParentBeaconBlockRoot *common.Hash                        `json:"parentBeaconBlockRoot"`

cmd/evm/internal/t8ntool/transition.go

@@ -17,14 +17,12 @@
 package t8ntool
 
 import (
-	"crypto/ecdsa"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"math/big"
 	"os"
 	"path"
-	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -33,11 +31,9 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/tests"
 	"github.com/urfave/cli/v2"
 )
@@ -147,7 +143,7 @@ func Transition(ctx *cli.Context) error {
 	// Check if anything needs to be read from stdin
 	var (
 		prestate Prestate
-		txs      types.Transactions // txs to apply
+		txIt     txIterator // txs to apply
 		allocStr = ctx.String(InputAllocFlag.Name)
 
 		envStr    = ctx.String(InputEnvFlag.Name)
@@ -192,7 +188,7 @@ func Transition(ctx *cli.Context) error {
 	// Set the chain id
 	chainConfig.ChainID = big.NewInt(ctx.Int64(ChainIDFlag.Name))
 
-	if txs, err = loadTransactions(txStr, inputData, prestate.Env, chainConfig); err != nil {
+	if txIt, err = loadTransactions(txStr, inputData, prestate.Env, chainConfig); err != nil {
 		return err
 	}
 	if err := applyLondonChecks(&prestate.Env, chainConfig); err != nil {
@@ -208,136 +204,16 @@ func Transition(ctx *cli.Context) error {
 		return err
 	}
 	// Run the test and aggregate the result
-	s, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer)
+	s, result, body, err := prestate.Apply(vmConfig, chainConfig, txIt, ctx.Int64(RewardFlag.Name), getTracer)
 	if err != nil {
 		return err
 	}
-	body, _ := rlp.EncodeToBytes(txs)
 	// Dump the excution result
 	collector := make(Alloc)
 	s.DumpToCollector(collector, nil)
 	return dispatchOutput(ctx, baseDir, result, collector, body)
 }
 
-// txWithKey is a helper-struct, to allow us to use the types.Transaction along with
-// a `secretKey`-field, for input
-type txWithKey struct {
-	key       *ecdsa.PrivateKey
-	tx        *types.Transaction
-	protected bool
-}
-
-func (t *txWithKey) UnmarshalJSON(input []byte) error {
-	// Read the metadata, if present
-	type txMetadata struct {
-		Key       *common.Hash `json:"secretKey"`
-		Protected *bool        `json:"protected"`
-	}
-	var data txMetadata
-	if err := json.Unmarshal(input, &data); err != nil {
-		return err
-	}
-	if data.Key != nil {
-		k := data.Key.Hex()[2:]
-		if ecdsaKey, err := crypto.HexToECDSA(k); err != nil {
-			return err
-		} else {
-			t.key = ecdsaKey
-		}
-	}
-	if data.Protected != nil {
-		t.protected = *data.Protected
-	} else {
-		t.protected = true
-	}
-	// Now, read the transaction itself
-	var tx types.Transaction
-	if err := json.Unmarshal(input, &tx); err != nil {
-		return err
-	}
-	t.tx = &tx
-	return nil
-}
-
-// signUnsignedTransactions converts the input txs to canonical transactions.
-//
-// The transactions can have two forms, either
-//  1. unsigned or
-//  2. signed
-//
-// For (1), r, s, v, need so be zero, and the `secretKey` needs to be set.
-// If so, we sign it here and now, with the given `secretKey`
-// If the condition above is not met, then it's considered a signed transaction.
-//
-// To manage this, we read the transactions twice, first trying to read the secretKeys,
-// and secondly to read them with the standard tx json format
-func signUnsignedTransactions(txs []*txWithKey, signer types.Signer) (types.Transactions, error) {
-	var signedTxs []*types.Transaction
-	for i, tx := range txs {
-		var (
-			v, r, s = tx.tx.RawSignatureValues()
-			signed  *types.Transaction
-			err     error
-		)
-		if tx.key == nil || v.BitLen()+r.BitLen()+s.BitLen() != 0 {
-			// Already signed
-			signedTxs = append(signedTxs, tx.tx)
-			continue
-		}
-		// This transaction needs to be signed
-		if tx.protected {
-			signed, err = types.SignTx(tx.tx, signer, tx.key)
-		} else {
-			signed, err = types.SignTx(tx.tx, types.FrontierSigner{}, tx.key)
-		}
-		if err != nil {
-			return nil, NewError(ErrorJson, fmt.Errorf("tx %d: failed to sign tx: %v", i, err))
-		}
-		signedTxs = append(signedTxs, signed)
-	}
-	return signedTxs, nil
-}
-
-func loadTransactions(txStr string, inputData *input, env stEnv, chainConfig *params.ChainConfig) (types.Transactions, error) {
-	var txsWithKeys []*txWithKey
-	var signed types.Transactions
-	if txStr != stdinSelector {
-		data, err := os.ReadFile(txStr)
-		if err != nil {
-			return nil, NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err))
-		}
-		if strings.HasSuffix(txStr, ".rlp") { // A file containing an rlp list
-			var body hexutil.Bytes
-			if err := json.Unmarshal(data, &body); err != nil {
-				return nil, err
-			}
-			// Already signed transactions
-			if err := rlp.DecodeBytes(body, &signed); err != nil {
-				return nil, err
-			}
-			return signed, nil
-		}
-		if err := json.Unmarshal(data, &txsWithKeys); err != nil {
-			return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err))
-		}
-	} else {
-		if len(inputData.TxRlp) > 0 {
-			// Decode the body of already signed transactions
-			body := common.FromHex(inputData.TxRlp)
-			// Already signed transactions
-			if err := rlp.DecodeBytes(body, &signed); err != nil {
-				return nil, err
-			}
-			return signed, nil
-		}
-		// JSON encoded transactions
-		txsWithKeys = inputData.Txs
-	}
-	// We may have to sign the transactions.
-	signer := types.MakeSigner(chainConfig, big.NewInt(int64(env.Number)), env.Timestamp)
-	return signUnsignedTransactions(txsWithKeys, signer)
-}
-
 func applyLondonChecks(env *stEnv, chainConfig *params.ChainConfig) error {
 	if !chainConfig.IsLondon(big.NewInt(int64(env.Number))) {
 		return nil

cmd/evm/internal/t8ntool/tx_iterator.go

@@ -0,0 +1,194 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package t8ntool
+
+import (
+	"bytes"
+	"crypto/ecdsa"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// txWithKey is a helper-struct, to allow us to use the types.Transaction along with
+// a `secretKey`-field, for input
+type txWithKey struct {
+	key       *ecdsa.PrivateKey
+	tx        *types.Transaction
+	protected bool
+}
+
+func (t *txWithKey) UnmarshalJSON(input []byte) error {
+	// Read the metadata, if present
+	type txMetadata struct {
+		Key       *common.Hash `json:"secretKey"`
+		Protected *bool        `json:"protected"`
+	}
+	var data txMetadata
+	if err := json.Unmarshal(input, &data); err != nil {
+		return err
+	}
+	if data.Key != nil {
+		k := data.Key.Hex()[2:]
+		if ecdsaKey, err := crypto.HexToECDSA(k); err != nil {
+			return err
+		} else {
+			t.key = ecdsaKey
+		}
+	}
+	if data.Protected != nil {
+		t.protected = *data.Protected
+	} else {
+		t.protected = true
+	}
+	// Now, read the transaction itself
+	var tx types.Transaction
+	if err := json.Unmarshal(input, &tx); err != nil {
+		return err
+	}
+	t.tx = &tx
+	return nil
+}
+
+// signUnsignedTransactions converts the input txs to canonical transactions.
+//
+// The transactions can have two forms, either
+//  1. unsigned or
+//  2. signed
+//
+// For (1), r, s, v, need so be zero, and the `secretKey` needs to be set.
+// If so, we sign it here and now, with the given `secretKey`
+// If the condition above is not met, then it's considered a signed transaction.
+//
+// To manage this, we read the transactions twice, first trying to read the secretKeys,
+// and secondly to read them with the standard tx json format
+func signUnsignedTransactions(txs []*txWithKey, signer types.Signer) (types.Transactions, error) {
+	var signedTxs []*types.Transaction
+	for i, tx := range txs {
+		var (
+			v, r, s = tx.tx.RawSignatureValues()
+			signed  *types.Transaction
+			err     error
+		)
+		if tx.key == nil || v.BitLen()+r.BitLen()+s.BitLen() != 0 {
+			// Already signed
+			signedTxs = append(signedTxs, tx.tx)
+			continue
+		}
+		// This transaction needs to be signed
+		if tx.protected {
+			signed, err = types.SignTx(tx.tx, signer, tx.key)
+		} else {
+			signed, err = types.SignTx(tx.tx, types.FrontierSigner{}, tx.key)
+		}
+		if err != nil {
+			return nil, NewError(ErrorJson, fmt.Errorf("tx %d: failed to sign tx: %v", i, err))
+		}
+		signedTxs = append(signedTxs, signed)
+	}
+	return signedTxs, nil
+}
+
+func loadTransactions(txStr string, inputData *input, env stEnv, chainConfig *params.ChainConfig) (txIterator, error) {
+	var txsWithKeys []*txWithKey
+	if txStr != stdinSelector {
+		data, err := os.ReadFile(txStr)
+		if err != nil {
+			return nil, NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err))
+		}
+		if strings.HasSuffix(txStr, ".rlp") { // A file containing an rlp list
+			var body hexutil.Bytes
+			if err := json.Unmarshal(data, &body); err != nil {
+				return nil, err
+			}
+			return newRlpTxIterator(body), nil
+		}
+		if err := json.Unmarshal(data, &txsWithKeys); err != nil {
+			return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err))
+		}
+	} else {
+		if len(inputData.TxRlp) > 0 {
+			// Decode the body of already signed transactions
+			return newRlpTxIterator(common.FromHex(inputData.TxRlp)), nil
+		}
+		// JSON encoded transactions
+		txsWithKeys = inputData.Txs
+	}
+	// We may have to sign the transactions.
+	signer := types.LatestSignerForChainID(chainConfig.ChainID)
+	txs, err := signUnsignedTransactions(txsWithKeys, signer)
+	return newSliceTxIterator(txs), err
+}
+
+type txIterator interface {
+	// Next returns true until EOF
+	Next() bool
+	// Tx returns the next transaction, OR an error.
+	Tx() (*types.Transaction, error)
+}
+
+type sliceTxIterator struct {
+	idx int
+	txs []*types.Transaction
+}
+
+func newSliceTxIterator(transactions types.Transactions) txIterator {
+	return &sliceTxIterator{0, transactions}
+}
+
+func (ait *sliceTxIterator) Next() bool {
+	return ait.idx < len(ait.txs)
+}
+
+func (ait *sliceTxIterator) Tx() (*types.Transaction, error) {
+	if ait.idx < len(ait.txs) {
+		ait.idx++
+		return ait.txs[ait.idx-1], nil
+	}
+	return nil, io.EOF
+}
+
+type rlpTxIterator struct {
+	in *rlp.Stream
+}
+
+func newRlpTxIterator(rlpData []byte) txIterator {
+	in := rlp.NewStream(bytes.NewBuffer(rlpData), 1024*1024)
+	in.List()
+	return &rlpTxIterator{in}
+}
+
+func (it *rlpTxIterator) Next() bool {
+	return it.in.MoreDataInList()
+}
+
+func (it *rlpTxIterator) Tx() (*types.Transaction, error) {
+	var a types.Transaction
+	if err := it.in.Decode(&a); err != nil {
+		return nil, err
+	}
+	return &a, nil
+}

cmd/evm/main.go

@@ -23,107 +23,116 @@ import (
 	"os"
 
 	"github.com/ethereum/go-ethereum/cmd/evm/internal/t8ntool"
+	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/urfave/cli/v2"
 )
 
 var (
 	DebugFlag = &cli.BoolFlag{
-		Name:  "debug",
+		Name:     "debug",
-		Usage: "output full trace logs",
+		Usage:    "output full trace logs",
-	}
+		Category: flags.VMCategory,
-	MemProfileFlag = &cli.StringFlag{
-		Name:  "memprofile",
-		Usage: "creates a memory profile at the given path",
-	}
-	CPUProfileFlag = &cli.StringFlag{
-		Name:  "cpuprofile",
-		Usage: "creates a CPU profile at the given path",
 	}
 	StatDumpFlag = &cli.BoolFlag{
-		Name:  "statdump",
+		Name:     "statdump",
-		Usage: "displays stack and heap memory information",
+		Usage:    "displays stack and heap memory information",
+		Category: flags.VMCategory,
 	}
 	CodeFlag = &cli.StringFlag{
-		Name:  "code",
+		Name:     "code",
-		Usage: "EVM code",
+		Usage:    "EVM code",
+		Category: flags.VMCategory,
 	}
 	CodeFileFlag = &cli.StringFlag{
-		Name:  "codefile",
+		Name:     "codefile",
-		Usage: "File containing EVM code. If '-' is specified, code is read from stdin ",
+		Usage:    "File containing EVM code. If '-' is specified, code is read from stdin ",
+		Category: flags.VMCategory,
 	}
 	GasFlag = &cli.Uint64Flag{
-		Name:  "gas",
+		Name:     "gas",
-		Usage: "gas limit for the evm",
+		Usage:    "gas limit for the evm",
-		Value: 10000000000,
+		Value:    10000000000,
+		Category: flags.VMCategory,
 	}
 	PriceFlag = &flags.BigFlag{
-		Name:  "price",
+		Name:     "price",
-		Usage: "price set for the evm",
+		Usage:    "price set for the evm",
-		Value: new(big.Int),
+		Value:    new(big.Int),
+		Category: flags.VMCategory,
 	}
 	ValueFlag = &flags.BigFlag{
-		Name:  "value",
+		Name:     "value",
-		Usage: "value set for the evm",
+		Usage:    "value set for the evm",
-		Value: new(big.Int),
+		Value:    new(big.Int),
+		Category: flags.VMCategory,
 	}
 	DumpFlag = &cli.BoolFlag{
-		Name:  "dump",
+		Name:     "dump",
-		Usage: "dumps the state after the run",
+		Usage:    "dumps the state after the run",
+		Category: flags.VMCategory,
 	}
 	InputFlag = &cli.StringFlag{
-		Name:  "input",
+		Name:     "input",
-		Usage: "input for the EVM",
+		Usage:    "input for the EVM",
+		Category: flags.VMCategory,
 	}
 	InputFileFlag = &cli.StringFlag{
-		Name:  "inputfile",
+		Name:     "inputfile",
-		Usage: "file containing input for the EVM",
+		Usage:    "file containing input for the EVM",
-	}
+		Category: flags.VMCategory,
-	VerbosityFlag = &cli.IntFlag{
-		Name:  "verbosity",
-		Usage: "sets the verbosity level",
 	}
 	BenchFlag = &cli.BoolFlag{
-		Name:  "bench",
+		Name:     "bench",
-		Usage: "benchmark the execution",
+		Usage:    "benchmark the execution",
+		Category: flags.VMCategory,
 	}
 	CreateFlag = &cli.BoolFlag{
-		Name:  "create",
+		Name:     "create",
-		Usage: "indicates the action should be create rather than call",
+		Usage:    "indicates the action should be create rather than call",
+		Category: flags.VMCategory,
 	}
 	GenesisFlag = &cli.StringFlag{
-		Name:  "prestate",
+		Name:     "prestate",
-		Usage: "JSON file with prestate (genesis) config",
+		Usage:    "JSON file with prestate (genesis) config",
+		Category: flags.VMCategory,
 	}
 	MachineFlag = &cli.BoolFlag{
-		Name:  "json",
+		Name:     "json",
-		Usage: "output trace logs in machine readable format (json)",
+		Usage:    "output trace logs in machine readable format (json)",
+		Category: flags.VMCategory,
 	}
 	SenderFlag = &cli.StringFlag{
-		Name:  "sender",
+		Name:     "sender",
-		Usage: "The transaction origin",
+		Usage:    "The transaction origin",
+		Category: flags.VMCategory,
 	}
 	ReceiverFlag = &cli.StringFlag{
-		Name:  "receiver",
+		Name:     "receiver",
-		Usage: "The transaction receiver (execution context)",
+		Usage:    "The transaction receiver (execution context)",
+		Category: flags.VMCategory,
 	}
 	DisableMemoryFlag = &cli.BoolFlag{
-		Name:  "nomemory",
+		Name:     "nomemory",
-		Value: true,
+		Value:    true,
-		Usage: "disable memory output",
+		Usage:    "disable memory output",
+		Category: flags.VMCategory,
 	}
 	DisableStackFlag = &cli.BoolFlag{
-		Name:  "nostack",
+		Name:     "nostack",
-		Usage: "disable stack output",
+		Usage:    "disable stack output",
+		Category: flags.VMCategory,
 	}
 	DisableStorageFlag = &cli.BoolFlag{
-		Name:  "nostorage",
+		Name:     "nostorage",
-		Usage: "disable storage output",
+		Usage:    "disable storage output",
+		Category: flags.VMCategory,
 	}
 	DisableReturnDataFlag = &cli.BoolFlag{
-		Name:  "noreturndata",
+		Name:     "noreturndata",
-		Value: true,
+		Value:    true,
-		Usage: "enable return data output",
+		Usage:    "enable return data output",
+		Category: flags.VMCategory,
 	}
 )
 
@@ -183,34 +192,38 @@ var blockBuilderCommand = &cli.Command{
 	},
 }
 
+// vmFlags contains flags related to running the EVM.
+var vmFlags = []cli.Flag{
+	CodeFlag,
+	CodeFileFlag,
+	CreateFlag,
+	GasFlag,
+	PriceFlag,
+	ValueFlag,
+	InputFlag,
+	InputFileFlag,
+	GenesisFlag,
+	SenderFlag,
+	ReceiverFlag,
+}
+
+// traceFlags contains flags that configure tracing output.
+var traceFlags = []cli.Flag{
+	BenchFlag,
+	DebugFlag,
+	DumpFlag,
+	MachineFlag,
+	StatDumpFlag,
+	DisableMemoryFlag,
+	DisableStackFlag,
+	DisableStorageFlag,
+	DisableReturnDataFlag,
+}
+
 var app = flags.NewApp("the evm command line interface")
 
 func init() {
-	app.Flags = []cli.Flag{
+	app.Flags = flags.Merge(vmFlags, traceFlags, debug.Flags)
-		BenchFlag,
-		CreateFlag,
-		DebugFlag,
-		VerbosityFlag,
-		CodeFlag,
-		CodeFileFlag,
-		GasFlag,
-		PriceFlag,
-		ValueFlag,
-		DumpFlag,
-		InputFlag,
-		InputFileFlag,
-		MemProfileFlag,
-		CPUProfileFlag,
-		StatDumpFlag,
-		GenesisFlag,
-		MachineFlag,
-		SenderFlag,
-		ReceiverFlag,
-		DisableMemoryFlag,
-		DisableStackFlag,
-		DisableStorageFlag,
-		DisableReturnDataFlag,
-	}
 	app.Commands = []*cli.Command{
 		compileCommand,
 		disasmCommand,
@@ -221,6 +234,14 @@ func init() {
 		transactionCommand,
 		blockBuilderCommand,
 	}
+	app.Before = func(ctx *cli.Context) error {
+		flags.MigrateGlobalFlags(ctx)
+		return debug.Setup(ctx)
+	}
+	app.After = func(ctx *cli.Context) error {
+		debug.Exit()
+		return nil
+	}
 }
 
 func main() {

cmd/evm/runner.go

@@ -24,7 +24,6 @@ import (
 	"math/big"
 	"os"
 	goruntime "runtime"
-	"runtime/pprof"
 	"testing"
 	"time"
 
@@ -34,12 +33,10 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/core/vm/runtime"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
 	"github.com/ethereum/go-ethereum/internal/flags"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
@@ -52,6 +49,7 @@ var runCommand = &cli.Command{
 	Usage:       "run arbitrary evm binary",
 	ArgsUsage:   "<code>",
 	Description: `The run command runs arbitrary EVM code.`,
+	Flags:       flags.Merge(vmFlags, traceFlags),
 }
 
 // readGenesis will read the given JSON format genesis file and return
@@ -109,9 +107,6 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) (output []by
 }
 
 func runCmd(ctx *cli.Context) error {
-	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
-	glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name)))
-	log.Root().SetHandler(glogger)
 	logconfig := &logger.Config{
 		EnableMemory:     !ctx.Bool(DisableMemoryFlag.Name),
 		DisableStack:     ctx.Bool(DisableStackFlag.Name),
@@ -121,15 +116,15 @@ func runCmd(ctx *cli.Context) error {
 	}
 
 	var (
-		tracer        vm.EVMLogger
+		tracer      vm.EVMLogger
-		debugLogger   *logger.StructLogger
+		debugLogger *logger.StructLogger
-		statedb       *state.StateDB
+		statedb     *state.StateDB
-		chainConfig   *params.ChainConfig
+		chainConfig *params.ChainConfig
-		sender        = common.BytesToAddress([]byte("sender"))
+		sender      = common.BytesToAddress([]byte("sender"))
-		receiver      = common.BytesToAddress([]byte("receiver"))
+		receiver    = common.BytesToAddress([]byte("receiver"))
-		genesisConfig *core.Genesis
+		preimages   = ctx.Bool(DumpFlag.Name)
-		preimages     = ctx.Bool(DumpFlag.Name)
+		blobHashes  []common.Hash  // TODO (MariusVanDerWijden) implement blob hashes in state tests
-		blobHashes    []common.Hash // TODO (MariusVanDerWijden) implement blob hashes in state tests
+		blobBaseFee = new(big.Int) // TODO (MariusVanDerWijden) implement blob fee in state tests
 	)
 	if ctx.Bool(MachineFlag.Name) {
 		tracer = logger.NewJSONLogger(logconfig, os.Stdout)
@@ -139,30 +134,30 @@ func runCmd(ctx *cli.Context) error {
 	} else {
 		debugLogger = logger.NewStructLogger(logconfig)
 	}
+
+	initialGas := ctx.Uint64(GasFlag.Name)
+	genesisConfig := new(core.Genesis)
+	genesisConfig.GasLimit = initialGas
 	if ctx.String(GenesisFlag.Name) != "" {
-		gen := readGenesis(ctx.String(GenesisFlag.Name))
+		genesisConfig = readGenesis(ctx.String(GenesisFlag.Name))
-		genesisConfig = gen
+		if genesisConfig.GasLimit != 0 {
-		db := rawdb.NewMemoryDatabase()
+			initialGas = genesisConfig.GasLimit
-		triedb := trie.NewDatabase(db, &trie.Config{
+		}
-			Preimages: preimages,
-			HashDB:    hashdb.Defaults,
-		})
-		defer triedb.Close()
-		genesis := gen.MustCommit(db, triedb)
-		sdb := state.NewDatabaseWithNodeDB(db, triedb)
-		statedb, _ = state.New(genesis.Root(), sdb, nil)
-		chainConfig = gen.Config
 	} else {
-		db := rawdb.NewMemoryDatabase()
+		genesisConfig.Config = params.AllEthashProtocolChanges
-		triedb := trie.NewDatabase(db, &trie.Config{
-			Preimages: preimages,
-			HashDB:    hashdb.Defaults,
-		})
-		defer triedb.Close()
-		sdb := state.NewDatabaseWithNodeDB(db, triedb)
-		statedb, _ = state.New(types.EmptyRootHash, sdb, nil)
-		genesisConfig = new(core.Genesis)
 	}
+
+	db := rawdb.NewMemoryDatabase()
+	triedb := trie.NewDatabase(db, &trie.Config{
+		Preimages: preimages,
+		HashDB:    hashdb.Defaults,
+	})
+	defer triedb.Close()
+	genesis := genesisConfig.MustCommit(db, triedb)
+	sdb := state.NewDatabaseWithNodeDB(db, triedb)
+	statedb, _ = state.New(genesis.Root(), sdb, nil)
+	chainConfig = genesisConfig.Config
+
 	if ctx.String(SenderFlag.Name) != "" {
 		sender = common.HexToAddress(ctx.String(SenderFlag.Name))
 	}
@@ -216,10 +211,6 @@ func runCmd(ctx *cli.Context) error {
 		}
 		code = common.Hex2Bytes(bin)
 	}
-	initialGas := ctx.Uint64(GasFlag.Name)
-	if genesisConfig.GasLimit != 0 {
-		initialGas = genesisConfig.GasLimit
-	}
 	runtimeConfig := runtime.Config{
 		Origin:      sender,
 		State:       statedb,
@@ -231,24 +222,12 @@ func runCmd(ctx *cli.Context) error {
 		Coinbase:    genesisConfig.Coinbase,
 		BlockNumber: new(big.Int).SetUint64(genesisConfig.Number),
 		BlobHashes:  blobHashes,
+		BlobBaseFee: blobBaseFee,
 		EVMConfig: vm.Config{
 			Tracer: tracer,
 		},
 	}
 
-	if cpuProfilePath := ctx.String(CPUProfileFlag.Name); cpuProfilePath != "" {
-		f, err := os.Create(cpuProfilePath)
-		if err != nil {
-			fmt.Println("could not create CPU profile: ", err)
-			os.Exit(1)
-		}
-		if err := pprof.StartCPUProfile(f); err != nil {
-			fmt.Println("could not start CPU profile: ", err)
-			os.Exit(1)
-		}
-		defer pprof.StopCPUProfile()
-	}
-
 	if chainConfig != nil {
 		runtimeConfig.ChainConfig = chainConfig
 	} else {
@@ -296,19 +275,6 @@ func runCmd(ctx *cli.Context) error {
 		fmt.Println(string(statedb.Dump(nil)))
 	}
 
-	if memProfilePath := ctx.String(MemProfileFlag.Name); memProfilePath != "" {
-		f, err := os.Create(memProfilePath)
-		if err != nil {
-			fmt.Println("could not create memory profile: ", err)
-			os.Exit(1)
-		}
-		if err := pprof.WriteHeapProfile(f); err != nil {
-			fmt.Println("could not write memory profile: ", err)
-			os.Exit(1)
-		}
-		f.Close()
-	}
-
 	if ctx.Bool(DebugFlag.Name) {
 		if debugLogger != nil {
 			fmt.Fprintln(os.Stderr, "#### TRACE ####")

cmd/evm/staterunner.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/eth/tracers/logger"
-	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/tests"
 	"github.com/urfave/cli/v2"
 )
@@ -52,11 +51,6 @@ type StatetestResult struct {
 }
 
 func stateTestCmd(ctx *cli.Context) error {
-	// Configure the go-ethereum logger
-	glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false)))
-	glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name)))
-	log.Root().SetHandler(glogger)
-
 	// Configure the EVM logger
 	config := &logger.Config{
 		EnableMemory:     !ctx.Bool(DisableMemoryFlag.Name),

cmd/evm/t8n_test.go

@@ -24,9 +24,9 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/docker/docker/pkg/reexec"
 	"github.com/ethereum/go-ethereum/cmd/evm/internal/t8ntool"
 	"github.com/ethereum/go-ethereum/internal/cmdtest"
+	"github.com/ethereum/go-ethereum/internal/reexec"
 )
 
 func TestMain(m *testing.M) {
@@ -275,6 +275,14 @@ func TestT8n(t *testing.T) {
 			output: t8nOutput{alloc: true, result: true},
 			expOut: "exp.json",
 		},
+		{ // More cancun test, plus example of rlp-transaction that cannot be decoded properly
+			base: "./testdata/30",
+			input: t8nInput{
+				"alloc.json", "txs_more.rlp", "env.json", "Cancun", "",
+			},
+			output: t8nOutput{alloc: true, result: true},
+			expOut: "exp.json",
+		},
 	} {
 		args := []string{"t8n"}
 		args = append(args, tc.output.get()...)

cmd/evm/testdata/28/env.json

@@ -9,8 +9,7 @@
     "parentDifficulty" : "0x00",
     "parentUncleHash" : "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
     "currentRandom" : "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
-    "withdrawals" : [
+    "withdrawals" : [],
-    ],
     "parentBaseFee" : "0x0a",
     "parentGasUsed" : "0x00",
     "parentGasLimit" : "0x7fffffffffffffff",
@@ -20,4 +19,4 @@
         "0" : "0x3a9b485972e7353edd9152712492f0c58d89ef80623686b6bf947a4a6dce6cb6"
     },
     "parentBeaconBlockRoot": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00"
-}
+}

cmd/evm/testdata/28/exp.json

@@ -42,6 +42,6 @@
     "currentBaseFee": "0x9",
     "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
     "currentExcessBlobGas": "0x0",
-    "currentBlobGasUsed": "0x20000"
+    "blobGasUsed": "0x20000"
   }
-}
+}

cmd/evm/testdata/29/alloc.json

@@ -6,7 +6,7 @@
     "storage" : {
     }
   },
-  "0xbEac00dDB15f3B6d645C48263dC93862413A222D" : {
+  "0x000F3df6D732807Ef1319fB7B8bB8522d0Beac02" : {
     "balance" : "0x1",
     "code" : "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500",
     "nonce" : "0x00",

cmd/evm/testdata/29/exp.json

@@ -1,6 +1,6 @@
 {
   "alloc": {
-    "0xbeac00ddb15f3b6d645c48263dc93862413a222d": {
+    "0x000f3df6d732807ef1319fb7b8bb8522d0beac02": {
       "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500",
       "storage": {
         "0x000000000000000000000000000000000000000000000000000000000000079e": "0x000000000000000000000000000000000000000000000000000000000000079e",
@@ -14,7 +14,7 @@
     }
   },
   "result": {
-    "stateRoot": "0x2db9f6bc233e8fd0af2d8023404493a19b37d9d69ace71f4e73158851fced574",
+    "stateRoot": "0x19a4f821a7c0a6f4c934f9acb0fe9ce5417b68086e12513ecbc3e3f57e01573c",
     "txRoot": "0x248074fabe112f7d93917f292b64932394f835bb98da91f21501574d58ec92ab",
     "receiptsRoot": "0xf78dfb743fbd92ade140711c8bbc542b5e307f0ab7984eff35d751969fe57efa",
     "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
@@ -40,6 +40,6 @@
     "currentBaseFee": "0x9",
     "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
     "currentExcessBlobGas": "0x0",
-    "currentBlobGasUsed": "0x0"
+    "blobGasUsed": "0x0"
   }
-}
+}

cmd/evm/testdata/29/readme.md

@@ -1,29 +1,29 @@
 ## EIP 4788
 
 This test contains testcases for EIP-4788. The 4788-contract is 
-located at address `0xbeac00ddb15f3b6d645c48263dc93862413a222d`, and this test executes a simple transaction. It also
+located at address `0x000F3df6D732807Ef1319fB7B8bB8522d0Beac02`, and this test executes a simple transaction. It also
 implicitly invokes the system tx, which sets calls the contract and sets the 
 storage values
+
 ```
 $ dir=./testdata/29/ && go run . t8n --state.fork=Cancun  --input.alloc=$dir/alloc.json --input.txs=$dir/txs.json --input.env=$dir/env.json --output.alloc=stdout
-INFO [08-15|20:07:56.335] Trie dumping started                     root=ecde45..2af8a7
+INFO [09-27|15:34:53.049] Trie dumping started                     root=19a4f8..01573c
-INFO [08-15|20:07:56.335] Trie dumping complete                    accounts=2 elapsed="225.848µs"
+INFO [09-27|15:34:53.049] Trie dumping complete                    accounts=2 elapsed="192.759µs"
-INFO [08-15|20:07:56.335] Wrote file                               file=result.json
+INFO [09-27|15:34:53.050] Wrote file                               file=result.json
 {
   "alloc": {
-    "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+    "0x000f3df6d732807ef1319fb7b8bb8522d0beac02": {
-      "balance": "0x16345785d871db8",
-      "nonce": "0x1"
-    },
-    "0xbeac00541d49391ed88abf392bfc1f4dea8c4143": {
       "code": "0x3373fffffffffffffffffffffffffffffffffffffffe14604457602036146024575f5ffd5b620180005f350680545f35146037575f5ffd5b6201800001545f5260205ff35b6201800042064281555f359062018000015500",
       "storage": {
         "0x000000000000000000000000000000000000000000000000000000000000079e": "0x000000000000000000000000000000000000000000000000000000000000079e",
         "0x000000000000000000000000000000000000000000000000000000000001879e": "0x0000beac00beac00beac00beac00beac00beac00beac00beac00beac00beac00"
       },
-      "balance": "0x
+      "balance": "0x1"
+    },
+    "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+      "balance": "0x16345785d871db8",
+      "nonce": "0x1"
     }
   }
 }
-
 ```

cmd/evm/testdata/30/README.txt

@@ -0,0 +1,77 @@
+This example comes from https://github.com/ethereum/go-ethereum/issues/27730.
+The input transactions contain three transactions, number `0` and `2` are taken from
+`testdata/13`, whereas number `1` is taken from #27730.
+
+The problematic second transaction cannot be RLP-decoded, and the expectation is
+that that particular transaction should be rejected, but number `0` and `1` should
+still be accepted.
+
+```
+$ go run . t8n --input.alloc=./testdata/30/alloc.json --input.txs=./testdata/30/txs_more.rlp --input.env=./testdata/30/env.json --output.result=stdout --output.alloc=stdout --state.fork=Cancun
+WARN [10-22|15:38:03.283] rejected tx                              index=1 error="rlp: input string too short for common.Address, decoding into (types.Transaction)(types.BlobTx).To"
+INFO [10-22|15:38:03.284] Trie dumping started                     root=348312..915c93
+INFO [10-22|15:38:03.284] Trie dumping complete                    accounts=3 elapsed="160.831µs"
+{
+  "alloc": {
+    "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": {
+      "code": "0x60004960005500",
+      "balance": "0xde0b6b3a7640000"
+    },
+    "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+      "balance": "0xde0b6b3a7640000"
+    },
+    "0xd02d72e067e77158444ef2020ff2d325f929b363": {
+      "balance": "0xfffffffb8390",
+      "nonce": "0x3"
+    }
+  },
+  "result": {
+    "stateRoot": "0x3483124b6710486c9fb3e07975669c66924697c88cccdcc166af5e1218915c93",
+    "txRoot": "0x013509c8563d41c0ae4bf38f2d6d19fc6512a1d0d6be045079c8c9f68bf45f9d",
+    "receiptsRoot": "0x75308898d571eafb5cd8cde8278bf5b3d13c5f6ec074926de3bb895b519264e1",
+    "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+    "receipts": [
+      {
+        "type": "0x2",
+        "root": "0x",
+        "status": "0x1",
+        "cumulativeGasUsed": "0x5208",
+        "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        "logs": null,
+        "transactionHash": "0xa98a24882ea90916c6a86da650fbc6b14238e46f0af04a131ce92be897507476",
+        "contractAddress": "0x0000000000000000000000000000000000000000",
+        "gasUsed": "0x5208",
+        "effectiveGasPrice": null,
+        "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+        "transactionIndex": "0x0"
+      },
+      {
+        "type": "0x2",
+        "root": "0x",
+        "status": "0x1",
+        "cumulativeGasUsed": "0xa410",
+        "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        "logs": null,
+        "transactionHash": "0x36bad80acce7040c45fd32764b5c2b2d2e6f778669fb41791f73f546d56e739a",
+        "contractAddress": "0x0000000000000000000000000000000000000000",
+        "gasUsed": "0x5208",
+        "effectiveGasPrice": null,
+        "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+        "transactionIndex": "0x1"
+      }
+    ],
+    "rejected": [
+      {
+        "index": 1,
+        "error": "rlp: input string too short for common.Address, decoding into (types.Transaction)(types.BlobTx).To"
+      }
+    ],
+    "currentDifficulty": null,
+    "gasUsed": "0xa410",
+    "currentBaseFee": "0x7",
+    "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
+  }
+}
+
+```

cmd/evm/testdata/30/alloc.json

@@ -0,0 +1,23 @@
+{
+    "0x095e7baea6a6c7c4c2dfeb977efac326af552d87" : {
+        "balance" : "0x0de0b6b3a7640000",
+        "code" : "0x60004960005500",
+        "nonce" : "0x00",
+        "storage" : {
+        }
+    },
+    "0xd02d72e067e77158444ef2020ff2d325f929b363" : {
+        "balance": "0x01000000000000",
+        "code": "0x",
+        "nonce": "0x01",
+        "storage": {
+        }
+    },
+    "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" : {
+        "balance" : "0x0de0b6b3a7640000",
+        "code" : "0x",
+        "nonce" : "0x00",
+        "storage" : {
+        }
+    }
+}

cmd/evm/testdata/30/env.json

@@ -0,0 +1,23 @@
+{
+    "currentCoinbase" : "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba",
+    "currentNumber" : "0x01",
+    "currentTimestamp" : "0x03e8",
+    "currentGasLimit" : "0x1000000000",
+    "previousHash" : "0xe4e2a30b340bec696242b67584264f878600dce98354ae0b6328740fd4ff18da",
+    "currentDataGasUsed" : "0x2000",
+    "parentTimestamp" : "0x00",
+    "parentDifficulty" : "0x00",
+    "parentUncleHash" : "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "parentBeaconBlockRoot" : "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "currentRandom" : "0x0000000000000000000000000000000000000000000000000000000000020000",
+    "withdrawals" : [
+    ],
+    "parentBaseFee" : "0x08",
+    "parentGasUsed" : "0x00",
+    "parentGasLimit" : "0x1000000000",
+    "parentExcessBlobGas" : "0x1000",
+    "parentBlobGasUsed" : "0x2000",
+    "blockHashes" : {
+        "0" : "0xe4e2a30b340bec696242b67584264f878600dce98354ae0b6328740fd4ff18da"
+    }
+}

cmd/evm/testdata/30/exp.json

@@ -0,0 +1,64 @@
+{
+  "alloc": {
+    "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": {
+      "code": "0x60004960005500",
+      "balance": "0xde0b6b3a7640000"
+    },
+    "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": {
+      "balance": "0xde0b6b3a7640000"
+    },
+    "0xd02d72e067e77158444ef2020ff2d325f929b363": {
+      "balance": "0xfffffffb8390",
+      "nonce": "0x3"
+    }
+  },
+  "result": {
+    "stateRoot": "0x3483124b6710486c9fb3e07975669c66924697c88cccdcc166af5e1218915c93",
+    "txRoot": "0x013509c8563d41c0ae4bf38f2d6d19fc6512a1d0d6be045079c8c9f68bf45f9d",
+    "receiptsRoot": "0x75308898d571eafb5cd8cde8278bf5b3d13c5f6ec074926de3bb895b519264e1",
+    "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347",
+    "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+    "receipts": [
+      {
+        "type": "0x2",
+        "root": "0x",
+        "status": "0x1",
+        "cumulativeGasUsed": "0x5208",
+        "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        "logs": null,
+        "transactionHash": "0xa98a24882ea90916c6a86da650fbc6b14238e46f0af04a131ce92be897507476",
+        "contractAddress": "0x0000000000000000000000000000000000000000",
+        "gasUsed": "0x5208",
+        "effectiveGasPrice": null,
+        "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+        "transactionIndex": "0x0"
+      },
+      {
+        "type": "0x2",
+        "root": "0x",
+        "status": "0x1",
+        "cumulativeGasUsed": "0xa410",
+        "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000",
+        "logs": null,
+        "transactionHash": "0x36bad80acce7040c45fd32764b5c2b2d2e6f778669fb41791f73f546d56e739a",
+        "contractAddress": "0x0000000000000000000000000000000000000000",
+        "gasUsed": "0x5208",
+        "effectiveGasPrice": null,
+        "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000",
+        "transactionIndex": "0x1"
+      }
+    ],
+    "rejected": [
+      {
+        "index": 1,
+        "error": "rlp: input string too short for common.Address, decoding into (types.Transaction)(types.BlobTx).To"
+      }
+    ],
+    "currentDifficulty": null,
+    "gasUsed": "0xa410",
+    "currentBaseFee": "0x7",
+    "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421",
+    "currentExcessBlobGas": "0x0",
+    "blobGasUsed": "0x0"
+  }
+}

cmd/evm/testdata/30/txs.rlp

@@ -0,0 +1 @@
+"0xf8dbb8d903f8d601800285012a05f200833d090080830186a000f85bf85994095e7baea6a6c7c4c2dfeb977efac326af552d87f842a00000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000010ae1a001a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d880a0fc12b67159a3567f8bdbc49e0be369a2e20e09d57a51c41310543a4128409464a02de0cfe5495c4f58ff60645ceda0afd67a4c90a70bc89fe207269435b35e5b67"

cmd/evm/testdata/30/txs_more.rlp

@@ -0,0 +1 @@
+"0xf901adb86702f864010180820fa08284d09411111111111111111111111111111111111111118080c001a0b7dfab36232379bb3d1497a4f91c1966b1f932eae3ade107bf5d723b9cb474e0a06261c359a10f2132f126d250485b90cf20f30340801244a08ef6142ab33d1904b8d903f8d601800285012a05f200833d090080830186a000f85bf85994095e7baea6a6c7c4c2dfeb977efac326af552d87f842a00000000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000000000000000000010ae1a001a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d880a0fc12b67159a3567f8bdbc49e0be369a2e20e09d57a51c41310543a4128409464a02de0cfe5495c4f58ff60645ceda0afd67a4c90a70bc89fe207269435b35e5b67b86702f864010280820fa08284d09411111111111111111111111111111111111111118080c080a0d4ec563b6568cd42d998fc4134b36933c6568d01533b5adf08769270243c6c7fa072bf7c21eac6bbeae5143371eef26d5e279637f3bd73482b55979d76d935b1e9"

cmd/evm/testdata/8/readme.md

@@ -32,7 +32,7 @@ dir=./testdata/8 && ./evm t8n --state.fork=Berlin --input.alloc=$dir/alloc.json
 {"pc":4,"op":84,"gas":"0x48456","gasCost":"0x64","memSize":0,"stack":["0x3"],"depth":1,"refund":0,"opName":"SLOAD"}
 ```
 
-Simlarly, we can provide the input transactions via `stdin` instead of as file: 
+Similarly, we can provide the input transactions via `stdin` instead of as file:
 
 ```
 $ dir=./testdata/8 \

cmd/evm/testdata/9/readme.md

@@ -1,6 +1,6 @@
 ## EIP-1559 testing
 
-This test contains testcases for EIP-1559, which uses an new transaction type and has a new block parameter. 
+This test contains testcases for EIP-1559, which uses a new transaction type and has a new block parameter.
 
 ### Prestate
 

cmd/geth/chaincmd.go

@@ -50,8 +50,9 @@ var (
 		ArgsUsage: "<genesisPath>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.CachePreimagesFlag,
-			utils.StateSchemeFlag,
+			utils.OverrideCancun,
-		}, utils.DatabasePathFlags),
+			utils.OverrideVerkle,
+		}, utils.DatabaseFlags),
 		Description: `
 The init command initializes a new genesis block and definition for the network.
 This is a destructive action and changes the network in which you will be
@@ -98,9 +99,8 @@ if one is set.  Otherwise it prints the genesis from the datadir.`,
 			utils.TxLookupLimitFlag,
 			utils.VMTraceFlag,
 			utils.TransactionHistoryFlag,
-			utils.StateSchemeFlag,
 			utils.StateHistoryFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabaseFlags),
 		Description: `
 The import command imports blocks from an RLP-encoded form. The form can be one file
 with several RLP-encoded blocks, or several files can be used.
@@ -116,8 +116,7 @@ processing will proceed even if an individual RLP-file import failure occurs.`,
 		Flags: flags.Merge([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-			utils.StateSchemeFlag,
+		}, utils.DatabaseFlags),
-		}, utils.DatabasePathFlags),
 		Description: `
 Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
@@ -133,7 +132,7 @@ be gzipped.`,
 		Flags: flags.Merge([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabaseFlags),
 		Description: `
 The import-preimages command imports hash preimages from an RLP encoded stream.
 It's deprecated, please use "geth db import" instead.
@@ -147,7 +146,7 @@ It's deprecated, please use "geth db import" instead.
 		Flags: flags.Merge([]cli.Flag{
 			utils.CacheFlag,
 			utils.SyncModeFlag,
-		}, utils.DatabasePathFlags),
+		}, utils.DatabaseFlags),
 		Description: `
 The export-preimages command exports hash preimages to an RLP encoded stream.
 It's deprecated, please use "geth db export" instead.
@@ -166,8 +165,7 @@ It's deprecated, please use "geth db export" instead.
 			utils.IncludeIncompletesFlag,
 			utils.StartKeyFlag,
 			utils.DumpLimitFlag,
-			utils.StateSchemeFlag,
+		}, utils.DatabaseFlags),
-		}, utils.DatabasePathFlags),
 		Description: `
 This command dumps out the state for a given block (or latest, if none provided).
 `,
@@ -198,6 +196,15 @@ func initGenesis(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
+	var overrides core.ChainOverrides
+	if ctx.IsSet(utils.OverrideCancun.Name) {
+		v := ctx.Uint64(utils.OverrideCancun.Name)
+		overrides.OverrideCancun = &v
+	}
+	if ctx.IsSet(utils.OverrideVerkle.Name) {
+		v := ctx.Uint64(utils.OverrideVerkle.Name)
+		overrides.OverrideVerkle = &v
+	}
 	for _, name := range []string{"chaindata", "lightchaindata"} {
 		chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false)
 		if err != nil {
@@ -208,7 +215,7 @@ func initGenesis(ctx *cli.Context) error {
 		triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false)
 		defer triedb.Close()
 
-		_, hash, err := core.SetupGenesisBlock(chaindb, triedb, genesis)
+		_, hash, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides)
 		if err != nil {
 			utils.Fatalf("Failed to write genesis block: %v", err)
 		}
@@ -341,7 +348,8 @@ func exportChain(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	chain, _ := utils.MakeChain(ctx, stack, true)
+	chain, db := utils.MakeChain(ctx, stack, true)
+	defer db.Close()
 	start := time.Now()
 
 	var err error
@@ -381,6 +389,7 @@ func importPreimages(ctx *cli.Context) error {
 	defer stack.Close()
 
 	db := utils.MakeChainDatabase(ctx, stack, false)
+	defer db.Close()
 	start := time.Now()
 
 	if err := utils.ImportPreimages(db, ctx.Args().First()); err != nil {
@@ -399,6 +408,7 @@ func exportPreimages(ctx *cli.Context) error {
 	defer stack.Close()
 
 	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
 	start := time.Now()
 
 	if err := utils.ExportPreimages(db, ctx.Args().First()); err != nil {
@@ -410,6 +420,8 @@ func exportPreimages(ctx *cli.Context) error {
 
 func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) {
 	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+
 	var header *types.Header
 	if ctx.NArg() > 1 {
 		return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg())
@@ -474,7 +486,7 @@ func dump(ctx *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	triedb := utils.MakeTrieDatabase(ctx, db, true, false) // always enable preimage lookup
+	triedb := utils.MakeTrieDatabase(ctx, db, true, true) // always enable preimage lookup
 	defer triedb.Close()
 
 	state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil)

cmd/geth/config.go

@@ -32,6 +32,8 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/scwallet"
 	"github.com/ethereum/go-ethereum/accounts/usbwallet"
 	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/eth/catalyst"
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/eth/ethconfig"
@@ -198,10 +200,10 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 			protos = append(protos, fmt.Sprintf("%v/%d", p.Name, p.Version))
 		}
 		metrics.NewRegisteredGaugeInfo("geth/info", nil).Update(metrics.GaugeInfoValue{
-			"arch":          runtime.GOARCH,
+			"arch":      runtime.GOARCH,
-			"os":            runtime.GOOS,
+			"os":        runtime.GOOS,
-			"version":       cfg.Node.Version,
+			"version":   cfg.Node.Version,
-			"eth_protocols": strings.Join(protos, ","),
+			"protocols": strings.Join(protos, ","),
 		})
 	}
 
@@ -212,17 +214,18 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 	if ctx.IsSet(utils.GraphQLEnabledFlag.Name) {
 		utils.RegisterGraphQLService(stack, backend, filterSystem, &cfg.Node)
 	}
-
 	// Add the Ethereum Stats daemon if requested.
 	if cfg.Ethstats.URL != "" {
 		utils.RegisterEthStatsService(stack, backend, cfg.Ethstats.URL)
 	}
-
 	// Configure full-sync tester service if requested
-	if ctx.IsSet(utils.SyncTargetFlag.Name) && cfg.Eth.SyncMode == downloader.FullSync {
+	if ctx.IsSet(utils.SyncTargetFlag.Name) {
-		utils.RegisterFullSyncTester(stack, eth, ctx.Path(utils.SyncTargetFlag.Name))
+		hex := hexutil.MustDecode(ctx.String(utils.SyncTargetFlag.Name))
+		if len(hex) != common.HashLength {
+			utils.Fatalf("invalid sync target length: have %d, want %d", len(hex), common.HashLength)
+		}
+		utils.RegisterFullSyncTester(stack, eth, common.BytesToHash(hex))
 	}
-
 	// Start the dev mode if requested, or launch the engine API for
 	// interacting with external consensus client.
 	if ctx.IsSet(utils.DeveloperFlag.Name) {

cmd/geth/dbcmd.go

@@ -48,7 +48,7 @@ var (
 		Name:      "removedb",
 		Usage:     "Remove blockchain and state databases",
 		ArgsUsage: "",
-		Flags:     utils.DatabasePathFlags,
+		Flags:     utils.DatabaseFlags,
 		Description: `
 Remove blockchain and state databases`,
 	}
@@ -77,7 +77,7 @@ Remove blockchain and state databases`,
 		ArgsUsage: "<prefix> <start>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Usage:       "Inspect the storage size for each type of data in the database",
 		Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
 	}
@@ -85,7 +85,7 @@ Remove blockchain and state databases`,
 		Action:    checkStateContent,
 		Name:      "check-state-content",
 		ArgsUsage: "<start (optional)>",
-		Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+		Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 		Usage:     "Verify that state data is cryptographically correct",
 		Description: `This command iterates the entire database for 32-byte keys, looking for rlp-encoded trie nodes.
 For each trie node encountered, it checks that the key corresponds to the keccak256(value). If this is not true, this indicates
@@ -97,7 +97,7 @@ a data corruption.`,
 		Usage:  "Print leveldb statistics",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 	}
 	dbCompactCmd = &cli.Command{
 		Action: dbCompact,
@@ -107,7 +107,7 @@ a data corruption.`,
 			utils.SyncModeFlag,
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: `This command performs a database compaction. 
 WARNING: This operation may take a very long time to finish, and may cause database
 corruption if it is aborted during execution'!`,
@@ -119,7 +119,7 @@ corruption if it is aborted during execution'!`,
 		ArgsUsage: "<hex-encoded key>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDeleteCmd = &cli.Command{
@@ -129,7 +129,7 @@ corruption if it is aborted during execution'!`,
 		ArgsUsage: "<hex-encoded key>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: `This command deletes the specified database key from the database. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -140,7 +140,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<hex-encoded key> <hex-encoded value>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: `This command sets a given database key to the given value. 
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -151,8 +151,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<hex-encoded state root> <hex-encoded account hash> <hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-			utils.StateSchemeFlag,
+		}, utils.NetworkFlags, utils.DatabaseFlags),
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDumpFreezerIndex = &cli.Command{
@@ -162,7 +161,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<freezer-type> <table-type> <start (int)> <end (int)>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "This command displays information about the freezer index.",
 	}
 	dbImportCmd = &cli.Command{
@@ -172,7 +171,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<dumpfile> <start (optional)",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "The import command imports the specific chain data from an RLP encoded stream.",
 	}
 	dbExportCmd = &cli.Command{
@@ -182,7 +181,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		ArgsUsage: "<type> <dumpfile>",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
 	}
 	dbMetadataCmd = &cli.Command{
@@ -191,7 +190,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Usage:  "Shows metadata about the chain status.",
 		Flags: flags.Merge([]cli.Flag{
 			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabasePathFlags),
+		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Shows metadata about the chain status.",
 	}
 )
@@ -595,6 +594,7 @@ func importLDBdata(ctx *cli.Context) error {
 		close(stop)
 	}()
 	db := utils.MakeChainDatabase(ctx, stack, false)
+	defer db.Close()
 	return utils.ImportLDBData(db, fName, int64(start), stop)
 }
 
@@ -691,6 +691,7 @@ func exportChaindata(ctx *cli.Context) error {
 		close(stop)
 	}()
 	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
 	return utils.ExportChaindata(ctx.Args().Get(1), kind, exporter(db), stop)
 }
 
@@ -698,6 +699,8 @@ func showMetaData(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+
 	ancients, err := db.Ancients()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Error accessing ancients: %v", err)

cmd/geth/genesis_test.go

@@ -176,12 +176,12 @@ func TestCustomBackend(t *testing.T) {
 		{ // Can't start pebble on top of leveldb
 			initArgs:   []string{"--db.engine", "leveldb"},
 			execArgs:   []string{"--db.engine", "pebble"},
-			execExpect: `Fatal: Could not open database: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
+			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was pebble but found pre-existing leveldb database in specified data directory`,
 		},
 		{ // Can't start leveldb on top of pebble
 			initArgs:   []string{"--db.engine", "pebble"},
 			execArgs:   []string{"--db.engine", "leveldb"},
-			execExpect: `Fatal: Could not open database: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
+			execExpect: `Fatal: Failed to register the Ethereum service: db.engine choice was leveldb but found pre-existing pebble database in specified data directory`,
 		},
 		{ // Reject invalid backend choice
 			initArgs:   []string{"--db.engine", "mssql"},

cmd/geth/logging_test.go

@@ -0,0 +1,185 @@
+//go:build integrationtests
+
+// Copyright 2023 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/internal/reexec"
+)
+
+func runSelf(args ...string) ([]byte, error) {
+	cmd := &exec.Cmd{
+		Path: reexec.Self(),
+		Args: append([]string{"geth-test"}, args...),
+	}
+	return cmd.CombinedOutput()
+}
+
+func split(input io.Reader) []string {
+	var output []string
+	scanner := bufio.NewScanner(input)
+	scanner.Split(bufio.ScanLines)
+	for scanner.Scan() {
+		output = append(output, strings.TrimSpace(scanner.Text()))
+	}
+	return output
+}
+
+func censor(input string, start, end int) string {
+	if len(input) < end {
+		return input
+	}
+	return input[:start] + strings.Repeat("X", end-start) + input[end:]
+}
+
+func TestLogging(t *testing.T) {
+	testConsoleLogging(t, "terminal", 6, 24)
+	testConsoleLogging(t, "logfmt", 2, 26)
+}
+
+func testConsoleLogging(t *testing.T, format string, tStart, tEnd int) {
+	haveB, err := runSelf("--log.format", format, "logtest")
+	if err != nil {
+		t.Fatal(err)
+	}
+	readFile, err := os.Open(fmt.Sprintf("testdata/logging/logtest-%v.txt", format))
+	if err != nil {
+		t.Fatal(err)
+	}
+	wantLines := split(readFile)
+	haveLines := split(bytes.NewBuffer(haveB))
+	for i, want := range wantLines {
+		if i > len(haveLines)-1 {
+			t.Fatalf("format %v, line %d missing, want:%v", format, i, want)
+		}
+		have := haveLines[i]
+		for strings.Contains(have, "Unknown config environment variable") {
+			// This can happen on CI runs. Drop it.
+			haveLines = append(haveLines[:i], haveLines[i+1:]...)
+			have = haveLines[i]
+		}
+
+		// Black out the timestamp
+		have = censor(have, tStart, tEnd)
+		want = censor(want, tStart, tEnd)
+		if have != want {
+			t.Logf(nicediff([]byte(have), []byte(want)))
+			t.Fatalf("format %v, line %d\nhave %v\nwant %v", format, i, have, want)
+		}
+	}
+	if len(haveLines) != len(wantLines) {
+		t.Errorf("format %v, want %d lines, have %d", format, len(haveLines), len(wantLines))
+	}
+}
+
+func TestVmodule(t *testing.T) {
+	checkOutput := func(level int, want, wantNot string) {
+		t.Helper()
+		output, err := runSelf("--log.format", "terminal", "--verbosity=0", "--log.vmodule", fmt.Sprintf("logtestcmd_active.go=%d", level), "logtest")
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(want) > 0 && !strings.Contains(string(output), want) { // trace should be present at 5
+			t.Errorf("failed to find expected string ('%s') in output", want)
+		}
+		if len(wantNot) > 0 && strings.Contains(string(output), wantNot) { // trace should be present at 5
+			t.Errorf("string ('%s') should not be present in output", wantNot)
+		}
+	}
+	checkOutput(5, "log at level trace", "")                   // trace should be present at 5
+	checkOutput(4, "log at level debug", "log at level trace") // debug should be present at 4, but trace should be missing
+	checkOutput(3, "log at level info", "log at level debug")  // info should be present at 3, but debug should be missing
+	checkOutput(2, "log at level warn", "log at level info")   // warn should be present at 2, but info should be missing
+	checkOutput(1, "log at level error", "log at level warn")  // error should be present at 1, but warn should be missing
+}
+
+func nicediff(have, want []byte) string {
+	var i = 0
+	for ; i < len(have) && i < len(want); i++ {
+		if want[i] != have[i] {
+			break
+		}
+	}
+	var end = i + 40
+	var start = i - 50
+	if start < 0 {
+		start = 0
+	}
+	var h, w string
+	if end < len(have) {
+		h = string(have[start:end])
+	} else {
+		h = string(have[start:])
+	}
+	if end < len(want) {
+		w = string(want[start:end])
+	} else {
+		w = string(want[start:])
+	}
+	return fmt.Sprintf("have vs want:\n%q\n%q\n", h, w)
+}
+
+func TestFileOut(t *testing.T) {
+	var (
+		have, want []byte
+		err        error
+		path       = fmt.Sprintf("%s/test_file_out-%d", os.TempDir(), rand.Int63())
+	)
+	t.Cleanup(func() { os.Remove(path) })
+	if want, err = runSelf(fmt.Sprintf("--log.file=%s", path), "logtest"); err != nil {
+		t.Fatal(err)
+	}
+	if have, err = os.ReadFile(path); err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(have, want) {
+		// show an intelligent diff
+		t.Logf(nicediff(have, want))
+		t.Errorf("file content wrong")
+	}
+}
+
+func TestRotatingFileOut(t *testing.T) {
+	var (
+		have, want []byte
+		err        error
+		path       = fmt.Sprintf("%s/test_file_out-%d", os.TempDir(), rand.Int63())
+	)
+	t.Cleanup(func() { os.Remove(path) })
+	if want, err = runSelf(fmt.Sprintf("--log.file=%s", path), "--log.rotate", "logtest"); err != nil {
+		t.Fatal(err)
+	}
+	if have, err = os.ReadFile(path); err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(have, want) {
+		// show an intelligent diff
+		t.Logf(nicediff(have, want))
+		t.Errorf("file content wrong")
+	}
+}

cmd/geth/logtestcmd_active.go

@@ -0,0 +1,134 @@
+//go:build integrationtests
+
+// Copyright 2023 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"fmt"
+	"math"
+	"math/big"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/holiman/uint256"
+	"github.com/urfave/cli/v2"
+)
+
+var logTestCommand = &cli.Command{
+	Action:    logTest,
+	Name:      "logtest",
+	Usage:     "Print some log messages",
+	ArgsUsage: " ",
+	Description: `
+This command is only meant for testing.
+`}
+
+// logTest is an entry point which spits out some logs. This is used by testing
+// to verify expected outputs
+func logTest(ctx *cli.Context) error {
+	log.ResetGlobalState()
+	{ // big.Int
+		ba, _ := new(big.Int).SetString("111222333444555678999", 10)    // "111,222,333,444,555,678,999"
+		bb, _ := new(big.Int).SetString("-111222333444555678999", 10)   // "-111,222,333,444,555,678,999"
+		bc, _ := new(big.Int).SetString("11122233344455567899900", 10)  // "11,122,233,344,455,567,899,900"
+		bd, _ := new(big.Int).SetString("-11122233344455567899900", 10) // "-11,122,233,344,455,567,899,900"
+		log.Info("big.Int", "111,222,333,444,555,678,999", ba)
+		log.Info("-big.Int", "-111,222,333,444,555,678,999", bb)
+		log.Info("big.Int", "11,122,233,344,455,567,899,900", bc)
+		log.Info("-big.Int", "-11,122,233,344,455,567,899,900", bd)
+	}
+	{ //uint256
+		ua, _ := uint256.FromDecimal("111222333444555678999")
+		ub, _ := uint256.FromDecimal("11122233344455567899900")
+		log.Info("uint256", "111,222,333,444,555,678,999", ua)
+		log.Info("uint256", "11,122,233,344,455,567,899,900", ub)
+	}
+	{ // int64
+		log.Info("int64", "1,000,000", int64(1000000))
+		log.Info("int64", "-1,000,000", int64(-1000000))
+		log.Info("int64", "9,223,372,036,854,775,807", int64(math.MaxInt64))
+		log.Info("int64", "-9,223,372,036,854,775,808", int64(math.MinInt64))
+	}
+	{ // uint64
+		log.Info("uint64", "1,000,000", uint64(1000000))
+		log.Info("uint64", "18,446,744,073,709,551,615", uint64(math.MaxUint64))
+	}
+	{ // Special characters
+		log.Info("Special chars in value", "key", "special \r\n\t chars")
+		log.Info("Special chars in key", "special \n\t chars", "value")
+
+		log.Info("nospace", "nospace", "nospace")
+		log.Info("with space", "with nospace", "with nospace")
+
+		log.Info("Bash escapes in value", "key", "\u001b[1G\u001b[K\u001b[1A")
+		log.Info("Bash escapes in key", "\u001b[1G\u001b[K\u001b[1A", "value")
+
+		log.Info("Bash escapes in message  \u001b[1G\u001b[K\u001b[1A end", "key", "value")
+
+		colored := fmt.Sprintf("\u001B[%dmColored\u001B[0m[", 35)
+		log.Info(colored, colored, colored)
+	}
+	{ // Custom Stringer() - type
+		log.Info("Custom Stringer value", "2562047h47m16.854s", common.PrettyDuration(time.Duration(9223372036854775807)))
+	}
+	{ // Lazy eval
+		log.Info("Lazy evaluation of value", "key", log.Lazy{Fn: func() interface{} { return "lazy value" }})
+	}
+	{ // Multi-line message
+		log.Info("A message with wonky \U0001F4A9 characters")
+		log.Info("A multiline message \nINFO [10-18|14:11:31.106] with wonky characters \U0001F4A9")
+		log.Info("A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above")
+	}
+	{ // Miscellaneous json-quirks
+		// This will check if the json output uses strings or json-booleans to represent bool values
+		log.Info("boolean", "true", true, "false", false)
+		// Handling of duplicate keys.
+		// This is actually ill-handled by the current handler: the format.go
+		// uses a global 'fieldPadding' map and mixes up the two keys. If 'alpha'
+		// is shorter than beta, it sometimes causes erroneous padding -- and what's more
+		// it causes _different_ padding in multi-handler context, e.g. both file-
+		// and console output, making the two mismatch.
+		log.Info("repeated-key 1", "foo", "alpha", "foo", "beta")
+		log.Info("repeated-key 2", "xx", "short", "xx", "longer")
+	}
+	{ // loglevels
+		log.Debug("log at level debug")
+		log.Trace("log at level trace")
+		log.Info("log at level info")
+		log.Warn("log at level warn")
+		log.Error("log at level error")
+	}
+	{
+		// The current log formatter has a global map of paddings, storing the
+		// longest seen padding per key in a map. This results in a statefulness
+		// which has some odd side-effects. Demonstrated here:
+		log.Info("test", "bar", "short", "a", "aligned left")
+		log.Info("test", "bar", "a long message", "a", 1)
+		log.Info("test", "bar", "short", "a", "aligned right")
+	}
+	{
+		// This sequence of logs should be output with alignment, so each field becoems a column.
+		log.Info("The following logs should align so that the key-fields make 5 columns")
+		log.Info("Inserted known block", "number", 1_012, "hash", common.HexToHash("0x1234"), "txs", 200, "gas", 1_123_123, "other", "first")
+		log.Info("Inserted new block", "number", 1, "hash", common.HexToHash("0x1235"), "txs", 2, "gas", 1_123, "other", "second")
+		log.Info("Inserted known block", "number", 99, "hash", common.HexToHash("0x12322"), "txs", 10, "gas", 1, "other", "third")
+		log.Warn("Inserted known block", "number", 1_012, "hash", common.HexToHash("0x1234"), "txs", 200, "gas", 99, "other", "fourth")
+	}
+	return nil
+}

cmd/geth/logtestcmd_inactive.go

@@ -0,0 +1,23 @@
+//go:build !integrationtests
+
+// Copyright 2023 The go-ethereum Authors
+// This file is part of go-ethereum.
+//
+// go-ethereum is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// go-ethereum is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import "github.com/urfave/cli/v2"
+
+var logTestCommand *cli.Command

cmd/geth/main.go

@@ -89,7 +89,6 @@ var (
 		utils.SnapshotFlag,
 		utils.TxLookupLimitFlag,
 		utils.TransactionHistoryFlag,
-		utils.StateSchemeFlag,
 		utils.StateHistoryFlag,
 		utils.LightServeFlag,
 		utils.LightIngressFlag,
@@ -146,7 +145,7 @@ var (
 		utils.GpoMaxGasPriceFlag,
 		utils.GpoIgnoreGasPriceFlag,
 		configFileFlag,
-	}, utils.NetworkFlags, utils.DatabasePathFlags)
+	}, utils.NetworkFlags, utils.DatabaseFlags)
 
 	rpcFlags = []cli.Flag{
 		utils.HTTPEnabledFlag,
@@ -236,6 +235,9 @@ func init() {
 		// See verkle.go
 		verkleCommand,
 	}
+	if logTestCommand != nil {
+		app.Commands = append(app.Commands, logTestCommand)
+	}
 	sort.Sort(cli.CommandsByName(app.Commands))
 
 	app.Flags = flags.Merge(
@@ -245,11 +247,16 @@ func init() {
 		debug.Flags,
 		metricsFlags,
 	)
+	flags.AutoEnvVars(app.Flags, "GETH")
 
 	app.Before = func(ctx *cli.Context) error {
 		maxprocs.Set() // Automatically set GOMAXPROCS to match Linux container CPU quota.
 		flags.MigrateGlobalFlags(ctx)
-		return debug.Setup(ctx)
+		if err := debug.Setup(ctx); err != nil {
+			return err
+		}
+		flags.CheckEnvVars(ctx, app.Flags, "GETH")
+		return nil
 	}
 	app.After = func(ctx *cli.Context) error {
 		debug.Exit()

cmd/geth/run_test.go

@@ -23,8 +23,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/docker/docker/pkg/reexec"
 	"github.com/ethereum/go-ethereum/internal/cmdtest"
+	"github.com/ethereum/go-ethereum/internal/reexec"
 	"github.com/ethereum/go-ethereum/rpc"
 )
 

cmd/geth/snapshot.go

@@ -51,7 +51,7 @@ var (
 				Action:    pruneState,
 				Flags: flags.Merge([]cli.Flag{
 					utils.BloomFilterSizeFlag,
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
+				}, utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth snapshot prune-state <state-root>
 will prune historical state data with the help of the state snapshot.
@@ -69,9 +69,7 @@ WARNING: it's only supported in hash mode(--state.scheme=hash)".
 				Usage:     "Recalculate state hash based on the snapshot for verification",
 				ArgsUsage: "<root>",
 				Action:    verifyState,
-				Flags: flags.Merge([]cli.Flag{
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
-					utils.StateSchemeFlag,
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
 				Description: `
 geth snapshot verify-state <state-root>
 will traverse the whole accounts and storages set based on the specified
@@ -84,7 +82,7 @@ In other words, this command does the snapshot to trie conversion.
 				Usage:     "Check that there is no 'dangling' snap storage",
 				ArgsUsage: "<root>",
 				Action:    checkDanglingStorage,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth snapshot check-dangling-storage <state-root> traverses the snap storage 
 data, and verifies that all snapshot storage data has a corresponding account. 
@@ -95,7 +93,7 @@ data, and verifies that all snapshot storage data has a corresponding account.
 				Usage:     "Check all snapshot layers for the a specific account",
 				ArgsUsage: "<address | hash>",
 				Action:    checkAccount,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth snapshot inspect-account <address | hash> checks all snapshot layers and prints out
 information about the specified address. 
@@ -106,9 +104,7 @@ information about the specified address.
 				Usage:     "Traverse the state with given root hash and perform quick verification",
 				ArgsUsage: "<root>",
 				Action:    traverseState,
-				Flags: flags.Merge([]cli.Flag{
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
-					utils.StateSchemeFlag,
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
 				Description: `
 geth snapshot traverse-state <state-root>
 will traverse the whole state from the given state root and will abort if any
@@ -123,9 +119,7 @@ It's also usable without snapshot enabled.
 				Usage:     "Traverse the state with given root hash and perform detailed verification",
 				ArgsUsage: "<root>",
 				Action:    traverseRawState,
-				Flags: flags.Merge([]cli.Flag{
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
-					utils.StateSchemeFlag,
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
 				Description: `
 geth snapshot traverse-rawstate <state-root>
 will traverse the whole state from the given root and will abort if any referenced
@@ -146,8 +140,7 @@ It's also usable without snapshot enabled.
 					utils.ExcludeStorageFlag,
 					utils.StartKeyFlag,
 					utils.DumpLimitFlag,
-					utils.StateSchemeFlag,
+				}, utils.NetworkFlags, utils.DatabaseFlags),
-				}, utils.NetworkFlags, utils.DatabasePathFlags),
 				Description: `
 This command is semantically equivalent to 'geth dump', but uses the snapshots
 as the backend data source, making this command a lot faster. 
@@ -252,7 +245,9 @@ func checkDanglingStorage(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	return snapshot.CheckDanglingStorage(utils.MakeChainDatabase(ctx, stack, true))
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+	return snapshot.CheckDanglingStorage(db)
 }
 
 // traverseState is a helper function used for pruning verification.
@@ -332,6 +327,11 @@ func traverseState(ctx *cli.Context) error {
 			storageIter := trie.NewIterator(storageIt)
 			for storageIter.Next() {
 				slots += 1
+
+				if time.Since(lastReport) > time.Second*8 {
+					log.Info("Traversing state", "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))
+					lastReport = time.Now()
+				}
 			}
 			if storageIter.Err != nil {
 				log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Err)
@@ -486,6 +486,10 @@ func traverseRawState(ctx *cli.Context) error {
 					if storageIter.Leaf() {
 						slots += 1
 					}
+					if time.Since(lastReport) > time.Second*8 {
+						log.Info("Traversing state", "nodes", nodes, "accounts", accounts, "slots", slots, "codes", codes, "elapsed", common.PrettyDuration(time.Since(start)))
+						lastReport = time.Now()
+					}
 				}
 				if storageIter.Error() != nil {
 					log.Error("Failed to traverse storage trie", "root", acc.Root, "err", storageIter.Error())

cmd/geth/testdata/logging/logtest-logfmt.txt

@@ -0,0 +1,39 @@
+t=2023-10-20T12:56:08+0200 lvl=info msg=big.Int 111,222,333,444,555,678,999=111,222,333,444,555,678,999
+t=2023-10-20T12:56:08+0200 lvl=info msg=-big.Int -111,222,333,444,555,678,999=-111,222,333,444,555,678,999
+t=2023-10-20T12:56:08+0200 lvl=info msg=big.Int  11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900
+t=2023-10-20T12:56:08+0200 lvl=info msg=-big.Int -11,122,233,344,455,567,899,900=-11,122,233,344,455,567,899,900
+t=2023-10-20T12:56:08+0200 lvl=info msg=uint256  111,222,333,444,555,678,999=111,222,333,444,555,678,999
+t=2023-10-20T12:56:08+0200 lvl=info msg=uint256  11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900
+t=2023-10-20T12:56:08+0200 lvl=info msg=int64    1,000,000=1,000,000
+t=2023-10-20T12:56:08+0200 lvl=info msg=int64    -1,000,000=-1,000,000
+t=2023-10-20T12:56:08+0200 lvl=info msg=int64    9,223,372,036,854,775,807=9,223,372,036,854,775,807
+t=2023-10-20T12:56:08+0200 lvl=info msg=int64    -9,223,372,036,854,775,808=-9,223,372,036,854,775,808
+t=2023-10-20T12:56:08+0200 lvl=info msg=uint64   1,000,000=1,000,000
+t=2023-10-20T12:56:08+0200 lvl=info msg=uint64   18,446,744,073,709,551,615=18,446,744,073,709,551,615
+t=2023-10-20T12:56:08+0200 lvl=info msg="Special chars in value" key="special \r\n\t chars"
+t=2023-10-20T12:56:08+0200 lvl=info msg="Special chars in key"   "special \n\t chars"=value
+t=2023-10-20T12:56:08+0200 lvl=info msg=nospace                  nospace=nospace
+t=2023-10-20T12:56:08+0200 lvl=info msg="with space"             "with nospace"="with nospace"
+t=2023-10-20T12:56:08+0200 lvl=info msg="Bash escapes in value"  key="\x1b[1G\x1b[K\x1b[1A"
+t=2023-10-20T12:56:08+0200 lvl=info msg="Bash escapes in key"    "\x1b[1G\x1b[K\x1b[1A"=value
+t=2023-10-20T12:56:08+0200 lvl=info msg="Bash escapes in message  \x1b[1G\x1b[K\x1b[1A end" key=value
+t=2023-10-20T12:56:08+0200 lvl=info msg="\x1b[35mColored\x1b[0m[" "\x1b[35mColored\x1b[0m["="\x1b[35mColored\x1b[0m["
+t=2023-10-20T12:56:08+0200 lvl=info msg="Custom Stringer value"   2562047h47m16.854s=2562047h47m16.854s
+t=2023-10-20T12:56:08+0200 lvl=info msg="Lazy evaluation of value" key="lazy value"
+t=2023-10-20T12:56:08+0200 lvl=info msg="A message with wonky 💩 characters"
+t=2023-10-20T12:56:08+0200 lvl=info msg="A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩"
+t=2023-10-20T12:56:08+0200 lvl=info msg="A multiline message \nLALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above"
+t=2023-10-20T12:56:08+0200 lvl=info msg=boolean                             true=true false=false
+t=2023-10-20T12:56:08+0200 lvl=info msg="repeated-key 1"                    foo=alpha foo=beta
+t=2023-10-20T12:56:08+0200 lvl=info msg="repeated-key 2"                    xx=short xx=longer
+t=2023-10-20T12:56:08+0200 lvl=info msg="log at level info"
+t=2023-10-20T12:56:08+0200 lvl=warn msg="log at level warn"
+t=2023-10-20T12:56:08+0200 lvl=eror msg="log at level error"
+t=2023-10-20T12:56:08+0200 lvl=info msg=test                                bar=short a="aligned left"
+t=2023-10-20T12:56:08+0200 lvl=info msg=test                                bar="a long message" a=1
+t=2023-10-20T12:56:08+0200 lvl=info msg=test                                bar=short            a="aligned right"
+t=2023-10-20T12:56:08+0200 lvl=info msg="The following logs should align so that the key-fields make 5 columns"
+t=2023-10-20T12:56:08+0200 lvl=info msg="Inserted known block"              number=1012 hash=0x0000000000000000000000000000000000000000000000000000000000001234 txs=200 gas=1,123,123 other=first
+t=2023-10-20T12:56:08+0200 lvl=info msg="Inserted new block"                number=1    hash=0x0000000000000000000000000000000000000000000000000000000000001235 txs=2   gas=1123      other=second
+t=2023-10-20T12:56:08+0200 lvl=info msg="Inserted known block"              number=99   hash=0x0000000000000000000000000000000000000000000000000000000000012322 txs=10  gas=1         other=third
+t=2023-10-20T12:56:08+0200 lvl=warn msg="Inserted known block"              number=1012 hash=0x0000000000000000000000000000000000000000000000000000000000001234 txs=200 gas=99        other=fourth

cmd/geth/testdata/logging/logtest-terminal.txt

@@ -0,0 +1,40 @@
+INFO [10-20|12:56:42.532] big.Int                                  111,222,333,444,555,678,999=111,222,333,444,555,678,999
+INFO [10-20|12:56:42.532] -big.Int                                 -111,222,333,444,555,678,999=-111,222,333,444,555,678,999
+INFO [10-20|12:56:42.532] big.Int                                  11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900
+INFO [10-20|12:56:42.532] -big.Int                                 -11,122,233,344,455,567,899,900=-11,122,233,344,455,567,899,900
+INFO [10-20|12:56:42.532] uint256                                  111,222,333,444,555,678,999=111,222,333,444,555,678,999
+INFO [10-20|12:56:42.532] uint256                                  11,122,233,344,455,567,899,900=11,122,233,344,455,567,899,900
+INFO [10-20|12:56:42.532] int64                                    1,000,000=1,000,000
+INFO [10-20|12:56:42.532] int64                                    -1,000,000=-1,000,000
+INFO [10-20|12:56:42.532] int64                                    9,223,372,036,854,775,807=9,223,372,036,854,775,807
+INFO [10-20|12:56:42.532] int64                                    -9,223,372,036,854,775,808=-9,223,372,036,854,775,808
+INFO [10-20|12:56:42.532] uint64                                   1,000,000=1,000,000
+INFO [10-20|12:56:42.532] uint64                                   18,446,744,073,709,551,615=18,446,744,073,709,551,615
+INFO [10-20|12:56:42.532] Special chars in value                   key="special \r\n\t chars"
+INFO [10-20|12:56:42.532] Special chars in key                     "special \n\t chars"=value
+INFO [10-20|12:56:42.532] nospace                                  nospace=nospace
+INFO [10-20|12:56:42.532] with space                               "with nospace"="with nospace"
+INFO [10-20|12:56:42.532] Bash escapes in value                    key="\x1b[1G\x1b[K\x1b[1A"
+INFO [10-20|12:56:42.532] Bash escapes in key                      "\x1b[1G\x1b[K\x1b[1A"=value
+INFO [10-20|12:56:42.532] "Bash escapes in message  \x1b[1G\x1b[K\x1b[1A end" key=value
+INFO [10-20|12:56:42.532] "\x1b[35mColored\x1b[0m["                "\x1b[35mColored\x1b[0m["="\x1b[35mColored\x1b[0m["
+INFO [10-20|12:56:42.532] Custom Stringer value                    2562047h47m16.854s=2562047h47m16.854s
+INFO [10-20|12:56:42.532] Lazy evaluation of value                 key="lazy value"
+INFO [10-20|12:56:42.532] "A message with wonky 💩 characters"
+INFO [10-20|12:56:42.532] "A multiline message \nINFO [10-18|14:11:31.106] with wonky characters 💩"
+INFO [10-20|12:56:42.532] A multiline message
+LALA [ZZZZZZZZZZZZZZZZZZ] Actually part of message above
+INFO [10-20|12:56:42.532] boolean                                  true=true false=false
+INFO [10-20|12:56:42.532] repeated-key 1                           foo=alpha foo=beta
+INFO [10-20|12:56:42.532] repeated-key 2                           xx=short xx=longer
+INFO [10-20|12:56:42.532] log at level info
+WARN [10-20|12:56:42.532] log at level warn
+ERROR[10-20|12:56:42.532] log at level error
+INFO [10-20|12:56:42.532] test                                     bar=short a="aligned left"
+INFO [10-20|12:56:42.532] test                                     bar="a long message" a=1
+INFO [10-20|12:56:42.532] test                                     bar=short            a="aligned right"
+INFO [10-20|12:56:42.532] The following logs should align so that the key-fields make 5 columns
+INFO [10-20|12:56:42.532] Inserted known block                     number=1012 hash=000000..001234 txs=200 gas=1,123,123 other=first
+INFO [10-20|12:56:42.532] Inserted new block                       number=1    hash=000000..001235 txs=2   gas=1123      other=second
+INFO [10-20|12:56:42.532] Inserted known block                     number=99   hash=000000..012322 txs=10  gas=1         other=third
+WARN [10-20|12:56:42.532] Inserted known block                     number=1012 hash=000000..001234 txs=200 gas=99        other=fourth

cmd/geth/verkle.go

@@ -45,7 +45,7 @@ var (
 				Usage:     "verify the conversion of a MPT into a verkle tree",
 				ArgsUsage: "<root>",
 				Action:    verifyVerkle,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth verkle verify <state-root>
 This command takes a root commitment and attempts to rebuild the tree.
@@ -56,7 +56,7 @@ This command takes a root commitment and attempts to rebuild the tree.
 				Usage:     "Dump a verkle tree to a DOT file",
 				ArgsUsage: "<root> <key1> [<key 2> ...]",
 				Action:    expandVerkle,
-				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabasePathFlags),
+				Flags:     flags.Merge(utils.NetworkFlags, utils.DatabaseFlags),
 				Description: `
 geth verkle dump <state-root> <key 1> [<key 2> ...]
 This command will produce a dot file representing the tree, rooted at <root>.
@@ -115,6 +115,7 @@ func verifyVerkle(ctx *cli.Context) error {
 	defer stack.Close()
 
 	chaindb := utils.MakeChainDatabase(ctx, stack, true)
+	defer chaindb.Close()
 	headBlock := rawdb.ReadHeadBlock(chaindb)
 	if headBlock == nil {
 		log.Error("Failed to load head block")
@@ -163,6 +164,7 @@ func expandVerkle(ctx *cli.Context) error {
 	defer stack.Close()
 
 	chaindb := utils.MakeChainDatabase(ctx, stack, true)
+	defer chaindb.Close()
 	var (
 		rootC   common.Hash
 		keylist [][]byte

cmd/utils/flags.go

@@ -18,7 +18,6 @@
 package utils
 
 import (
-	"bytes"
 	"context"
 	"crypto/ecdsa"
 	"encoding/hex"
@@ -39,11 +38,9 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/fdlimit"
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/txpool/legacypool"
-	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/crypto/kzg4844"
@@ -73,7 +70,6 @@ import (
 	"github.com/ethereum/go-ethereum/p2p/nat"
 	"github.com/ethereum/go-ethereum/p2p/netutil"
 	"github.com/ethereum/go-ethereum/params"
-	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/ethereum/go-ethereum/trie/triedb/hashdb"
@@ -273,7 +269,6 @@ var (
 	StateSchemeFlag = &cli.StringFlag{
 		Name:     "state.scheme",
 		Usage:    "Scheme to use for storing ethereum state ('hash' or 'path')",
-		Value:    rawdb.HashScheme,
 		Category: flags.StateCategory,
 	}
 	StateHistoryFlag = &cli.Uint64Flag{
@@ -601,9 +596,9 @@ var (
 	}
 
 	// MISC settings
-	SyncTargetFlag = &cli.PathFlag{
+	SyncTargetFlag = &cli.StringFlag{
 		Name:      "synctarget",
-		Usage:     `File for containing the hex-encoded block-rlp as sync target(dev feature)`,
+		Usage:     `Hash of the block to full sync to (dev testing feature)`,
 		TakesFile: true,
 		Category:  flags.MiscCategory,
 	}
@@ -967,21 +962,17 @@ var (
 	// NetworkFlags is the flag group of all built-in supported networks.
 	NetworkFlags = append([]cli.Flag{MainnetFlag}, TestnetFlags...)
 
-	// DatabasePathFlags is the flag group of all database path flags.
+	// DatabaseFlags is the flag group of all database flags.
-	DatabasePathFlags = []cli.Flag{
+	DatabaseFlags = []cli.Flag{
 		DataDirFlag,
 		AncientFlag,
 		RemoteDBFlag,
+		DBEngineFlag,
+		StateSchemeFlag,
 		HttpHeaderFlag,
 	}
 )
 
-func init() {
-	if rawdb.PebbleEnabled {
-		DatabasePathFlags = append(DatabasePathFlags, DBEngineFlag)
-	}
-}
-
 // MakeDataDir retrieves the currently requested data directory, terminating
 // if none (or the empty string) is specified. If the node is starting a testnet,
 // then a subdirectory of the specified datadir will be used.
@@ -1004,7 +995,7 @@ func MakeDataDir(ctx *cli.Context) string {
 
 // setNodeKey creates a node key from set command line flags, either loading it
 // from a file or as a specified hex value. If neither flags were provided, this
-// method returns nil and an emphemeral key is to be generated.
+// method returns nil and an ephemeral key is to be generated.
 func setNodeKey(ctx *cli.Context, cfg *p2p.Config) {
 	var (
 		hex  = ctx.String(NodeKeyHexFlag.Name)
@@ -1037,35 +1028,45 @@ func setNodeUserIdent(ctx *cli.Context, cfg *node.Config) {
 
 // setBootstrapNodes creates a list of bootstrap nodes from the command line
 // flags, reverting to pre-configured ones if none have been specified.
+// Priority order for bootnodes configuration:
+//
+// 1. --bootnodes flag
+// 2. Config file
+// 3. Network preset flags (e.g. --goerli)
+// 4. default to mainnet nodes
 func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) {
 	urls := params.MainnetBootnodes
-	switch {
+	if ctx.IsSet(BootnodesFlag.Name) {
-	case ctx.IsSet(BootnodesFlag.Name):
 		urls = SplitAndTrim(ctx.String(BootnodesFlag.Name))
-	case ctx.Bool(HoleskyFlag.Name):
+	} else {
-		urls = params.HoleskyBootnodes
+		if cfg.BootstrapNodes != nil {
-	case ctx.Bool(SepoliaFlag.Name):
+			return // Already set by config file, don't apply defaults.
-		urls = params.SepoliaBootnodes
+		}
-	case ctx.Bool(GoerliFlag.Name):
+		switch {
-		urls = params.GoerliBootnodes
+		case ctx.Bool(HoleskyFlag.Name):
+			urls = params.HoleskyBootnodes
+		case ctx.Bool(SepoliaFlag.Name):
+			urls = params.SepoliaBootnodes
+		case ctx.Bool(GoerliFlag.Name):
+			urls = params.GoerliBootnodes
+		}
 	}
+	cfg.BootstrapNodes = mustParseBootnodes(urls)
+}
 
-	// don't apply defaults if BootstrapNodes is already set
+func mustParseBootnodes(urls []string) []*enode.Node {
-	if cfg.BootstrapNodes != nil {
+	nodes := make([]*enode.Node, 0, len(urls))
-		return
-	}
-
-	cfg.BootstrapNodes = make([]*enode.Node, 0, len(urls))
 	for _, url := range urls {
 		if url != "" {
 			node, err := enode.Parse(enode.ValidSchemes, url)
 			if err != nil {
 				log.Crit("Bootstrap URL invalid", "enode", url, "err", err)
-				continue
+				return nil
 			}
-			cfg.BootstrapNodes = append(cfg.BootstrapNodes, node)
+			nodes = append(nodes, node)
 		}
 	}
+	return nodes
 }
 
 // setBootstrapNodesV5 creates a list of bootstrap nodes from the command line
@@ -1686,7 +1687,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	log.Debug("Sanitizing Go's GC trigger", "percent", int(gogc))
 	godebug.SetGCPercent(int(gogc))
 
-	if ctx.IsSet(SyncModeFlag.Name) {
+	if ctx.IsSet(SyncTargetFlag.Name) {
+		cfg.SyncMode = downloader.FullSync // dev sync target forces full sync
+	} else if ctx.IsSet(SyncModeFlag.Name) {
 		cfg.SyncMode = *flags.GlobalTextMarshaler(ctx, SyncModeFlag.Name).(*downloader.SyncMode)
 	}
 	if ctx.IsSet(NetworkIdFlag.Name) {
@@ -1718,15 +1721,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.IsSet(StateHistoryFlag.Name) {
 		cfg.StateHistory = ctx.Uint64(StateHistoryFlag.Name)
 	}
-	// Parse state scheme, abort the process if it's not compatible.
+	if ctx.IsSet(StateSchemeFlag.Name) {
-	chaindb := tryMakeReadOnlyDatabase(ctx, stack)
+		cfg.StateScheme = ctx.String(StateSchemeFlag.Name)
-	scheme, err := ParseStateScheme(ctx, chaindb)
-	chaindb.Close()
-	if err != nil {
-		Fatalf("%v", err)
 	}
-	cfg.StateScheme = scheme
-
 	// Parse transaction history flag, if user is still using legacy config
 	// file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'.
 	if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit {
@@ -1971,21 +1968,9 @@ func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconf
 }
 
 // RegisterFullSyncTester adds the full-sync tester service into node.
-func RegisterFullSyncTester(stack *node.Node, eth *eth.Ethereum, path string) {
+func RegisterFullSyncTester(stack *node.Node, eth *eth.Ethereum, target common.Hash) {
-	blob, err := os.ReadFile(path)
+	catalyst.RegisterFullSyncTester(stack, eth, target)
-	if err != nil {
+	log.Info("Registered full-sync tester", "hash", target)
-		Fatalf("Failed to read block file: %v", err)
-	}
-	rlpBlob, err := hexutil.Decode(string(bytes.TrimRight(blob, "\r\n")))
-	if err != nil {
-		Fatalf("Failed to decode block blob: %v", err)
-	}
-	var block types.Block
-	if err := rlp.DecodeBytes(rlpBlob, &block); err != nil {
-		Fatalf("Failed to decode block: %v", err)
-	}
-	catalyst.RegisterFullSyncTester(stack, eth, &block)
-	log.Info("Registered full-sync tester", "number", block.NumberU64(), "hash", block.Hash())
 }
 
 func SetupMetrics(ctx *cli.Context) {
@@ -2127,7 +2112,7 @@ func DialRPCWithHeaders(endpoint string, headers []string) (*rpc.Client, error)
 	}
 	var opts []rpc.ClientOption
 	if len(headers) > 0 {
-		var customHeaders = make(http.Header)
+		customHeaders := make(http.Header)
 		for _, h := range headers {
 			kv := strings.Split(h, ":")
 			if len(kv) != 2 {
@@ -2174,7 +2159,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
 	if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
 		Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
 	}
-	scheme, err := ParseStateScheme(ctx, chainDb)
+	scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), chainDb)
 	if err != nil {
 		Fatalf("%v", err)
 	}
@@ -2242,47 +2227,12 @@ func MakeConsolePreloads(ctx *cli.Context) []string {
 	return preloads
 }
 
-// ParseStateScheme resolves scheme identifier from CLI flag. If the provided
-// state scheme is not compatible with the one of persistent scheme, an error
-// will be returned.
-//
-//   - none: use the scheme consistent with persistent state, or fallback
-//     to hash-based scheme if state is empty.
-//   - hash: use hash-based scheme or error out if not compatible with
-//     persistent state scheme.
-//   - path: use path-based scheme or error out if not compatible with
-//     persistent state scheme.
-func ParseStateScheme(ctx *cli.Context, disk ethdb.Database) (string, error) {
-	// If state scheme is not specified, use the scheme consistent
-	// with persistent state, or fallback to hash mode if database
-	// is empty.
-	stored := rawdb.ReadStateScheme(disk)
-	if !ctx.IsSet(StateSchemeFlag.Name) {
-		if stored == "" {
-			// use default scheme for empty database, flip it when
-			// path mode is chosen as default
-			log.Info("State schema set to default", "scheme", "hash")
-			return rawdb.HashScheme, nil
-		}
-		log.Info("State scheme set to already existing", "scheme", stored)
-		return stored, nil // reuse scheme of persistent scheme
-	}
-	// If state scheme is specified, ensure it's compatible with
-	// persistent state.
-	scheme := ctx.String(StateSchemeFlag.Name)
-	if stored == "" || scheme == stored {
-		log.Info("State scheme set by user", "scheme", scheme)
-		return scheme, nil
-	}
-	return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, scheme)
-}
-
 // MakeTrieDatabase constructs a trie database based on the configured scheme.
 func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) *trie.Database {
 	config := &trie.Config{
 		Preimages: preimage,
 	}
-	scheme, err := ParseStateScheme(ctx, disk)
+	scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), disk)
 	if err != nil {
 		Fatalf("%v", err)
 	}

common/types.go

@@ -44,6 +44,12 @@ const (
 var (
 	hashT    = reflect.TypeOf(Hash{})
 	addressT = reflect.TypeOf(Address{})
+
+	// MaxAddress represents the maximum possible address value.
+	MaxAddress = HexToAddress("0xffffffffffffffffffffffffffffffffffffffff")
+
+	// MaxHash represents the maximum possible hash value.
+	MaxHash = HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
 )
 
 // Hash represents the 32 byte Keccak256 hash of arbitrary data.
@@ -239,9 +245,6 @@ func (a Address) Cmp(other Address) int {
 // Bytes gets the string representation of the underlying address.
 func (a Address) Bytes() []byte { return a[:] }
 
-// Hash converts an address to a hash by left-padding it with zeros.
-func (a Address) Hash() Hash { return BytesToHash(a[:]) }
-
 // Big converts an address to a big integer.
 func (a Address) Big() *big.Int { return new(big.Int).SetBytes(a[:]) }
 

core/asm/asm.go

@@ -34,7 +34,7 @@ type instructionIterator struct {
 	started bool
 }
 
-// NewInstructionIterator create a new instruction iterator.
+// NewInstructionIterator creates a new instruction iterator.
 func NewInstructionIterator(code []byte) *instructionIterator {
 	it := new(instructionIterator)
 	it.code = code

core/asm/compiler.go

@@ -17,6 +17,8 @@
 package asm
 
 import (
+	"encoding/hex"
+	"errors"
 	"fmt"
 	"math/big"
 	"os"
@@ -30,7 +32,7 @@ import (
 // and holds the tokens for the program.
 type Compiler struct {
 	tokens []token
-	binary []interface{}
+	out    []byte
 
 	labels map[string]int
 
@@ -47,15 +49,13 @@ func NewCompiler(debug bool) *Compiler {
 	}
 }
 
-// Feed feeds tokens in to ch and are interpreted by
+// Feed feeds tokens into ch and are interpreted by
 // the compiler.
 //
-// feed is the first pass in the compile stage as it
+// feed is the first pass in the compile stage as it collects the used labels in the
-// collects the used labels in the program and keeps a
+// program and keeps a program counter which is used to determine the locations of the
-// program counter which is used to determine the locations
+// jump dests. The labels can than be used in the second stage to push labels and
-// of the jump dests. The labels can than be used in the
+// determine the right position.
-// second stage to push labels and determine the right
-// position.
 func (c *Compiler) Feed(ch <-chan token) {
 	var prev token
 	for i := range ch {
@@ -79,7 +79,6 @@ func (c *Compiler) Feed(ch <-chan token) {
 				c.pc++
 			}
 		}
-
 		c.tokens = append(c.tokens, i)
 		prev = i
 	}
@@ -88,12 +87,11 @@ func (c *Compiler) Feed(ch <-chan token) {
 	}
 }
 
-// Compile compiles the current tokens and returns a
+// Compile compiles the current tokens and returns a binary string that can be interpreted
-// binary string that can be interpreted by the EVM
+// by the EVM and an error if it failed.
-// and an error if it failed.
 //
-// compile is the second stage in the compile phase
+// compile is the second stage in the compile phase which compiles the tokens to EVM
-// which compiles the tokens to EVM instructions.
+// instructions.
 func (c *Compiler) Compile() (string, []error) {
 	var errors []error
 	// continue looping over the tokens until
@@ -105,16 +103,8 @@ func (c *Compiler) Compile() (string, []error) {
 	}
 
 	// turn the binary to hex
-	var bin strings.Builder
+	h := hex.EncodeToString(c.out)
-	for _, v := range c.binary {
+	return h, errors
-		switch v := v.(type) {
-		case vm.OpCode:
-			bin.WriteString(fmt.Sprintf("%x", []byte{byte(v)}))
-		case []byte:
-			bin.WriteString(fmt.Sprintf("%x", v))
-		}
-	}
-	return bin.String(), errors
 }
 
 // next returns the next token and increments the
@@ -156,87 +146,114 @@ func (c *Compiler) compileLine() error {
 	return nil
 }
 
-// compileNumber compiles the number to bytes
+// parseNumber compiles the number to bytes
-func (c *Compiler) compileNumber(element token) {
+func parseNumber(tok token) ([]byte, error) {
-	num := math.MustParseBig256(element.text).Bytes()
+	if tok.typ != number {
-	if len(num) == 0 {
+		panic("parseNumber of non-number token")
-		num = []byte{0}
 	}
-	c.pushBin(num)
+	num, ok := math.ParseBig256(tok.text)
+	if !ok {
+		return nil, errors.New("invalid number")
+	}
+	bytes := num.Bytes()
+	if len(bytes) == 0 {
+		bytes = []byte{0}
+	}
+	return bytes, nil
 }
 
 // compileElement compiles the element (push & label or both)
 // to a binary representation and may error if incorrect statements
 // where fed.
 func (c *Compiler) compileElement(element token) error {
-	// check for a jump. jumps must be read and compiled
+	switch {
-	// from right to left.
+	case isJump(element.text):
-	if isJump(element.text) {
+		return c.compileJump(element.text)
-		rvalue := c.next()
+	case isPush(element.text):
-		switch rvalue.typ {
+		return c.compilePush()
-		case number:
+	default:
-			// TODO figure out how to return the error properly
+		c.outputOpcode(toBinary(element.text))
-			c.compileNumber(rvalue)
-		case stringValue:
-			// strings are quoted, remove them.
-			c.pushBin(rvalue.text[1 : len(rvalue.text)-2])
-		case label:
-			c.pushBin(vm.PUSH4)
-			pos := big.NewInt(int64(c.labels[rvalue.text])).Bytes()
-			pos = append(make([]byte, 4-len(pos)), pos...)
-			c.pushBin(pos)
-		case lineEnd:
-			c.pos--
-		default:
-			return compileErr(rvalue, rvalue.text, "number, string or label")
-		}
-		// push the operation
-		c.pushBin(toBinary(element.text))
 		return nil
-	} else if isPush(element.text) {
-		// handle pushes. pushes are read from left to right.
-		var value []byte
-
-		rvalue := c.next()
-		switch rvalue.typ {
-		case number:
-			value = math.MustParseBig256(rvalue.text).Bytes()
-			if len(value) == 0 {
-				value = []byte{0}
-			}
-		case stringValue:
-			value = []byte(rvalue.text[1 : len(rvalue.text)-1])
-		case label:
-			value = big.NewInt(int64(c.labels[rvalue.text])).Bytes()
-			value = append(make([]byte, 4-len(value)), value...)
-		default:
-			return compileErr(rvalue, rvalue.text, "number, string or label")
-		}
-
-		if len(value) > 32 {
-			return fmt.Errorf("%d type error: unsupported string or number with size > 32", rvalue.lineno)
-		}
-
-		c.pushBin(vm.OpCode(int(vm.PUSH1) - 1 + len(value)))
-		c.pushBin(value)
-	} else {
-		c.pushBin(toBinary(element.text))
 	}
+}
 
+func (c *Compiler) compileJump(jumpType string) error {
+	rvalue := c.next()
+	switch rvalue.typ {
+	case number:
+		numBytes, err := parseNumber(rvalue)
+		if err != nil {
+			return err
+		}
+		c.outputBytes(numBytes)
+
+	case stringValue:
+		// strings are quoted, remove them.
+		str := rvalue.text[1 : len(rvalue.text)-2]
+		c.outputBytes([]byte(str))
+
+	case label:
+		c.outputOpcode(vm.PUSH4)
+		pos := big.NewInt(int64(c.labels[rvalue.text])).Bytes()
+		pos = append(make([]byte, 4-len(pos)), pos...)
+		c.outputBytes(pos)
+
+	case lineEnd:
+		// push without argument is supported, it just takes the destination from the stack.
+		c.pos--
+
+	default:
+		return compileErr(rvalue, rvalue.text, "number, string or label")
+	}
+	// push the operation
+	c.outputOpcode(toBinary(jumpType))
+	return nil
+}
+
+func (c *Compiler) compilePush() error {
+	// handle pushes. pushes are read from left to right.
+	var value []byte
+	rvalue := c.next()
+	switch rvalue.typ {
+	case number:
+		value = math.MustParseBig256(rvalue.text).Bytes()
+		if len(value) == 0 {
+			value = []byte{0}
+		}
+	case stringValue:
+		value = []byte(rvalue.text[1 : len(rvalue.text)-1])
+	case label:
+		value = big.NewInt(int64(c.labels[rvalue.text])).Bytes()
+		value = append(make([]byte, 4-len(value)), value...)
+	default:
+		return compileErr(rvalue, rvalue.text, "number, string or label")
+	}
+	if len(value) > 32 {
+		return fmt.Errorf("%d: string or number size > 32 bytes", rvalue.lineno+1)
+	}
+	c.outputOpcode(vm.OpCode(int(vm.PUSH1) - 1 + len(value)))
+	c.outputBytes(value)
 	return nil
 }
 
 // compileLabel pushes a jumpdest to the binary slice.
 func (c *Compiler) compileLabel() {
-	c.pushBin(vm.JUMPDEST)
+	c.outputOpcode(vm.JUMPDEST)
 }
 
-// pushBin pushes the value v to the binary stack.
+func (c *Compiler) outputOpcode(op vm.OpCode) {
-func (c *Compiler) pushBin(v interface{}) {
 	if c.debug {
-		fmt.Printf("%d: %v\n", len(c.binary), v)
+		fmt.Printf("%d: %v\n", len(c.out), op)
 	}
-	c.binary = append(c.binary, v)
+	c.out = append(c.out, byte(op))
+}
+
+// output pushes the value v to the binary stack.
+func (c *Compiler) outputBytes(b []byte) {
+	if c.debug {
+		fmt.Printf("%d: %x\n", len(c.out), b)
+	}
+	c.out = append(c.out, b...)
 }
 
 // isPush returns whether the string op is either any of
@@ -263,13 +280,13 @@ type compileError struct {
 }
 
 func (err compileError) Error() string {
-	return fmt.Sprintf("%d syntax error: unexpected %v, expected %v", err.lineno, err.got, err.want)
+	return fmt.Sprintf("%d: syntax error: unexpected %v, expected %v", err.lineno, err.got, err.want)
 }
 
 func compileErr(c token, got, want string) error {
 	return compileError{
 		got:    got,
 		want:   want,
-		lineno: c.lineno,
+		lineno: c.lineno + 1,
 	}
 }

core/asm/compiler_test.go

@@ -54,6 +54,14 @@ func TestCompiler(t *testing.T) {
 `,
 			output: "6300000006565b",
 		},
+		{
+			input: `
+	JUMP @label
+label: ;; comment
+	ADD ;; comment
+`,
+			output: "6300000006565b01",
+		},
 	}
 	for _, test := range tests {
 		ch := Lex([]byte(test.input), false)

core/asm/lex_test.go

@@ -72,6 +72,16 @@ func TestLexer(t *testing.T) {
 			input:  "@label123",
 			tokens: []token{{typ: lineStart}, {typ: label, text: "label123"}, {typ: eof}},
 		},
+		// Comment after label
+		{
+			input:  "@label123 ;; comment",
+			tokens: []token{{typ: lineStart}, {typ: label, text: "label123"}, {typ: eof}},
+		},
+		// Comment after instruction
+		{
+			input:  "push 3 ;; comment\nadd",
+			tokens: []token{{typ: lineStart}, {typ: element, text: "push"}, {typ: number, text: "3"}, {typ: lineEnd, text: "\n"}, {typ: lineStart, lineno: 1}, {typ: element, lineno: 1, text: "add"}, {typ: eof, lineno: 1}},
+		},
 	}
 
 	for _, test := range tests {

core/asm/lexer.go

@@ -42,6 +42,8 @@ type token struct {
 // is able to parse and return.
 type tokenType int
 
+//go:generate go run golang.org/x/tools/cmd/stringer -type tokenType
+
 const (
 	eof              tokenType = iota // end of file
 	lineStart                         // emitted when a line starts
@@ -52,31 +54,13 @@ const (
 	labelDef                          // label definition is emitted when a new label is found
 	number                            // number is emitted when a number is found
 	stringValue                       // stringValue is emitted when a string has been found
-
-	Numbers            = "1234567890"                                           // characters representing any decimal number
-	HexadecimalNumbers = Numbers + "aAbBcCdDeEfF"                               // characters representing any hexadecimal
-	Alpha              = "abcdefghijklmnopqrstuwvxyzABCDEFGHIJKLMNOPQRSTUWVXYZ" // characters representing alphanumeric
 )
 
-// String implements stringer
+const (
-func (it tokenType) String() string {
+	decimalNumbers = "1234567890"                                           // characters representing any decimal number
-	if int(it) > len(stringtokenTypes) {
+	hexNumbers     = decimalNumbers + "aAbBcCdDeEfF"                        // characters representing any hexadecimal
-		return "invalid"
+	alpha          = "abcdefghijklmnopqrstuwvxyzABCDEFGHIJKLMNOPQRSTUWVXYZ" // characters representing alphanumeric
-	}
+)
-	return stringtokenTypes[it]
-}
-
-var stringtokenTypes = []string{
-	eof:              "EOF",
-	lineStart:        "new line",
-	lineEnd:          "end of line",
-	invalidStatement: "invalid statement",
-	element:          "element",
-	label:            "label",
-	labelDef:         "label definition",
-	number:           "number",
-	stringValue:      "string",
-}
 
 // lexer is the basic construct for parsing
 // source code and turning them in to tokens.
@@ -200,7 +184,6 @@ func lexLine(l *lexer) stateFn {
 			l.emit(lineEnd)
 			l.ignore()
 			l.lineno++
-
 			l.emit(lineStart)
 		case r == ';' && l.peek() == ';':
 			return lexComment
@@ -225,6 +208,7 @@ func lexLine(l *lexer) stateFn {
 // of the line and discards the text.
 func lexComment(l *lexer) stateFn {
 	l.acceptRunUntil('\n')
+	l.backup()
 	l.ignore()
 
 	return lexLine
@@ -234,7 +218,7 @@ func lexComment(l *lexer) stateFn {
 // the lex text state function to advance the parsing
 // process.
 func lexLabel(l *lexer) stateFn {
-	l.acceptRun(Alpha + "_" + Numbers)
+	l.acceptRun(alpha + "_" + decimalNumbers)
 
 	l.emit(label)
 
@@ -253,9 +237,9 @@ func lexInsideString(l *lexer) stateFn {
 }
 
 func lexNumber(l *lexer) stateFn {
-	acceptance := Numbers
+	acceptance := decimalNumbers
 	if l.accept("xX") {
-		acceptance = HexadecimalNumbers
+		acceptance = hexNumbers
 	}
 	l.acceptRun(acceptance)
 
@@ -265,7 +249,7 @@ func lexNumber(l *lexer) stateFn {
 }
 
 func lexElement(l *lexer) stateFn {
-	l.acceptRun(Alpha + "_" + Numbers)
+	l.acceptRun(alpha + "_" + decimalNumbers)
 
 	if l.peek() == ':' {
 		l.emit(labelDef)

core/asm/tokentype_string.go

@@ -0,0 +1,31 @@
+// Code generated by "stringer -type tokenType"; DO NOT EDIT.
+
+package asm
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[eof-0]
+	_ = x[lineStart-1]
+	_ = x[lineEnd-2]
+	_ = x[invalidStatement-3]
+	_ = x[element-4]
+	_ = x[label-5]
+	_ = x[labelDef-6]
+	_ = x[number-7]
+	_ = x[stringValue-8]
+}
+
+const _tokenType_name = "eoflineStartlineEndinvalidStatementelementlabellabelDefnumberstringValue"
+
+var _tokenType_index = [...]uint8{0, 3, 12, 19, 35, 42, 47, 55, 61, 72}
+
+func (i tokenType) String() string {
+	if i < 0 || i >= tokenType(len(_tokenType_index)-1) {
+		return "tokenType(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _tokenType_name[_tokenType_index[i]:_tokenType_index[i+1]]
+}

core/bench_test.go

@@ -84,7 +84,7 @@ func genValueTx(nbytes int) func(int, *BlockGen) {
 		toaddr := common.Address{}
 		data := make([]byte, nbytes)
 		gas, _ := IntrinsicGas(data, nil, false, false, false, false)
-		signer := types.MakeSigner(gen.config, big.NewInt(int64(i)), gen.header.Time)
+		signer := gen.Signer()
 		gasPrice := big.NewInt(0)
 		if gen.header.BaseFee != nil {
 			gasPrice = gen.header.BaseFee
@@ -128,7 +128,7 @@ func genTxRing(naccounts int) func(int, *BlockGen) {
 		if gen.header.BaseFee != nil {
 			gasPrice = gen.header.BaseFee
 		}
-		signer := types.MakeSigner(gen.config, big.NewInt(int64(i)), gen.header.Time)
+		signer := gen.Signer()
 		for {
 			gas -= params.TxGas
 			if gas < params.TxGas {

core/blockchain.go

@@ -359,31 +359,41 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	if err := bc.loadLastState(); err != nil {
 		return nil, err
 	}
-	// Make sure the state associated with the block is available
+	// Make sure the state associated with the block is available, or log out
+	// if there is no available state, waiting for state sync.
 	head := bc.CurrentBlock()
 	if !bc.HasState(head.Root) {
-		// Head state is missing, before the state recovery, find out the
+		if head.Number.Uint64() == 0 {
-		// disk layer point of snapshot(if it's enabled). Make sure the
+			// The genesis state is missing, which is only possible in the path-based
-		// rewound point is lower than disk layer.
+			// scheme. This situation occurs when the initial state sync is not finished
-		var diskRoot common.Hash
+			// yet, or the chain head is rewound below the pivot point. In both scenarios,
-		if bc.cacheConfig.SnapshotLimit > 0 {
+			// there is no possible recovery approach except for rerunning a snap sync.
-			diskRoot = rawdb.ReadSnapshotRoot(bc.db)
+			// Do nothing here until the state syncer picks it up.
-		}
+			log.Info("Genesis state is missing, wait state sync")
-		if diskRoot != (common.Hash{}) {
-			log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash(), "snaproot", diskRoot)
-
-			snapDisk, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, diskRoot, true)
-			if err != nil {
-				return nil, err
-			}
-			// Chain rewound, persist old snapshot number to indicate recovery procedure
-			if snapDisk != 0 {
-				rawdb.WriteSnapshotRecoveryNumber(bc.db, snapDisk)
-			}
 		} else {
-			log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash())
+			// Head state is missing, before the state recovery, find out the
-			if _, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, common.Hash{}, true); err != nil {
+			// disk layer point of snapshot(if it's enabled). Make sure the
-				return nil, err
+			// rewound point is lower than disk layer.
+			var diskRoot common.Hash
+			if bc.cacheConfig.SnapshotLimit > 0 {
+				diskRoot = rawdb.ReadSnapshotRoot(bc.db)
+			}
+			if diskRoot != (common.Hash{}) {
+				log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash(), "snaproot", diskRoot)
+
+				snapDisk, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, diskRoot, true)
+				if err != nil {
+					return nil, err
+				}
+				// Chain rewound, persist old snapshot number to indicate recovery procedure
+				if snapDisk != 0 {
+					rawdb.WriteSnapshotRecoveryNumber(bc.db, snapDisk)
+				}
+			} else {
+				log.Warn("Head state missing, repairing", "number", head.Number, "hash", head.Hash())
+				if _, err := bc.setHeadBeyondRoot(head.Number.Uint64(), 0, common.Hash{}, true); err != nil {
+					return nil, err
+				}
 			}
 		}
 	}
@@ -603,7 +613,7 @@ func (bc *BlockChain) SetHead(head uint64) error {
 	header := bc.CurrentBlock()
 	block := bc.GetBlock(header.Hash(), header.Number.Uint64())
 	if block == nil {
-		// This should never happen. In practice, previsouly currentBlock
+		// This should never happen. In practice, previously currentBlock
 		// contained the entire block whereas now only a "marker", so there
 		// is an ever so slight chance for a race we should handle.
 		log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash())
@@ -625,7 +635,7 @@ func (bc *BlockChain) SetHeadWithTimestamp(timestamp uint64) error {
 	header := bc.CurrentBlock()
 	block := bc.GetBlock(header.Hash(), header.Number.Uint64())
 	if block == nil {
-		// This should never happen. In practice, previsouly currentBlock
+		// This should never happen. In practice, previously currentBlock
 		// contained the entire block whereas now only a "marker", so there
 		// is an ever so slight chance for a race we should handle.
 		log.Error("Current block not found in database", "block", header.Number, "hash", header.Hash())
@@ -683,25 +693,6 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 	pivot := rawdb.ReadLastPivotNumber(bc.db)
 	frozen, _ := bc.db.Ancients()
 
-	// resetState resets the persistent state to genesis if it's not available.
-	resetState := func() {
-		// Short circuit if the genesis state is already present.
-		if bc.HasState(bc.genesisBlock.Root()) {
-			return
-		}
-		// Reset the state database to empty for committing genesis state.
-		// Note, it should only happen in path-based scheme and Reset function
-		// is also only call-able in this mode.
-		if bc.triedb.Scheme() == rawdb.PathScheme {
-			if err := bc.triedb.Reset(types.EmptyRootHash); err != nil {
-				log.Crit("Failed to clean state", "err", err) // Shouldn't happen
-			}
-		}
-		// Write genesis state into database.
-		if err := CommitGenesisState(bc.db, bc.triedb, bc.genesisBlock.Hash()); err != nil {
-			log.Crit("Failed to commit genesis state", "err", err)
-		}
-	}
 	updateFn := func(db ethdb.KeyValueWriter, header *types.Header) (*types.Header, bool) {
 		// Rewind the blockchain, ensuring we don't end up with a stateless head
 		// block. Note, depth equality is permitted to allow using SetHead as a
@@ -711,11 +702,9 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 			if newHeadBlock == nil {
 				log.Error("Gap in the chain, rewinding to genesis", "number", header.Number, "hash", header.Hash())
 				newHeadBlock = bc.genesisBlock
-				resetState()
 			} else {
-				// Block exists, keep rewinding until we find one with state,
+				// Block exists. Keep rewinding until either we find one with state
-				// keeping rewinding until we exceed the optional threshold
+				// or until we exceed the optional threshold root hash
-				// root hash
 				beyondRoot := (root == common.Hash{}) // Flag whether we're beyond the requested root (no root, always true)
 
 				for {
@@ -739,16 +728,14 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 						}
 					}
 					if beyondRoot || newHeadBlock.NumberU64() == 0 {
-						if newHeadBlock.NumberU64() == 0 {
+						if !bc.HasState(newHeadBlock.Root()) && bc.stateRecoverable(newHeadBlock.Root()) {
-							resetState()
-						} else if !bc.HasState(newHeadBlock.Root()) {
 							// Rewind to a block with recoverable state. If the state is
 							// missing, run the state recovery here.
 							if err := bc.triedb.Recover(newHeadBlock.Root()); err != nil {
 								log.Crit("Failed to rollback state", "err", err) // Shouldn't happen
 							}
+							log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
 						}
-						log.Debug("Rewound to block with state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash())
 						break
 					}
 					log.Debug("Skipping block with threshold state", "number", newHeadBlock.NumberU64(), "hash", newHeadBlock.Hash(), "root", newHeadBlock.Root())
@@ -763,6 +750,15 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 			// to low, so it's safe to update in-memory markers directly.
 			bc.currentBlock.Store(newHeadBlock.Header())
 			headBlockGauge.Update(int64(newHeadBlock.NumberU64()))
+
+			// The head state is missing, which is only possible in the path-based
+			// scheme. This situation occurs when the chain head is rewound below
+			// the pivot point. In this scenario, there is no possible recovery
+			// approach except for rerunning a snap sync. Do nothing here until the
+			// state syncer picks it up.
+			if !bc.HasState(newHeadBlock.Root()) {
+				log.Info("Chain is stateless, wait state sync", "number", newHeadBlock.Number(), "hash", newHeadBlock.Hash())
+			}
 		}
 		// Rewind the snap block in a simpleton way to the target head
 		if currentSnapBlock := bc.CurrentSnapBlock(); currentSnapBlock != nil && header.Number.Uint64() < currentSnapBlock.Number.Uint64() {
@@ -862,7 +858,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error {
 	// Reset the trie database with the fresh snap synced state.
 	root := block.Root()
 	if bc.triedb.Scheme() == rawdb.PathScheme {
-		if err := bc.triedb.Reset(root); err != nil {
+		if err := bc.triedb.Enable(root); err != nil {
 			return err
 		}
 	}
@@ -1022,13 +1018,14 @@ func (bc *BlockChain) stopWithoutSaving() {
 func (bc *BlockChain) Stop() {
 	bc.stopWithoutSaving()
 
-	// Ensure that the entirety of the state snapshot is journalled to disk.
+	// Ensure that the entirety of the state snapshot is journaled to disk.
 	var snapBase common.Hash
 	if bc.snaps != nil {
 		var err error
 		if snapBase, err = bc.snaps.Journal(bc.CurrentBlock().Root); err != nil {
 			log.Error("Failed to journal state snapshot", "err", err)
 		}
+		bc.snaps.Release()
 	}
 	if bc.triedb.Scheme() == rawdb.PathScheme {
 		// Ensure that the in-memory trie nodes are journaled to disk properly.
@@ -1233,7 +1230,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
 		// a background routine to re-indexed all indices in [ancients - txlookupLimit, ancients)
 		// range. In this case, all tx indices of newly imported blocks should be
 		// generated.
-		var batch = bc.db.NewBatch()
+		batch := bc.db.NewBatch()
 		for i, block := range blockChain {
 			if bc.txLookupLimit == 0 || ancientLimit <= bc.txLookupLimit || block.NumberU64() >= ancientLimit-bc.txLookupLimit {
 				rawdb.WriteTxLookupEntriesByBlock(batch, block)
@@ -2643,7 +2640,7 @@ func (bc *BlockChain) SetTrieFlushInterval(interval time.Duration) {
 	bc.flushInterval.Store(int64(interval))
 }
 
-// GetTrieFlushInterval gets the in-memroy tries flush interval
+// GetTrieFlushInterval gets the in-memory tries flush interval
 func (bc *BlockChain) GetTrieFlushInterval() time.Duration {
 	return time.Duration(bc.flushInterval.Load())
 }

core/blockchain_test.go

@@ -48,7 +48,7 @@ var (
 )
 
 // newCanonical creates a chain database, and injects a deterministic canonical
-// chain. Depending on the full flag, if creates either a full block chain or a
+// chain. Depending on the full flag, it creates either a full block chain or a
 // header only chain. The database and genesis specification for block generation
 // are also returned in case more test blocks are needed later.
 func newCanonical(engine consensus.Engine, n int, full bool, scheme string) (ethdb.Database, *Genesis, *BlockChain, error) {

core/bloombits/matcher.go

@@ -58,7 +58,7 @@ type partialMatches struct {
 // bit with the given number of fetch elements, or a response for such a request.
 // It can also have the actual results set to be used as a delivery data struct.
 //
-// The contest and error fields are used by the light client to terminate matching
+// The context and error fields are used by the light client to terminate matching
 // early if an error is encountered on some path of the pipeline.
 type Retrieval struct {
 	Bit      uint
@@ -389,7 +389,7 @@ func (m *Matcher) distributor(dist chan *request, session *MatcherSession) {
 		shutdown   = session.quit            // Shutdown request channel, will gracefully wait for pending requests
 	)
 
-	// assign is a helper method fo try to assign a pending bit an actively
+	// assign is a helper method to try to assign a pending bit an actively
 	// listening servicer, or schedule it up for later when one arrives.
 	assign := func(bit uint) {
 		select {
@@ -630,13 +630,16 @@ func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan
 			request <- &Retrieval{Bit: bit, Sections: sections, Context: s.ctx}
 
 			result := <-request
+
+			// Deliver a result before s.Close() to avoid a deadlock
+			s.deliverSections(result.Bit, result.Sections, result.Bitsets)
+
 			if result.Error != nil {
 				s.errLock.Lock()
 				s.err = result.Error
 				s.errLock.Unlock()
 				s.Close()
 			}
-			s.deliverSections(result.Bit, result.Sections, result.Bitsets)
 		}
 	}
 }

core/bloombits/matcher_test.go

@@ -85,7 +85,7 @@ func TestMatcherRandom(t *testing.T) {
 }
 
 // Tests that the matcher can properly find matches if the starting block is
-// shifter from a multiple of 8. This is needed to cover an optimisation with
+// shifted from a multiple of 8. This is needed to cover an optimisation with
 // bitset matching https://github.com/ethereum/go-ethereum/issues/15309.
 func TestMatcherShifted(t *testing.T) {
 	t.Parallel()
@@ -106,7 +106,7 @@ func TestWildcardMatcher(t *testing.T) {
 	testMatcherBothModes(t, nil, 0, 10000, 0)
 }
 
-// makeRandomIndexes generates a random filter system, composed on multiple filter
+// makeRandomIndexes generates a random filter system, composed of multiple filter
 // criteria, each having one bloom list component for the address and arbitrarily
 // many topic bloom list components.
 func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {

core/chain_makers.go

@@ -38,8 +38,8 @@ import (
 // See GenerateChain for a detailed explanation.
 type BlockGen struct {
 	i       int
+	cm      *chainMaker
 	parent  *types.Block
-	chain   []*types.Block
 	header  *types.Header
 	statedb *state.StateDB
 
@@ -49,7 +49,6 @@ type BlockGen struct {
 	uncles      []*types.Header
 	withdrawals []*types.Withdrawal
 
-	config *params.ChainConfig
 	engine consensus.Engine
 }
 
@@ -88,9 +87,20 @@ func (b *BlockGen) SetPoS() {
 	b.header.Difficulty = new(big.Int)
 }
 
-// SetBlobGas sets the data gas used by the blob in the generated block.
+// Difficulty returns the currently calculated difficulty of the block.
-func (b *BlockGen) SetBlobGas(blobGasUsed uint64) {
+func (b *BlockGen) Difficulty() *big.Int {
-	b.header.BlobGasUsed = &blobGasUsed
+	return new(big.Int).Set(b.header.Difficulty)
+}
+
+// SetParentBeaconRoot sets the parent beacon root field of the generated
+// block.
+func (b *BlockGen) SetParentBeaconRoot(root common.Hash) {
+	b.header.ParentBeaconRoot = &root
+	var (
+		blockContext = NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase)
+		vmenv        = vm.NewEVM(blockContext, vm.TxContext{}, b.statedb, b.cm.config, vm.Config{})
+	)
+	ProcessBeaconBlockRoot(root, vmenv, b.statedb)
 }
 
 // addTx adds a transaction to the generated block. If no coinbase has
@@ -105,22 +115,25 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti
 		b.SetCoinbase(common.Address{})
 	}
 	b.statedb.SetTxContext(tx.Hash(), len(b.txs))
-	receipt, err := ApplyTransaction(b.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vmConfig)
+	receipt, err := ApplyTransaction(b.cm.config, bc, &b.header.Coinbase, b.gasPool, b.statedb, b.header, tx, &b.header.GasUsed, vmConfig)
 	if err != nil {
 		panic(err)
 	}
 	b.txs = append(b.txs, tx)
 	b.receipts = append(b.receipts, receipt)
+	if b.header.BlobGasUsed != nil {
+		*b.header.BlobGasUsed += receipt.BlobGasUsed
+	}
 }
 
 // AddTx adds a transaction to the generated block. If no coinbase has
 // been set, the block's coinbase is set to the zero address.
 //
-// AddTx panics if the transaction cannot be executed. In addition to
+// AddTx panics if the transaction cannot be executed. In addition to the protocol-imposed
-// the protocol-imposed limitations (gas limit, etc.), there are some
+// limitations (gas limit, etc.), there are some further limitations on the content of
-// further limitations on the content of transactions that can be
+// transactions that can be added. Notably, contract code relying on the BLOCKHASH
-// added. Notably, contract code relying on the BLOCKHASH instruction
+// instruction will panic during execution if it attempts to access a block number outside
-// will panic during execution.
+// of the range created by GenerateChain.
 func (b *BlockGen) AddTx(tx *types.Transaction) {
 	b.addTx(nil, vm.Config{}, tx)
 }
@@ -128,11 +141,10 @@ func (b *BlockGen) AddTx(tx *types.Transaction) {
 // AddTxWithChain adds a transaction to the generated block. If no coinbase has
 // been set, the block's coinbase is set to the zero address.
 //
-// AddTxWithChain panics if the transaction cannot be executed. In addition to
+// AddTxWithChain panics if the transaction cannot be executed. In addition to the
-// the protocol-imposed limitations (gas limit, etc.), there are some
+// protocol-imposed limitations (gas limit, etc.), there are some further limitations on
-// further limitations on the content of transactions that can be
+// the content of transactions that can be added. If contract code relies on the BLOCKHASH
-// added. If contract code relies on the BLOCKHASH instruction,
+// instruction, the block in chain will be returned.
-// the block in chain will be returned.
 func (b *BlockGen) AddTxWithChain(bc *BlockChain, tx *types.Transaction) {
 	b.addTx(bc, vm.Config{}, tx)
 }
@@ -149,8 +161,7 @@ func (b *BlockGen) GetBalance(addr common.Address) *big.Int {
 	return b.statedb.GetBalance(addr)
 }
 
-// AddUncheckedTx forcefully adds a transaction to the block without any
+// AddUncheckedTx forcefully adds a transaction to the block without any validation.
-// validation.
 //
 // AddUncheckedTx will cause consensus failures when used during real
 // chain processing. This is best used in conjunction with raw block insertion.
@@ -173,6 +184,16 @@ func (b *BlockGen) BaseFee() *big.Int {
 	return new(big.Int).Set(b.header.BaseFee)
 }
 
+// Gas returns the amount of gas left in the current block.
+func (b *BlockGen) Gas() uint64 {
+	return b.header.GasLimit - b.header.GasUsed
+}
+
+// Signer returns a valid signer instance for the current block.
+func (b *BlockGen) Signer() types.Signer {
+	return types.MakeSigner(b.cm.config, b.header.Number, b.header.Time)
+}
+
 // AddUncheckedReceipt forcefully adds a receipts to the block without a
 // backing transaction.
 //
@@ -198,20 +219,19 @@ func (b *BlockGen) AddUncle(h *types.Header) {
 
 	var parent *types.Header
 	for i := b.i - 1; i >= 0; i-- {
-		if b.chain[i].Hash() == h.ParentHash {
+		if b.cm.chain[i].Hash() == h.ParentHash {
-			parent = b.chain[i].Header()
+			parent = b.cm.chain[i].Header()
 			break
 		}
 	}
-	chainreader := &fakeChainReader{config: b.config}
+	h.Difficulty = b.engine.CalcDifficulty(b.cm, b.header.Time, parent)
-	h.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time, parent)
 
 	// The gas limit and price should be derived from the parent
 	h.GasLimit = parent.GasLimit
-	if b.config.IsLondon(h.Number) {
+	if b.cm.config.IsLondon(h.Number) {
-		h.BaseFee = eip1559.CalcBaseFee(b.config, parent)
+		h.BaseFee = eip1559.CalcBaseFee(b.cm.config, parent)
-		if !b.config.IsLondon(parent.Number) {
+		if !b.cm.config.IsLondon(parent.Number) {
-			parentGasLimit := parent.GasLimit * b.config.ElasticityMultiplier()
+			parentGasLimit := parent.GasLimit * b.cm.config.ElasticityMultiplier()
 			h.GasLimit = CalcGasLimit(parentGasLimit, parentGasLimit)
 		}
 	}
@@ -233,12 +253,12 @@ func (b *BlockGen) nextWithdrawalIndex() uint64 {
 		return b.withdrawals[len(b.withdrawals)-1].Index + 1
 	}
 	for i := b.i - 1; i >= 0; i-- {
-		if wd := b.chain[i].Withdrawals(); len(wd) != 0 {
+		if wd := b.cm.chain[i].Withdrawals(); len(wd) != 0 {
 			return wd[len(wd)-1].Index + 1
 		}
 		if i == 0 {
 			// Correctly set the index if no parent had withdrawals.
-			if wd := b.parent.Withdrawals(); len(wd) != 0 {
+			if wd := b.cm.bottom.Withdrawals(); len(wd) != 0 {
 				return wd[len(wd)-1].Index + 1
 			}
 		}
@@ -254,9 +274,9 @@ func (b *BlockGen) PrevBlock(index int) *types.Block {
 		panic(fmt.Errorf("block index %d out of range (%d,%d)", index, -1, b.i))
 	}
 	if index == -1 {
-		return b.parent
+		return b.cm.bottom
 	}
-	return b.chain[index]
+	return b.cm.chain[index]
 }
 
 // OffsetTime modifies the time instance of a block, implicitly changing its
@@ -264,11 +284,10 @@ func (b *BlockGen) PrevBlock(index int) *types.Block {
 // tied to chain length directly.
 func (b *BlockGen) OffsetTime(seconds int64) {
 	b.header.Time += uint64(seconds)
-	if b.header.Time <= b.parent.Header().Time {
+	if b.header.Time <= b.cm.bottom.Header().Time {
 		panic("block time out of range")
 	}
-	chainreader := &fakeChainReader{config: b.config}
+	b.header.Difficulty = b.engine.CalcDifficulty(b.cm, b.header.Time, b.parent.Header())
-	b.header.Difficulty = b.engine.CalcDifficulty(chainreader, b.header.Time, b.parent.Header())
 }
 
 // GenerateChain creates a chain of n blocks. The first block's
@@ -287,11 +306,14 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 	if config == nil {
 		config = params.TestChainConfig
 	}
-	blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
+	if engine == nil {
-	chainreader := &fakeChainReader{config: config}
+		panic("nil consensus engine")
+	}
+	cm := newChainMaker(parent, config, engine)
+
 	genblock := func(i int, parent *types.Block, triedb *trie.Database, statedb *state.StateDB) (*types.Block, types.Receipts) {
-		b := &BlockGen{i: i, chain: blocks, parent: parent, statedb: statedb, config: config, engine: engine}
+		b := &BlockGen{i: i, cm: cm, parent: parent, statedb: statedb, engine: engine}
-		b.header = makeHeader(chainreader, parent, statedb, b.engine)
+		b.header = cm.makeHeader(parent, statedb, b.engine)
 
 		// Set the difficulty for clique block. The chain maker doesn't have access
 		// to a chain, so the difficulty will be left unset (nil). Set it here to the
@@ -321,24 +343,23 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		if gen != nil {
 			gen(i, b)
 		}
-		if b.engine != nil {
-			block, err := b.engine.FinalizeAndAssemble(chainreader, b.header, statedb, b.txs, b.uncles, b.receipts, b.withdrawals)
-			if err != nil {
-				panic(err)
-			}
 
-			// Write state changes to db
+		block, err := b.engine.FinalizeAndAssemble(cm, b.header, statedb, b.txs, b.uncles, b.receipts, b.withdrawals)
-			root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number))
+		if err != nil {
-			if err != nil {
+			panic(err)
-				panic(fmt.Sprintf("state write error: %v", err))
-			}
-			if err = triedb.Commit(root, false); err != nil {
-				panic(fmt.Sprintf("trie write error: %v", err))
-			}
-			return block, b.receipts
 		}
-		return nil, nil
+
+		// Write state changes to db
+		root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number))
+		if err != nil {
+			panic(fmt.Sprintf("state write error: %v", err))
+		}
+		if err = triedb.Commit(root, false); err != nil {
+			panic(fmt.Sprintf("trie write error: %v", err))
+		}
+		return block, b.receipts
 	}
+
 	// Forcibly use hash-based state scheme for retaining all nodes in disk.
 	triedb := trie.NewDatabase(db, trie.HashDefaults)
 	defer triedb.Close()
@@ -348,12 +369,36 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 		if err != nil {
 			panic(err)
 		}
-		block, receipt := genblock(i, parent, triedb, statedb)
+		block, receipts := genblock(i, parent, triedb, statedb)
-		blocks[i] = block
+
-		receipts[i] = receipt
+		// Post-process the receipts.
+		// Here we assign the final block hash and other info into the receipt.
+		// In order for DeriveFields to work, the transaction and receipt lists need to be
+		// of equal length. If AddUncheckedTx or AddUncheckedReceipt are used, there will be
+		// extra ones, so we just trim the lists here.
+		receiptsCount := len(receipts)
+		txs := block.Transactions()
+		if len(receipts) > len(txs) {
+			receipts = receipts[:len(txs)]
+		} else if len(receipts) < len(txs) {
+			txs = txs[:len(receipts)]
+		}
+		var blobGasPrice *big.Int
+		if block.ExcessBlobGas() != nil {
+			blobGasPrice = eip4844.CalcBlobFee(*block.ExcessBlobGas())
+		}
+		if err := receipts.DeriveFields(config, block.Hash(), block.NumberU64(), block.Time(), block.BaseFee(), blobGasPrice, txs); err != nil {
+			panic(err)
+		}
+
+		// Re-expand to ensure all receipts are returned.
+		receipts = receipts[:receiptsCount]
+
+		// Advance the chain.
+		cm.add(block, receipts)
 		parent = block
 	}
-	return blocks, receipts
+	return cm.chain, cm.receipts
 }
 
 // GenerateChainWithGenesis is a wrapper of GenerateChain which will initialize
@@ -372,35 +417,26 @@ func GenerateChainWithGenesis(genesis *Genesis, engine consensus.Engine, n int,
 	return db, blocks, receipts
 }
 
-func makeHeader(chain consensus.ChainReader, parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header {
+func (cm *chainMaker) makeHeader(parent *types.Block, state *state.StateDB, engine consensus.Engine) *types.Header {
-	var time uint64
+	time := parent.Time() + 10 // block time is fixed at 10 seconds
-	if parent.Time() == 0 {
-		time = 10
-	} else {
-		time = parent.Time() + 10 // block time is fixed at 10 seconds
-	}
 	header := &types.Header{
-		Root:       state.IntermediateRoot(chain.Config().IsEIP158(parent.Number())),
+		Root:       state.IntermediateRoot(cm.config.IsEIP158(parent.Number())),
 		ParentHash: parent.Hash(),
 		Coinbase:   parent.Coinbase(),
-		Difficulty: engine.CalcDifficulty(chain, time, &types.Header{
+		Difficulty: engine.CalcDifficulty(cm, time, parent.Header()),
-			Number:     parent.Number(),
+		GasLimit:   parent.GasLimit(),
-			Time:       time - 10,
+		Number:     new(big.Int).Add(parent.Number(), common.Big1),
-			Difficulty: parent.Difficulty(),
+		Time:       time,
-			UncleHash:  parent.UncleHash(),
-		}),
-		GasLimit: parent.GasLimit(),
-		Number:   new(big.Int).Add(parent.Number(), common.Big1),
-		Time:     time,
 	}
-	if chain.Config().IsLondon(header.Number) {
+
-		header.BaseFee = eip1559.CalcBaseFee(chain.Config(), parent.Header())
+	if cm.config.IsLondon(header.Number) {
-		if !chain.Config().IsLondon(parent.Number()) {
+		header.BaseFee = eip1559.CalcBaseFee(cm.config, parent.Header())
-			parentGasLimit := parent.GasLimit() * chain.Config().ElasticityMultiplier()
+		if !cm.config.IsLondon(parent.Number()) {
+			parentGasLimit := parent.GasLimit() * cm.config.ElasticityMultiplier()
 			header.GasLimit = CalcGasLimit(parentGasLimit, parentGasLimit)
 		}
 	}
-	if chain.Config().IsCancun(header.Number, header.Time) {
+	if cm.config.IsCancun(header.Number, header.Time) {
 		var (
 			parentExcessBlobGas uint64
 			parentBlobGasUsed   uint64
@@ -453,18 +489,86 @@ func makeBlockChainWithGenesis(genesis *Genesis, n int, engine consensus.Engine,
 	return db, blocks
 }
 
-type fakeChainReader struct {
+// chainMaker contains the state of chain generation.
-	config *params.ChainConfig
+type chainMaker struct {
+	bottom      *types.Block
+	engine      consensus.Engine
+	config      *params.ChainConfig
+	chain       []*types.Block
+	chainByHash map[common.Hash]*types.Block
+	receipts    []types.Receipts
 }
 
-// Config returns the chain configuration.
+func newChainMaker(bottom *types.Block, config *params.ChainConfig, engine consensus.Engine) *chainMaker {
-func (cr *fakeChainReader) Config() *params.ChainConfig {
+	return &chainMaker{
-	return cr.config
+		bottom:      bottom,
+		config:      config,
+		engine:      engine,
+		chainByHash: make(map[common.Hash]*types.Block),
+	}
 }
 
-func (cr *fakeChainReader) CurrentHeader() *types.Header                            { return nil }
+func (cm *chainMaker) add(b *types.Block, r []*types.Receipt) {
-func (cr *fakeChainReader) GetHeaderByNumber(number uint64) *types.Header           { return nil }
+	cm.chain = append(cm.chain, b)
-func (cr *fakeChainReader) GetHeaderByHash(hash common.Hash) *types.Header          { return nil }
+	cm.chainByHash[b.Hash()] = b
-func (cr *fakeChainReader) GetHeader(hash common.Hash, number uint64) *types.Header { return nil }
+	cm.receipts = append(cm.receipts, r)
-func (cr *fakeChainReader) GetBlock(hash common.Hash, number uint64) *types.Block   { return nil }
+}
-func (cr *fakeChainReader) GetTd(hash common.Hash, number uint64) *big.Int          { return nil }
+
+func (cm *chainMaker) blockByNumber(number uint64) *types.Block {
+	if number == cm.bottom.NumberU64() {
+		return cm.bottom
+	}
+	cur := cm.CurrentHeader().Number.Uint64()
+	lowest := cm.bottom.NumberU64() + 1
+	if number < lowest || number > cur {
+		return nil
+	}
+	return cm.chain[number-lowest]
+}
+
+// ChainReader/ChainContext implementation
+
+// Config returns the chain configuration (for consensus.ChainReader).
+func (cm *chainMaker) Config() *params.ChainConfig {
+	return cm.config
+}
+
+// Engine returns the consensus engine (for ChainContext).
+func (cm *chainMaker) Engine() consensus.Engine {
+	return cm.engine
+}
+
+func (cm *chainMaker) CurrentHeader() *types.Header {
+	if len(cm.chain) == 0 {
+		return cm.bottom.Header()
+	}
+	return cm.chain[len(cm.chain)-1].Header()
+}
+
+func (cm *chainMaker) GetHeaderByNumber(number uint64) *types.Header {
+	b := cm.blockByNumber(number)
+	if b == nil {
+		return nil
+	}
+	return b.Header()
+}
+
+func (cm *chainMaker) GetHeaderByHash(hash common.Hash) *types.Header {
+	b := cm.chainByHash[hash]
+	if b == nil {
+		return nil
+	}
+	return b.Header()
+}
+
+func (cm *chainMaker) GetHeader(hash common.Hash, number uint64) *types.Header {
+	return cm.GetHeaderByNumber(number)
+}
+
+func (cm *chainMaker) GetBlock(hash common.Hash, number uint64) *types.Block {
+	return cm.blockByNumber(number)
+}
+
+func (cm *chainMaker) GetTd(hash common.Hash, number uint64) *big.Int {
+	return nil // not supported
+}

core/chain_makers_test.go

@@ -19,8 +19,10 @@ package core
 import (
 	"fmt"
 	"math/big"
+	"reflect"
 	"testing"
 
+	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus/beacon"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
@@ -32,7 +34,7 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 )
 
-func TestGenerateWithdrawalChain(t *testing.T) {
+func TestGeneratePOSChain(t *testing.T) {
 	var (
 		keyHex  = "9c647b8b7c4e7c3490668fb6c11473619db80c93704c70893d3813af4090c39c"
 		key, _  = crypto.HexToECDSA(keyHex)
@@ -41,21 +43,25 @@ func TestGenerateWithdrawalChain(t *testing.T) {
 		bb      = common.Address{0xbb}
 		funds   = big.NewInt(0).Mul(big.NewInt(1337), big.NewInt(params.Ether))
 		config  = *params.AllEthashProtocolChanges
+		asm4788 = common.Hex2Bytes("3373fffffffffffffffffffffffffffffffffffffffe14604d57602036146024575f5ffd5b5f35801560495762001fff810690815414603c575f5ffd5b62001fff01545f5260205ff35b5f5ffd5b62001fff42064281555f359062001fff015500")
 		gspec   = &Genesis{
-			Config:     &config,
+			Config: &config,
-			Alloc:      GenesisAlloc{address: {Balance: funds}},
+			Alloc: GenesisAlloc{
+				address:                          {Balance: funds},
+				params.BeaconRootsStorageAddress: {Balance: common.Big0, Code: asm4788},
+			},
 			BaseFee:    big.NewInt(params.InitialBaseFee),
 			Difficulty: common.Big1,
 			GasLimit:   5_000_000,
 		}
-		gendb  = rawdb.NewMemoryDatabase()
+		gendb = rawdb.NewMemoryDatabase()
-		signer = types.LatestSigner(gspec.Config)
+		db    = rawdb.NewMemoryDatabase()
-		db     = rawdb.NewMemoryDatabase()
 	)
 
 	config.TerminalTotalDifficultyPassed = true
 	config.TerminalTotalDifficulty = common.Big0
 	config.ShanghaiTime = u64(0)
+	config.CancunTime = u64(0)
 
 	// init 0xaa with some storage elements
 	storage := make(map[common.Hash]common.Hash)
@@ -77,9 +83,20 @@ func TestGenerateWithdrawalChain(t *testing.T) {
 	}
 	genesis := gspec.MustCommit(gendb, trie.NewDatabase(gendb, trie.HashDefaults))
 
-	chain, _ := GenerateChain(gspec.Config, genesis, beacon.NewFaker(), gendb, 4, func(i int, gen *BlockGen) {
+	genchain, genreceipts := GenerateChain(gspec.Config, genesis, beacon.NewFaker(), gendb, 4, func(i int, gen *BlockGen) {
-		tx, _ := types.SignTx(types.NewTransaction(gen.TxNonce(address), address, big.NewInt(1000), params.TxGas, new(big.Int).Add(gen.BaseFee(), common.Big1), nil), signer, key)
+		gen.SetParentBeaconRoot(common.Hash{byte(i + 1)})
+
+		// Add value transfer tx.
+		tx := types.MustSignNewTx(key, gen.Signer(), &types.LegacyTx{
+			Nonce:    gen.TxNonce(address),
+			To:       &address,
+			Value:    big.NewInt(1000),
+			Gas:      params.TxGas,
+			GasPrice: new(big.Int).Add(gen.BaseFee(), common.Big1),
+		})
 		gen.AddTx(tx)
+
+		// Add withdrawals.
 		if i == 1 {
 			gen.AddWithdrawal(&types.Withdrawal{
 				Validator: 42,
@@ -110,21 +127,42 @@ func TestGenerateWithdrawalChain(t *testing.T) {
 	blockchain, _ := NewBlockChain(db, nil, gspec, nil, beacon.NewFaker(), vm.Config{}, nil, nil)
 	defer blockchain.Stop()
 
-	if i, err := blockchain.InsertChain(chain); err != nil {
+	if i, err := blockchain.InsertChain(genchain); err != nil {
-		fmt.Printf("insert error (block %d): %v\n", chain[i].NumberU64(), err)
+		t.Fatalf("insert error (block %d): %v\n", genchain[i].NumberU64(), err)
-		return
 	}
 
 	// enforce that withdrawal indexes are monotonically increasing from 0
 	var (
 		withdrawalIndex uint64
-		head            = blockchain.CurrentBlock().Number.Uint64()
 	)
-	for i := 0; i < int(head); i++ {
+	for i := range genchain {
-		block := blockchain.GetBlockByNumber(uint64(i))
+		blocknum := genchain[i].NumberU64()
+		block := blockchain.GetBlockByNumber(blocknum)
 		if block == nil {
-			t.Fatalf("block %d not found", i)
+			t.Fatalf("block %d not found", blocknum)
 		}
+
+		// Verify receipts.
+		genBlockReceipts := genreceipts[i]
+		for _, r := range genBlockReceipts {
+			if r.BlockNumber.Cmp(block.Number()) != 0 {
+				t.Errorf("receipt has wrong block number %d, want %d", r.BlockNumber, block.Number())
+			}
+			if r.BlockHash != block.Hash() {
+				t.Errorf("receipt has wrong block hash %v, want %v", r.BlockHash, block.Hash())
+			}
+
+			// patch up empty logs list to make DeepEqual below work
+			if r.Logs == nil {
+				r.Logs = []*types.Log{}
+			}
+		}
+		blockchainReceipts := blockchain.GetReceiptsByHash(block.Hash())
+		if !reflect.DeepEqual(genBlockReceipts, blockchainReceipts) {
+			t.Fatalf("receipts mismatch\ngenerated: %s\nblockchain: %s", spew.Sdump(genBlockReceipts), spew.Sdump(blockchainReceipts))
+		}
+
+		// Verify withdrawals.
 		if len(block.Withdrawals()) == 0 {
 			continue
 		}
@@ -134,6 +172,18 @@ func TestGenerateWithdrawalChain(t *testing.T) {
 			}
 			withdrawalIndex += 1
 		}
+
+		// Verify parent beacon root.
+		want := common.Hash{byte(blocknum)}
+		if got := block.BeaconRoot(); *got != want {
+			t.Fatalf("block %d, wrong parent beacon root: got %s, want %s", i, got, want)
+		}
+		state, _ := blockchain.State()
+		idx := block.Time()%8191 + 8191
+		got := state.GetState(params.BeaconRootsStorageAddress, common.BigToHash(new(big.Int).SetUint64(idx)))
+		if got != want {
+			t.Fatalf("block %d, wrong parent beacon root in state: got %s, want %s", i, got, want)
+		}
 	}
 }
 

core/evm.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/consensus"
+	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
@@ -41,6 +42,7 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common
 	var (
 		beneficiary common.Address
 		baseFee     *big.Int
+		blobBaseFee *big.Int
 		random      *common.Hash
 	)
 
@@ -53,21 +55,24 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common
 	if header.BaseFee != nil {
 		baseFee = new(big.Int).Set(header.BaseFee)
 	}
+	if header.ExcessBlobGas != nil {
+		blobBaseFee = eip4844.CalcBlobFee(*header.ExcessBlobGas)
+	}
 	if header.Difficulty.Cmp(common.Big0) == 0 {
 		random = &header.MixDigest
 	}
 	return vm.BlockContext{
-		CanTransfer:   CanTransfer,
+		CanTransfer: CanTransfer,
-		Transfer:      Transfer,
+		Transfer:    Transfer,
-		GetHash:       GetHashFn(header, chain),
+		GetHash:     GetHashFn(header, chain),
-		Coinbase:      beneficiary,
+		Coinbase:    beneficiary,
-		BlockNumber:   new(big.Int).Set(header.Number),
+		BlockNumber: new(big.Int).Set(header.Number),
-		Time:          header.Time,
+		Time:        header.Time,
-		Difficulty:    new(big.Int).Set(header.Difficulty),
+		Difficulty:  new(big.Int).Set(header.Difficulty),
-		BaseFee:       baseFee,
+		BaseFee:     baseFee,
-		GasLimit:      header.GasLimit,
+		BlobBaseFee: blobBaseFee,
-		Random:        random,
+		GasLimit:    header.GasLimit,
-		ExcessBlobGas: header.ExcessBlobGas,
+		Random:      random,
 	}
 }
 

core/forkid/forkid_test.go

@@ -107,6 +107,16 @@ func TestCreation(t *testing.T) {
 				{1735372, 1677557088, ID{Hash: checksumToBytes(0xf7f9bc08), Next: 0}},          // First Shanghai block
 			},
 		},
+		// Holesky test cases
+		{
+			params.HoleskyChainConfig,
+			core.DefaultHoleskyGenesisBlock().ToBlock(),
+			[]testcase{
+				{0, 0, ID{Hash: checksumToBytes(0xc61a6098), Next: 1696000704}},   // Unsynced, last Frontier, Homestead, Tangerine, Spurious, Byzantium, Constantinople, Petersburg, Istanbul, Berlin, London, Paris block
+				{123, 0, ID{Hash: checksumToBytes(0xc61a6098), Next: 1696000704}}, // First MergeNetsplit block
+				{123, 1696000704, ID{Hash: checksumToBytes(0xfd4f016b), Next: 0}}, // Last MergeNetsplit block
+			},
+		},
 	}
 	for i, tt := range tests {
 		for j, ttt := range tt.cases {

core/genesis.go

@@ -120,8 +120,8 @@ func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
-// deriveHash computes the state root according to the genesis specification.
+// hash computes the state root according to the genesis specification.
-func (ga *GenesisAlloc) deriveHash() (common.Hash, error) {
+func (ga *GenesisAlloc) hash() (common.Hash, error) {
 	// Create an ephemeral in-memory database for computing hash,
 	// all the derived states will be discarded to not pollute disk.
 	db := state.NewDatabase(rawdb.NewMemoryDatabase())
@@ -142,9 +142,9 @@ func (ga *GenesisAlloc) deriveHash() (common.Hash, error) {
 	return statedb.Commit(0, false)
 }
 
-// flush is very similar with deriveHash, but the main difference is
+// flush is very similar with hash, but the main difference is all the generated
-// all the generated states will be persisted into the given database.
+// states will be persisted into the given database. Also, the genesis state
-// Also, the genesis state specification will be flushed as well.
+// specification will be flushed as well.
 func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhash common.Hash, bcLogger BlockchainLogger) error {
 	statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil)
 	statedb.SetLogger(bcLogger)
@@ -203,6 +203,8 @@ func getGenesisState(db ethdb.Database, blockhash common.Hash) (alloc GenesisAll
 		genesis = DefaultGoerliGenesisBlock()
 	case params.SepoliaGenesisHash:
 		genesis = DefaultSepoliaGenesisBlock()
+	case params.HoleskyGenesisHash:
+		genesis = DefaultHoleskyGenesisBlock()
 	}
 	if genesis != nil {
 		return genesis.Alloc, nil
@@ -211,21 +213,6 @@ func getGenesisState(db ethdb.Database, blockhash common.Hash) (alloc GenesisAll
 	return nil, nil
 }
 
-// CommitGenesisState loads the stored genesis state with the given block
-// hash and commits it into the provided trie database.
-func CommitGenesisState(db ethdb.Database, triedb *trie.Database, blockhash common.Hash) error {
-	alloc, err := getGenesisState(db, blockhash)
-	if err != nil {
-		return err
-	}
-
-	if alloc == nil {
-		return errors.New("not found")
-	}
-
-	return alloc.flush(db, triedb, blockhash, nil)
-}
-
 // GenesisAccount is an account in the state of the genesis block.
 type GenesisAccount struct {
 	Code       []byte                      `json:"code,omitempty"`
@@ -334,11 +321,12 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen
 		} else {
 			log.Info("Writing custom genesis block")
 		}
+
+		applyOverrides(genesis.Config)
 		block, err := genesis.Commit(db, triedb, bcLogger)
 		if err != nil {
 			return genesis.Config, common.Hash{}, err
 		}
-		applyOverrides(genesis.Config)
 		return genesis.Config, block.Hash(), nil
 	}
 	// The genesis block is present(perhaps in ancient database) while the
@@ -350,6 +338,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen
 		if genesis == nil {
 			genesis = DefaultGenesisBlock()
 		}
+		applyOverrides(genesis.Config)
 		// Ensure the stored genesis matches with the given one.
 		hash := genesis.ToBlock().Hash()
 		if hash != stored {
@@ -359,11 +348,11 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen
 		if err != nil {
 			return genesis.Config, hash, err
 		}
-		applyOverrides(genesis.Config)
 		return genesis.Config, block.Hash(), nil
 	}
 	// Check whether the genesis block is already written.
 	if genesis != nil {
+		applyOverrides(genesis.Config)
 		hash := genesis.ToBlock().Hash()
 		if hash != stored {
 			return genesis.Config, hash, &GenesisMismatchError{stored, hash}
@@ -458,7 +447,7 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig {
 
 // ToBlock returns the genesis block according to genesis specification.
 func (g *Genesis) ToBlock() *types.Block {
-	root, err := g.Alloc.deriveHash()
+	root, err := g.Alloc.hash()
 	if err != nil {
 		panic(err)
 	}
@@ -606,10 +595,9 @@ func DefaultHoleskyGenesisBlock() *Genesis {
 	return &Genesis{
 		Config:     params.HoleskyChainConfig,
 		Nonce:      0x1234,
-		ExtraData:  hexutil.MustDecode("0x686f77206d7563682069732074686520666973683f"),
 		GasLimit:   0x17d7840,
 		Difficulty: big.NewInt(0x01),
-		Timestamp:  1694786100,
+		Timestamp:  1695902100,
 		Alloc:      decodePrealloc(holeskyAllocData),
 	}
 }

core/genesis_test.go

@@ -231,7 +231,7 @@ func TestReadWriteGenesisAlloc(t *testing.T) {
 			{1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}},
 			{2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}},
 		}
-		hash, _ = alloc.deriveHash()
+		hash, _ = alloc.hash()
 	)
 	blob, _ := json.Marshal(alloc)
 	rawdb.WriteGenesisStateSpec(db, hash, blob)

core/rawdb/accessors_chain.go

@@ -334,13 +334,18 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu
 		return rlpHeaders
 	}
 	// read remaining from ancients
-	max := count * 700
+	data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 0)
-	data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, max)
+	if err != nil {
-	if err == nil && uint64(len(data)) == count {
+		log.Error("Failed to read headers from freezer", "err", err)
-		// the data is on the order [h, h+1, .., n] -- reordering needed
+		return rlpHeaders
-		for i := range data {
+	}
-			rlpHeaders = append(rlpHeaders, data[len(data)-1-i])
+	if uint64(len(data)) != count {
-		}
+		log.Warn("Incomplete read of headers from freezer", "wanted", count, "read", len(data))
+		return rlpHeaders
+	}
+	// The data is on the order [h, h+1, .., n] -- reordering needed
+	for i := range data {
+		rlpHeaders = append(rlpHeaders, data[len(data)-1-i])
 	}
 	return rlpHeaders
 }

core/rawdb/accessors_sync.go

@@ -76,3 +76,25 @@ func DeleteSkeletonHeader(db ethdb.KeyValueWriter, number uint64) {
 		log.Crit("Failed to delete skeleton header", "err", err)
 	}
 }
+
+const (
+	StateSyncUnknown  = uint8(0) // flags the state snap sync is unknown
+	StateSyncRunning  = uint8(1) // flags the state snap sync is not completed yet
+	StateSyncFinished = uint8(2) // flags the state snap sync is completed
+)
+
+// ReadSnapSyncStatusFlag retrieves the state snap sync status flag.
+func ReadSnapSyncStatusFlag(db ethdb.KeyValueReader) uint8 {
+	blob, err := db.Get(snapSyncStatusFlagKey)
+	if err != nil || len(blob) != 1 {
+		return StateSyncUnknown
+	}
+	return blob[0]
+}
+
+// WriteSnapSyncStatusFlag stores the state snap sync status flag into database.
+func WriteSnapSyncStatusFlag(db ethdb.KeyValueWriter, flag uint8) {
+	if err := db.Put(snapSyncStatusFlagKey, []byte{flag}); err != nil {
+		log.Crit("Failed to store sync status flag", "err", err)
+	}
+}

core/rawdb/accessors_trie.go

@@ -89,6 +89,16 @@ func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte, hash common.Hash)
 	return h.hash(data) == hash
 }
 
+// ExistsAccountTrieNode checks the presence of the account trie node with the
+// specified node path, regardless of the node hash.
+func ExistsAccountTrieNode(db ethdb.KeyValueReader, path []byte) bool {
+	has, err := db.Has(accountTrieNodeKey(path))
+	if err != nil {
+		return false
+	}
+	return has
+}
+
 // WriteAccountTrieNode writes the provided account trie node into database.
 func WriteAccountTrieNode(db ethdb.KeyValueWriter, path []byte, node []byte) {
 	if err := db.Put(accountTrieNodeKey(path), node); err != nil {
@@ -127,6 +137,16 @@ func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path [
 	return h.hash(data) == hash
 }
 
+// ExistsStorageTrieNode checks the presence of the storage trie node with the
+// specified account hash and node path, regardless of the node hash.
+func ExistsStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) bool {
+	has, err := db.Has(storageTrieNodeKey(accountHash, path))
+	if err != nil {
+		return false
+	}
+	return has
+}
+
 // WriteStorageTrieNode writes the provided storage trie node into database.
 func WriteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte, node []byte) {
 	if err := db.Put(storageTrieNodeKey(accountHash, path), node); err != nil {
@@ -285,3 +305,38 @@ func ReadStateScheme(db ethdb.Reader) string {
 	}
 	return HashScheme
 }
+
+// ParseStateScheme checks if the specified state scheme is compatible with
+// the stored state.
+//
+//   - If the provided scheme is none, use the scheme consistent with persistent
+//     state, or fallback to hash-based scheme if state is empty.
+//
+//   - If the provided scheme is hash, use hash-based scheme or error out if not
+//     compatible with persistent state scheme.
+//
+//   - If the provided scheme is path: use path-based scheme or error out if not
+//     compatible with persistent state scheme.
+func ParseStateScheme(provided string, disk ethdb.Database) (string, error) {
+	// If state scheme is not specified, use the scheme consistent
+	// with persistent state, or fallback to hash mode if database
+	// is empty.
+	stored := ReadStateScheme(disk)
+	if provided == "" {
+		if stored == "" {
+			// use default scheme for empty database, flip it when
+			// path mode is chosen as default
+			log.Info("State schema set to default", "scheme", "hash")
+			return HashScheme, nil
+		}
+		log.Info("State scheme set to already existing", "scheme", stored)
+		return stored, nil // reuse scheme of persistent scheme
+	}
+	// If state scheme is specified, ensure it's compatible with
+	// persistent state.
+	if stored == "" || provided == stored {
+		log.Info("State scheme set by user", "scheme", provided)
+		return provided, nil
+	}
+	return "", fmt.Errorf("incompatible state scheme, stored: %s, provided: %s", stored, provided)
+}

core/rawdb/ancient_utils.go

@@ -88,6 +88,9 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
 			infos = append(infos, info)
 
 		case stateFreezerName:
+			if ReadStateScheme(db) != PathScheme {
+				continue
+			}
 			datadir, err := db.AncientDatadir()
 			if err != nil {
 				return nil, err

core/rawdb/chain_freezer.go

@@ -200,7 +200,7 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 		}
 		batch.Reset()
 
-		// Step into the future and delete and dangling side chains
+		// Step into the future and delete any dangling side chains
 		if frozen > 0 {
 			tip := frozen
 			for len(dangling) > 0 {

core/rawdb/database.go

@@ -30,11 +30,12 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/ethdb/leveldb"
 	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/ethdb/pebble"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/olekukonko/tablewriter"
 )
 
-// freezerdb is a database wrapper that enabled freezer data retrievals.
+// freezerdb is a database wrapper that enables freezer data retrievals.
 type freezerdb struct {
 	ancientRoot string
 	ethdb.KeyValueStore
@@ -141,7 +142,7 @@ func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReaderOp) error)
 	// Unlike other ancient-related methods, this method does not return
 	// errNotSupported when invoked.
 	// The reason for this is that the caller might want to do several things:
-	// 1. Check if something is in freezer,
+	// 1. Check if something is in the freezer,
 	// 2. If not, check leveldb.
 	//
 	// This will work, since the ancient-checks inside 'fn' will return errors,
@@ -209,7 +210,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 	// of the freezer and database. Ensure that we don't shoot ourselves in the foot
 	// by serving up conflicting data, leading to both datastores getting corrupted.
 	//
-	//   - If both the freezer and key-value store is empty (no genesis), we just
+	//   - If both the freezer and key-value store are empty (no genesis), we just
 	//     initialized a new empty freezer, so everything's fine.
 	//   - If the key-value store is empty, but the freezer is not, we need to make
 	//     sure the user's genesis matches the freezer. That will be checked in the
@@ -218,7 +219,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 	//   - If neither the key-value store nor the freezer is empty, cross validate
 	//     the genesis hashes to make sure they are compatible. If they are, also
 	//     ensure that there's no gap between the freezer and subsequently leveldb.
-	//   - If the key-value store is not empty, but the freezer is we might just be
+	//   - If the key-value store is not empty, but the freezer is, we might just be
 	//     upgrading to the freezer release, or we might have had a small chain and
 	//     not frozen anything yet. Ensure that no blocks are missing yet from the
 	//     key-value store, since that would mean we already had an old freezer.
@@ -253,7 +254,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 							break
 						}
 					}
-					// We are about to exit on error. Print database metdata beore exiting
+					// We are about to exit on error. Print database metadata before exiting
 					printChainMetadata(db)
 					return nil, fmt.Errorf("gap in the chain between ancients [0 - #%d] and leveldb [#%d - #%d] ",
 						frozen-1, number, head)
@@ -321,6 +322,16 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r
 	return NewDatabase(db), nil
 }
 
+// NewPebbleDBDatabase creates a persistent key-value database without a freezer
+// moving immutable chain segments into cold storage.
+func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) {
+	db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral)
+	if err != nil {
+		return nil, err
+	}
+	return NewDatabase(db), nil
+}
+
 const (
 	dbPebble  = "pebble"
 	dbLeveldb = "leveldb"
@@ -375,26 +386,16 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) {
 		return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb)
 	}
 	if o.Type == dbPebble || existingDb == dbPebble {
-		if PebbleEnabled {
+		log.Info("Using pebble as the backing database")
-			log.Info("Using pebble as the backing database")
+		return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
-			return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
-		} else {
-			return nil, errors.New("db.engine 'pebble' not supported on this platform")
-		}
 	}
 	if o.Type == dbLeveldb || existingDb == dbLeveldb {
 		log.Info("Using leveldb as the backing database")
 		return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
 	}
-	// No pre-existing database, no user-requested one either. Default to Pebble
+	// No pre-existing database, no user-requested one either. Default to Pebble.
-	// on supported platforms and LevelDB on anything else.
+	log.Info("Defaulting to pebble as the backing database")
-	if PebbleEnabled {
+	return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
-		log.Info("Defaulting to pebble as the backing database")
-		return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral)
-	} else {
-		log.Info("Defaulting to leveldb as the backing database")
-		return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
-	}
 }
 
 // Open opens both a disk-based key-value database such as leveldb or pebble, but also
@@ -555,7 +556,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 				lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey,
 				snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey,
 				uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey,
-				persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey,
+				persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey,
 			} {
 				if bytes.Equal(key, meta) {
 					metadata.Add(size)
@@ -634,7 +635,7 @@ func printChainMetadata(db ethdb.KeyValueStore) {
 	fmt.Fprintf(os.Stderr, "\n\n")
 }
 
-// ReadChainMetadata returns a set of key/value pairs that contains informatin
+// ReadChainMetadata returns a set of key/value pairs that contains information
 // about the database chain status. This can be used for diagnostic purposes
 // when investigating the state of the node.
 func ReadChainMetadata(db ethdb.KeyValueStore) [][]string {

core/rawdb/databases_64bit.go

@@ -1,37 +0,0 @@
-// Copyright 2023 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
-
-//go:build (arm64 || amd64) && !openbsd
-
-package rawdb
-
-import (
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/ethdb/pebble"
-)
-
-// Pebble is unsuported on 32bit architecture
-const PebbleEnabled = true
-
-// NewPebbleDBDatabase creates a persistent key-value database without a freezer
-// moving immutable chain segments into cold storage.
-func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly, ephemeral bool) (ethdb.Database, error) {
-	db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral)
-	if err != nil {
-		return nil, err
-	}
-	return NewDatabase(db), nil
-}

core/rawdb/freezer.go

@@ -108,7 +108,11 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
 	// Leveldb uses LOCK as the filelock filename. To prevent the
 	// name collision, we use FLOCK as the lock name.
 	lock := flock.New(flockFile)
-	if locked, err := lock.TryLock(); err != nil {
+	tryLock := lock.TryLock
+	if readonly {
+		tryLock = lock.TryRLock
+	}
+	if locked, err := tryLock(); err != nil {
 		return nil, err
 	} else if !locked {
 		return nil, errors.New("locking failed")

core/rawdb/freezer_resettable.go

@@ -22,6 +22,7 @@ import (
 	"sync"
 
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
 )
 
 const tmpSuffix = ".tmp"
@@ -118,9 +119,10 @@ func (f *ResettableFreezer) Ancient(kind string, number uint64) ([]byte, error)
 
 // AncientRange retrieves multiple items in sequence, starting from the index 'start'.
 // It will return
-//   - at most 'max' items,
+//   - at most 'count' items,
-//   - at least 1 item (even if exceeding the maxByteSize), but will otherwise
+//   - if maxBytes is specified: at least 1 item (even if exceeding the maxByteSize),
-//     return as many items as fit into maxByteSize
+//     but will otherwise return as many items as fit into maxByteSize.
+//   - if maxBytes is not specified, 'count' items will be returned if they are present.
 func (f *ResettableFreezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
 	f.lock.RLock()
 	defer f.lock.RUnlock()
@@ -224,6 +226,7 @@ func cleanup(path string) error {
 	}
 	for _, name := range names {
 		if name == filepath.Base(path)+tmpSuffix {
+			log.Info("Removed leftover freezer directory", "name", name)
 			return os.RemoveAll(filepath.Join(parent, name))
 		}
 	}

core/rawdb/freezer_table.go

@@ -212,6 +212,9 @@ func (t *freezerTable) repair() error {
 	}
 	// Ensure the index is a multiple of indexEntrySize bytes
 	if overflow := stat.Size() % indexEntrySize; overflow != 0 {
+		if t.readonly {
+			return fmt.Errorf("index file(path: %s, name: %s) size is not a multiple of %d", t.path, t.name, indexEntrySize)
+		}
 		truncateFreezerFile(t.index, stat.Size()-overflow) // New file can't trigger this path
 	}
 	// Retrieve the file sizes and prepare for truncation
@@ -254,6 +257,12 @@ func (t *freezerTable) repair() error {
 		t.index.ReadAt(buffer, offsetsSize-indexEntrySize)
 		lastIndex.unmarshalBinary(buffer)
 	}
+	// Print an error log if the index is corrupted due to an incorrect
+	// last index item. While it is theoretically possible to have a zero offset
+	// by storing all zero-size items, it is highly unlikely to occur in practice.
+	if lastIndex.offset == 0 && offsetsSize%indexEntrySize > 1 {
+		log.Error("Corrupted index file detected", "lastOffset", lastIndex.offset, "items", offsetsSize%indexEntrySize-1)
+	}
 	if t.readonly {
 		t.head, err = t.openFile(lastIndex.filenum, openFreezerFileForReadOnly)
 	} else {
@@ -270,6 +279,9 @@ func (t *freezerTable) repair() error {
 	// Keep truncating both files until they come in sync
 	contentExp = int64(lastIndex.offset)
 	for contentExp != contentSize {
+		if t.readonly {
+			return fmt.Errorf("freezer table(path: %s, name: %s, num: %d) is corrupted", t.path, t.name, lastIndex.filenum)
+		}
 		verbose = true
 		// Truncate the head file to the last offset pointer
 		if contentExp < contentSize {
@@ -343,7 +355,7 @@ func (t *freezerTable) repair() error {
 		return err
 	}
 	if verbose {
-		t.logger.Info("Chain freezer table opened", "items", t.items.Load(), "size", t.headBytes)
+		t.logger.Info("Chain freezer table opened", "items", t.items.Load(), "deleted", t.itemOffset.Load(), "hidden", t.itemHidden.Load(), "tailId", t.tailId, "headId", t.headId, "size", t.headBytes)
 	} else {
 		t.logger.Debug("Chain freezer table opened", "items", t.items.Load(), "size", common.StorageSize(t.headBytes))
 	}
@@ -516,6 +528,10 @@ func (t *freezerTable) truncateTail(items uint64) error {
 	if err := t.meta.Sync(); err != nil {
 		return err
 	}
+	// Close the index file before shorten it.
+	if err := t.index.Close(); err != nil {
+		return err
+	}
 	// Truncate the deleted index entries from the index file.
 	err = copyFrom(t.index.Name(), t.index.Name(), indexEntrySize*(newDeleted-deleted+1), func(f *os.File) error {
 		tailIndex := indexEntry{
@@ -529,13 +545,14 @@ func (t *freezerTable) truncateTail(items uint64) error {
 		return err
 	}
 	// Reopen the modified index file to load the changes
-	if err := t.index.Close(); err != nil {
-		return err
-	}
 	t.index, err = openFreezerFileForAppend(t.index.Name())
 	if err != nil {
 		return err
 	}
+	// Sync the file to ensure changes are flushed to disk
+	if err := t.index.Sync(); err != nil {
+		return err
+	}
 	// Release any files before the current tail
 	t.tailId = newTailId
 	t.itemOffset.Store(newDeleted)
@@ -768,7 +785,7 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i
 			return fmt.Errorf("missing data file %d", fileId)
 		}
 		if _, err := dataFile.ReadAt(output[len(output)-length:], int64(start)); err != nil {
-			return err
+			return fmt.Errorf("%w, fileid: %d, start: %d, length: %d", err, fileId, start, length)
 		}
 		return nil
 	}

core/rawdb/freezer_test.go

@@ -283,6 +283,57 @@ func TestFreezerReadonlyValidate(t *testing.T) {
 	}
 }
 
+func TestFreezerConcurrentReadonly(t *testing.T) {
+	t.Parallel()
+
+	tables := map[string]bool{"a": true}
+	dir := t.TempDir()
+
+	f, err := NewFreezer(dir, "", false, 2049, tables)
+	if err != nil {
+		t.Fatal("can't open freezer", err)
+	}
+	var item = make([]byte, 1024)
+	batch := f.tables["a"].newBatch()
+	items := uint64(10)
+	for i := uint64(0); i < items; i++ {
+		require.NoError(t, batch.AppendRaw(i, item))
+	}
+	require.NoError(t, batch.commit())
+	if loaded := f.tables["a"].items.Load(); loaded != items {
+		t.Fatalf("unexpected number of items in table, want: %d, have: %d", items, loaded)
+	}
+	require.NoError(t, f.Close())
+
+	var (
+		wg   sync.WaitGroup
+		fs   = make([]*Freezer, 5)
+		errs = make([]error, 5)
+	)
+	for i := 0; i < 5; i++ {
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+
+			f, err := NewFreezer(dir, "", true, 2049, tables)
+			if err == nil {
+				fs[i] = f
+			} else {
+				errs[i] = err
+			}
+		}(i)
+	}
+
+	wg.Wait()
+
+	for i := range fs {
+		if err := errs[i]; err != nil {
+			t.Fatal("failed to open freezer", err)
+		}
+		require.NoError(t, fs[i].Close())
+	}
+}
+
 func newFreezerForTesting(t *testing.T, tables map[string]bool) (*Freezer, string) {
 	t.Helper()
 

core/rawdb/schema.go

@@ -91,6 +91,9 @@ var (
 	// transitionStatusKey tracks the eth2 transition status.
 	transitionStatusKey = []byte("eth2-transition")
 
+	// snapSyncStatusFlagKey flags that status of snap sync.
+	snapSyncStatusFlagKey = []byte("SnapSyncStatus")
+
 	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
 	headerPrefix       = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
 	headerTDSuffix     = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td

core/rawdb/table.go

@@ -219,7 +219,7 @@ func (b *tableBatch) Put(key, value []byte) error {
 	return b.batch.Put(append([]byte(b.prefix), key...), value)
 }
 
-// Delete inserts the a key removal into the batch for later committing.
+// Delete inserts a key removal into the batch for later committing.
 func (b *tableBatch) Delete(key []byte) error {
 	return b.batch.Delete(append([]byte(b.prefix), key...))
 }

core/state/pruner/pruner.go

@@ -125,12 +125,12 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
 	// dangling node is the state root is super low. So the dangling nodes in
 	// theory will never ever be visited again.
 	var (
-		count  int
+		skipped, count int
-		size   common.StorageSize
+		size           common.StorageSize
-		pstart = time.Now()
+		pstart         = time.Now()
-		logged = time.Now()
+		logged         = time.Now()
-		batch  = maindb.NewBatch()
+		batch          = maindb.NewBatch()
-		iter   = maindb.NewIterator(nil, nil)
+		iter           = maindb.NewIterator(nil, nil)
 	)
 	for iter.Next() {
 		key := iter.Key()
@@ -149,6 +149,7 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
 				log.Debug("Forcibly delete the middle state roots", "hash", common.BytesToHash(checkKey))
 			} else {
 				if stateBloom.Contain(checkKey) {
+					skipped += 1
 					continue
 				}
 			}
@@ -165,7 +166,7 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
 				eta = time.Duration(left/speed) * time.Millisecond
 			}
 			if time.Since(logged) > 8*time.Second {
-				log.Info("Pruning state data", "nodes", count, "size", size,
+				log.Info("Pruning state data", "nodes", count, "skipped", skipped, "size", size,
 					"elapsed", common.PrettyDuration(time.Since(pstart)), "eta", common.PrettyDuration(eta))
 				logged = time.Now()
 			}

core/state/snapshot/conversion.go

@@ -362,21 +362,15 @@ func generateTrieRoot(db ethdb.KeyValueWriter, scheme string, it Iterator, accou
 }
 
 func stackTrieGenerate(db ethdb.KeyValueWriter, scheme string, owner common.Hash, in chan trieKV, out chan common.Hash) {
-	var nodeWriter trie.NodeWriteFunc
+	options := trie.NewStackTrieOptions()
 	if db != nil {
-		nodeWriter = func(owner common.Hash, path []byte, hash common.Hash, blob []byte) {
+		options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) {
 			rawdb.WriteTrieNode(db, owner, path, hash, blob, scheme)
-		}
+		})
 	}
-	t := trie.NewStackTrieWithOwner(nodeWriter, owner)
+	t := trie.NewStackTrie(options)
 	for leaf := range in {
 		t.Update(leaf.key[:], leaf.value)
 	}
-	var root common.Hash
+	out <- t.Commit()
-	if db == nil {
-		root = t.Hash()
-	} else {
-		root, _ = t.Commit()
-	}
-	out <- root
 }

core/state/snapshot/disklayer.go

@@ -45,6 +45,16 @@ type diskLayer struct {
 	lock sync.RWMutex
 }
 
+// Release releases underlying resources; specifically the fastcache requires
+// Reset() in order to not leak memory.
+// OBS: It does not invoke Close on the diskdb
+func (dl *diskLayer) Release() error {
+	if dl.cache != nil {
+		dl.cache.Reset()
+	}
+	return nil
+}
+
 // Root returns  root hash for which this snapshot was made.
 func (dl *diskLayer) Root() common.Hash {
 	return dl.root

core/state/snapshot/generate.go

@@ -230,7 +230,9 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [
 	if origin == nil && !diskMore {
 		stackTr := trie.NewStackTrie(nil)
 		for i, key := range keys {
-			stackTr.Update(key, vals[i])
+			if err := stackTr.Update(key, vals[i]); err != nil {
+				return nil, err
+			}
 		}
 		if gotRoot := stackTr.Hash(); gotRoot != root {
 			return &proofResult{
@@ -247,11 +249,6 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [
 		ctx.stats.Log("Trie missing, state snapshotting paused", dl.root, dl.genMarker)
 		return nil, errMissingTrie
 	}
-	// Firstly find out the key of last iterated element.
-	var last []byte
-	if len(keys) > 0 {
-		last = keys[len(keys)-1]
-	}
 	// Generate the Merkle proofs for the first and last element
 	if origin == nil {
 		origin = common.Hash{}.Bytes()
@@ -266,9 +263,9 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [
 			tr:       tr,
 		}, nil
 	}
-	if last != nil {
+	if len(keys) > 0 {
-		if err := tr.Prove(last, proof); err != nil {
+		if err := tr.Prove(keys[len(keys)-1], proof); err != nil {
-			log.Debug("Failed to prove range", "kind", kind, "last", last, "err", err)
+			log.Debug("Failed to prove range", "kind", kind, "last", keys[len(keys)-1], "err", err)
 			return &proofResult{
 				keys:     keys,
 				vals:     vals,
@@ -280,7 +277,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [
 	}
 	// Verify the snapshot segment with range prover, ensure that all flat states
 	// in this range correspond to merkle trie.
-	cont, err := trie.VerifyRangeProof(root, origin, last, keys, vals, proof)
+	cont, err := trie.VerifyRangeProof(root, origin, keys, vals, proof)
 	return &proofResult{
 			keys:     keys,
 			vals:     vals,
@@ -446,6 +443,10 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi
 		internal += time.Since(istart)
 	}
 	if iter.Err != nil {
+		// Trie errors should never happen. Still, in case of a bug, expose the
+		// error here, as the outer code will presume errors are interrupts, not
+		// some deeper issues.
+		log.Error("State snapshotter failed to iterate trie", "err", err)
 		return false, nil, iter.Err
 	}
 	// Delete all stale snapshot states remaining

core/state/snapshot/snapshot.go

@@ -656,6 +656,13 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
 	return res
 }
 
+// Release releases resources
+func (t *Tree) Release() {
+	if dl := t.disklayer(); dl != nil {
+		dl.Release()
+	}
+}
+
 // Journal commits an entire diff hierarchy to disk into a single journal entry.
 // This is meant to be used during shutdown to persist the snapshot without
 // flattening everything down (bad for reorgs).

core/state/statedb.go

@@ -992,16 +992,18 @@ func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (boo
 		nodes = trienode.NewNodeSet(addrHash)
 		slots = make(map[common.Hash][]byte)
 	)
-	stack := trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) {
+	options := trie.NewStackTrieOptions()
+	options = options.WithWriter(func(path []byte, hash common.Hash, blob []byte) {
 		nodes.AddNode(path, trienode.NewDeleted())
 		size += common.StorageSize(len(path))
 	})
+	stack := trie.NewStackTrie(options)
 	for iter.Next() {
 		if size > storageDeleteLimit {
 			return true, size, nil, nil, nil
 		}
 		slot := common.CopyBytes(iter.Slot())
-		if iter.Error() != nil { // error might occur after Slot function
+		if err := iter.Error(); err != nil { // error might occur after Slot function
 			return false, 0, nil, nil, err
 		}
 		size += common.StorageSize(common.HashLength + len(slot))
@@ -1011,7 +1013,7 @@ func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (boo
 			return false, 0, nil, nil, err
 		}
 	}
-	if iter.Error() != nil { // error might occur during iteration
+	if err := iter.Error(); err != nil { // error might occur during iteration
 		return false, 0, nil, nil, err
 	}
 	if stack.Hash() != root {
@@ -1089,12 +1091,10 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root
 			slotDeletionSkip.Inc(1)
 		}
 		n := int64(len(slots))
-		if n > slotDeletionMaxCount.Value() {
+
-			slotDeletionMaxCount.Update(n)
+		slotDeletionMaxCount.UpdateIfGt(int64(len(slots)))
-		}
+		slotDeletionMaxSize.UpdateIfGt(int64(size))
-		if int64(size) > slotDeletionMaxSize.Value() {
+
-			slotDeletionMaxSize.Update(int64(size))
-		}
 		slotDeletionTimer.UpdateSince(start)
 		slotDeletionCount.Mark(n)
 		slotDeletionSize.Mark(int64(size))