merge: recent changes

.github/workflows/go.yml

@@ -7,6 +7,11 @@ on:
       - master
   workflow_dispatch:
 
+# Free runner capacity by cancelling superseded PR runs.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   lint:
     name: Lint

README.md

@@ -38,7 +38,6 @@ directory.
 |  Command   | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | :--------: | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI page](https://geth.ethereum.org/docs/fundamentals/command-line-options) for command line options. |
-|   `clef`   | Stand-alone signing tool, which can be used as a backend signer for `geth`.                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 |  `devp2p`  | Utilities to interact with nodes on the networking layer, without running a full blockchain.                                                                                                                                                                                                                                                                                                                                                                                                                                       |
 |  `abigen`  | Source code generator to convert Ethereum contract definitions into easy-to-use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://docs.soliditylang.org/en/develop/abi-spec.html) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://geth.ethereum.org/docs/developers/dapp-developer/native-bindings) page for details.                                  |
 |   `evm`    | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`).                                                                                                                                                                                                                                               |

SECURITY.md

@@ -11,7 +11,6 @@ Audit reports are published in the `docs` folder: https://github.com/ethereum/go
 | Scope | Date | Report Link |
 | ------- | ------- | ----------- |
 | `geth` | 20170425 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2017-04-25_Geth-audit_Truesec.pdf) |
-| `clef` | 20180914 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2018-09-14_Clef-audit_NCC.pdf) |
 | `Discv5` | 20191015 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2019-10-15_Discv5_audit_LeastAuthority.pdf) |
 | `Discv5` | 20200124 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2020-01-24_DiscV5_audit_Cure53.pdf) |
 

accounts/abi/argument.go

@@ -165,7 +165,7 @@ func (arguments Arguments) copyTuple(v any, marshalledValues []any) error {
 		}
 	case reflect.Slice, reflect.Array:
 		if value.Len() < len(marshalledValues) {
-			return fmt.Errorf("abi: insufficient number of arguments for unpack, want %d, got %d", len(arguments), value.Len())
+			return fmt.Errorf("abi: insufficient number of arguments for unpack, want %d, got %d", len(marshalledValues), value.Len())
 		}
 		for i := range nonIndexedArgs {
 			if err := set(value.Index(i), reflect.ValueOf(marshalledValues[i])); err != nil {

accounts/abi/unpack.go

@@ -154,7 +154,7 @@ func forEachUnpack(t Type, output []byte, start, size int) (interface{}, error)
 		return nil, fmt.Errorf("cannot marshal input to array, size is negative (%d)", size)
 	}
 	if start+32*size > len(output) {
-		return nil, fmt.Errorf("abi: cannot marshal into go array: offset %d would go over slice boundary (len=%d)", len(output), start+32*size)
+		return nil, fmt.Errorf("abi: cannot marshal into go array: offset %d would go over slice boundary (len=%d)", start+32*size, len(output))
 	}
 
 	// this value will become our slice or our array, depending on the type

accounts/usbwallet/ledger.go

@@ -110,6 +110,16 @@ func (w *ledgerDriver) offline() bool {
 	return w.version == [3]byte{0, 0, 0}
 }
 
+func ledgerVersionLessThan(version [3]byte, major, minor, patch byte) bool {
+	if version[0] != major {
+		return version[0] < major
+	}
+	if version[1] != minor {
+		return version[1] < minor
+	}
+	return version[2] < patch
+}
+
 // Open implements usbwallet.driver, attempting to initialize the connection to the
 // Ledger hardware wallet. The Ledger does not require a user passphrase, so that
 // parameter is silently discarded.
@@ -166,7 +176,19 @@ func (w *ledgerDriver) SignTx(path accounts.DerivationPath, tx *types.Transactio
 		return common.Address{}, nil, accounts.ErrWalletClosed
 	}
 	// Ensure the wallet is capable of signing the given transaction
-	if chainID != nil && (w.version[0] < 1 || (w.version[0] == 1 && w.version[1] == 0 && w.version[2] < 3)) {
+	switch tx.Type() {
+	case types.AccessListTxType, types.DynamicFeeTxType:
+		if ledgerVersionLessThan(w.version, 1, 9, 0) {
+			//lint:ignore ST1005 brand name displayed on the console
+			return common.Address{}, nil, fmt.Errorf("Ledger version >= 1.9.0 required for EIP-2930/EIP-1559 signing (found version v%d.%d.%d)", w.version[0], w.version[1], w.version[2])
+		}
+	case types.SetCodeTxType:
+		if ledgerVersionLessThan(w.version, 1, 17, 0) {
+			//lint:ignore ST1005 brand name displayed on the console
+			return common.Address{}, nil, fmt.Errorf("Ledger version >= 1.17.0 required for EIP-7702 signing (found version v%d.%d.%d)", w.version[0], w.version[1], w.version[2])
+		}
+	}
+	if chainID != nil && ledgerVersionLessThan(w.version, 1, 0, 3) {
 		//lint:ignore ST1005 brand name displayed on the console
 		return common.Address{}, nil, fmt.Errorf("Ledger v%d.%d.%d doesn't support signing this transaction, please update to v1.0.3 at least", w.version[0], w.version[1], w.version[2])
 	}
@@ -184,7 +206,7 @@ func (w *ledgerDriver) SignTypedMessage(path accounts.DerivationPath, domainHash
 		return nil, accounts.ErrWalletClosed
 	}
 	// Ensure the wallet is capable of signing the given transaction
-	if w.version[0] < 1 || (w.version[0] == 1 && w.version[1] < 5) {
+	if ledgerVersionLessThan(w.version, 1, 5, 0) {
 		//lint:ignore ST1005 brand name displayed on the console
 		return nil, fmt.Errorf("Ledger version >= 1.5.0 required for EIP-712 signing (found version v%d.%d.%d)", w.version[0], w.version[1], w.version[2])
 	}
@@ -334,26 +356,41 @@ func (w *ledgerDriver) ledgerSign(derivationPath []uint32, tx *types.Transaction
 		err   error
 	)
 	if chainID == nil {
-		if txrlp, err = rlp.EncodeToBytes([]interface{}{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data()}); err != nil {
+		if txrlp, err = rlp.EncodeToBytes([]any{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data()}); err != nil {
 			return common.Address{}, nil, err
 		}
 	} else {
-		if tx.Type() == types.DynamicFeeTxType {
-			if txrlp, err = rlp.EncodeToBytes([]interface{}{chainID, tx.Nonce(), tx.GasTipCap(), tx.GasFeeCap(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), tx.AccessList()}); err != nil {
+		switch tx.Type() {
+		case types.SetCodeTxType:
+			if txrlp, err = rlp.EncodeToBytes([]any{chainID, tx.Nonce(), tx.GasTipCap(), tx.GasFeeCap(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), tx.AccessList(), tx.SetCodeAuthorizations()}); err != nil {
 				return common.Address{}, nil, err
 			}
 			// append type to transaction
 			txrlp = append([]byte{tx.Type()}, txrlp...)
-		} else if tx.Type() == types.AccessListTxType {
-			if txrlp, err = rlp.EncodeToBytes([]interface{}{chainID, tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), tx.AccessList()}); err != nil {
+		case types.BlobTxType:
+			if txrlp, err = rlp.EncodeToBytes([]any{chainID, tx.Nonce(), tx.GasTipCap(), tx.GasFeeCap(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), tx.AccessList(), tx.BlobGasFeeCap(), tx.BlobHashes()}); err != nil {
 				return common.Address{}, nil, err
 			}
 			// append type to transaction
 			txrlp = append([]byte{tx.Type()}, txrlp...)
-		} else if tx.Type() == types.LegacyTxType {
-			if txrlp, err = rlp.EncodeToBytes([]interface{}{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), chainID, big.NewInt(0), big.NewInt(0)}); err != nil {
+		case types.DynamicFeeTxType:
+			if txrlp, err = rlp.EncodeToBytes([]any{chainID, tx.Nonce(), tx.GasTipCap(), tx.GasFeeCap(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), tx.AccessList()}); err != nil {
 				return common.Address{}, nil, err
 			}
+			// append type to transaction
+			txrlp = append([]byte{tx.Type()}, txrlp...)
+		case types.AccessListTxType:
+			if txrlp, err = rlp.EncodeToBytes([]any{chainID, tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), tx.AccessList()}); err != nil {
+				return common.Address{}, nil, err
+			}
+			// append type to transaction
+			txrlp = append([]byte{tx.Type()}, txrlp...)
+		case types.LegacyTxType:
+			if txrlp, err = rlp.EncodeToBytes([]any{tx.Nonce(), tx.GasPrice(), tx.Gas(), tx.To(), tx.Value(), tx.Data(), chainID, big.NewInt(0), big.NewInt(0)}); err != nil {
+				return common.Address{}, nil, err
+			}
+		default:
+			return common.Address{}, nil, fmt.Errorf("unsupported transaction type: %d", tx.Type())
 		}
 	}
 	payload := append(path, txrlp...)

beacon/engine/bapl_encode.go

@@ -0,0 +1,75 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package engine
+
+import (
+	"github.com/fjl/jsonw"
+)
+
+// MarshalJSON implements json.Marshaler.
+func (list BlobAndProofListV1) MarshalJSON() ([]byte, error) {
+	if list == nil {
+		return []byte("null"), nil
+	}
+	var b jsonw.Buffer
+	b.Array(func() {
+		for _, item := range list {
+			marshalBlobAndProofV1(&b, item)
+		}
+	})
+	return b.Output(), nil
+}
+
+func marshalBlobAndProofV1(b *jsonw.Buffer, item *BlobAndProofV1) {
+	if item == nil {
+		b.Null()
+	} else {
+		b.Object(func() {
+			b.Key("blob")
+			b.HexBytes(item.Blob)
+			b.Key("proof")
+			b.HexBytes(item.Proof)
+		})
+	}
+}
+
+// MarshalJSON implements json.Marshaler.
+func (list BlobAndProofListV2) MarshalJSON() ([]byte, error) {
+	if list == nil {
+		return []byte("null"), nil
+	}
+	var b jsonw.Buffer
+	b.Array(func() {
+		for _, item := range list {
+			marshalBlobAndProofV2(&b, item)
+		}
+	})
+	return b.Output(), nil
+}
+
+func marshalBlobAndProofV2(b *jsonw.Buffer, item *BlobAndProofV2) {
+	if item == nil {
+		b.Null()
+	} else {
+		b.Object(func() {
+			b.Key("blob")
+			b.HexBytes(item.Blob)
+			b.Key("proofs")
+			appendHexBytesArray(b, item.CellProofs)
+		})
+	}
+}

beacon/engine/ed_codec.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/types/bal"
 )
 
 var _ = (*executableDataMarshaling)(nil)
@@ -17,24 +18,25 @@ var _ = (*executableDataMarshaling)(nil)
 // MarshalJSON marshals as JSON.
 func (e ExecutableData) MarshalJSON() ([]byte, error) {
 	type ExecutableData struct {
-		ParentHash    common.Hash         `json:"parentHash"    gencodec:"required"`
-		FeeRecipient  common.Address      `json:"feeRecipient"  gencodec:"required"`
-		StateRoot     common.Hash         `json:"stateRoot"     gencodec:"required"`
-		ReceiptsRoot  common.Hash         `json:"receiptsRoot"  gencodec:"required"`
-		LogsBloom     hexutil.Bytes       `json:"logsBloom"     gencodec:"required"`
-		Random        common.Hash         `json:"prevRandao"    gencodec:"required"`
-		Number        hexutil.Uint64      `json:"blockNumber"   gencodec:"required"`
-		GasLimit      hexutil.Uint64      `json:"gasLimit"      gencodec:"required"`
-		GasUsed       hexutil.Uint64      `json:"gasUsed"       gencodec:"required"`
-		Timestamp     hexutil.Uint64      `json:"timestamp"     gencodec:"required"`
-		ExtraData     hexutil.Bytes       `json:"extraData"     gencodec:"required"`
-		BaseFeePerGas *hexutil.Big        `json:"baseFeePerGas" gencodec:"required"`
-		BlockHash     common.Hash         `json:"blockHash"     gencodec:"required"`
-		Transactions  []hexutil.Bytes     `json:"transactions"  gencodec:"required"`
-		Withdrawals   []*types.Withdrawal `json:"withdrawals"`
-		BlobGasUsed   *hexutil.Uint64     `json:"blobGasUsed"`
-		ExcessBlobGas *hexutil.Uint64     `json:"excessBlobGas"`
-		SlotNumber    *hexutil.Uint64     `json:"slotNumber,omitempty"`
+		ParentHash      common.Hash          `json:"parentHash"    gencodec:"required"`
+		FeeRecipient    common.Address       `json:"feeRecipient"  gencodec:"required"`
+		StateRoot       common.Hash          `json:"stateRoot"     gencodec:"required"`
+		ReceiptsRoot    common.Hash          `json:"receiptsRoot"  gencodec:"required"`
+		LogsBloom       hexutil.Bytes        `json:"logsBloom"     gencodec:"required"`
+		Random          common.Hash          `json:"prevRandao"    gencodec:"required"`
+		Number          hexutil.Uint64       `json:"blockNumber"   gencodec:"required"`
+		GasLimit        hexutil.Uint64       `json:"gasLimit"      gencodec:"required"`
+		GasUsed         hexutil.Uint64       `json:"gasUsed"       gencodec:"required"`
+		Timestamp       hexutil.Uint64       `json:"timestamp"     gencodec:"required"`
+		ExtraData       hexutil.Bytes        `json:"extraData"     gencodec:"required"`
+		BaseFeePerGas   *hexutil.Big         `json:"baseFeePerGas" gencodec:"required"`
+		BlockHash       common.Hash          `json:"blockHash"     gencodec:"required"`
+		Transactions    []hexutil.Bytes      `json:"transactions"  gencodec:"required"`
+		Withdrawals     []*types.Withdrawal  `json:"withdrawals"`
+		BlobGasUsed     *hexutil.Uint64      `json:"blobGasUsed"`
+		ExcessBlobGas   *hexutil.Uint64      `json:"excessBlobGas"`
+		SlotNumber      *hexutil.Uint64      `json:"slotNumber,omitempty"`
+		BlockAccessList *bal.BlockAccessList `json:"blockAccessList,omitempty"`
 	}
 	var enc ExecutableData
 	enc.ParentHash = e.ParentHash
@@ -60,30 +62,32 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) {
 	enc.BlobGasUsed = (*hexutil.Uint64)(e.BlobGasUsed)
 	enc.ExcessBlobGas = (*hexutil.Uint64)(e.ExcessBlobGas)
 	enc.SlotNumber = (*hexutil.Uint64)(e.SlotNumber)
+	enc.BlockAccessList = e.BlockAccessList
 	return json.Marshal(&enc)
 }
 
 // UnmarshalJSON unmarshals from JSON.
 func (e *ExecutableData) UnmarshalJSON(input []byte) error {
 	type ExecutableData struct {
-		ParentHash    *common.Hash        `json:"parentHash"    gencodec:"required"`
-		FeeRecipient  *common.Address     `json:"feeRecipient"  gencodec:"required"`
-		StateRoot     *common.Hash        `json:"stateRoot"     gencodec:"required"`
-		ReceiptsRoot  *common.Hash        `json:"receiptsRoot"  gencodec:"required"`
-		LogsBloom     *hexutil.Bytes      `json:"logsBloom"     gencodec:"required"`
-		Random        *common.Hash        `json:"prevRandao"    gencodec:"required"`
-		Number        *hexutil.Uint64     `json:"blockNumber"   gencodec:"required"`
-		GasLimit      *hexutil.Uint64     `json:"gasLimit"      gencodec:"required"`
-		GasUsed       *hexutil.Uint64     `json:"gasUsed"       gencodec:"required"`
-		Timestamp     *hexutil.Uint64     `json:"timestamp"     gencodec:"required"`
-		ExtraData     *hexutil.Bytes      `json:"extraData"     gencodec:"required"`
-		BaseFeePerGas *hexutil.Big        `json:"baseFeePerGas" gencodec:"required"`
-		BlockHash     *common.Hash        `json:"blockHash"     gencodec:"required"`
-		Transactions  []hexutil.Bytes     `json:"transactions"  gencodec:"required"`
-		Withdrawals   []*types.Withdrawal `json:"withdrawals"`
-		BlobGasUsed   *hexutil.Uint64     `json:"blobGasUsed"`
-		ExcessBlobGas *hexutil.Uint64     `json:"excessBlobGas"`
-		SlotNumber    *hexutil.Uint64     `json:"slotNumber,omitempty"`
+		ParentHash      *common.Hash         `json:"parentHash"    gencodec:"required"`
+		FeeRecipient    *common.Address      `json:"feeRecipient"  gencodec:"required"`
+		StateRoot       *common.Hash         `json:"stateRoot"     gencodec:"required"`
+		ReceiptsRoot    *common.Hash         `json:"receiptsRoot"  gencodec:"required"`
+		LogsBloom       *hexutil.Bytes       `json:"logsBloom"     gencodec:"required"`
+		Random          *common.Hash         `json:"prevRandao"    gencodec:"required"`
+		Number          *hexutil.Uint64      `json:"blockNumber"   gencodec:"required"`
+		GasLimit        *hexutil.Uint64      `json:"gasLimit"      gencodec:"required"`
+		GasUsed         *hexutil.Uint64      `json:"gasUsed"       gencodec:"required"`
+		Timestamp       *hexutil.Uint64      `json:"timestamp"     gencodec:"required"`
+		ExtraData       *hexutil.Bytes       `json:"extraData"     gencodec:"required"`
+		BaseFeePerGas   *hexutil.Big         `json:"baseFeePerGas" gencodec:"required"`
+		BlockHash       *common.Hash         `json:"blockHash"     gencodec:"required"`
+		Transactions    []hexutil.Bytes      `json:"transactions"  gencodec:"required"`
+		Withdrawals     []*types.Withdrawal  `json:"withdrawals"`
+		BlobGasUsed     *hexutil.Uint64      `json:"blobGasUsed"`
+		ExcessBlobGas   *hexutil.Uint64      `json:"excessBlobGas"`
+		SlotNumber      *hexutil.Uint64      `json:"slotNumber,omitempty"`
+		BlockAccessList *bal.BlockAccessList `json:"blockAccessList,omitempty"`
 	}
 	var dec ExecutableData
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -160,5 +164,8 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error {
 	if dec.SlotNumber != nil {
 		e.SlotNumber = (*uint64)(dec.SlotNumber)
 	}
+	if dec.BlockAccessList != nil {
+		e.BlockAccessList = dec.BlockAccessList
+	}
 	return nil
 }

beacon/engine/epe_decode.go

@@ -12,31 +12,6 @@ import (
 
 var _ = (*executionPayloadEnvelopeMarshaling)(nil)
 
-// MarshalJSON marshals as JSON.
-func (e ExecutionPayloadEnvelope) MarshalJSON() ([]byte, error) {
-	type ExecutionPayloadEnvelope struct {
-		ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
-		BlockValue       *hexutil.Big    `json:"blockValue"  gencodec:"required"`
-		BlobsBundle      *BlobsBundle    `json:"blobsBundle"`
-		Requests         []hexutil.Bytes `json:"executionRequests"`
-		Override         bool            `json:"shouldOverrideBuilder"`
-		Witness          *hexutil.Bytes  `json:"witness,omitempty"`
-	}
-	var enc ExecutionPayloadEnvelope
-	enc.ExecutionPayload = e.ExecutionPayload
-	enc.BlockValue = (*hexutil.Big)(e.BlockValue)
-	enc.BlobsBundle = e.BlobsBundle
-	if e.Requests != nil {
-		enc.Requests = make([]hexutil.Bytes, len(e.Requests))
-		for k, v := range e.Requests {
-			enc.Requests[k] = v
-		}
-	}
-	enc.Override = e.Override
-	enc.Witness = e.Witness
-	return json.Marshal(&enc)
-}
-
 // UnmarshalJSON unmarshals from JSON.
 func (e *ExecutionPayloadEnvelope) UnmarshalJSON(input []byte) error {
 	type ExecutionPayloadEnvelope struct {

beacon/engine/epe_encode.go

@@ -0,0 +1,98 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package engine
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/fjl/jsonw"
+)
+
+// marshalBlobsBundle writes BlobsBundle as JSON and appends it to buf.
+func marshalBlobsBundle(b *jsonw.Buffer, bundle *BlobsBundle) {
+	if bundle == nil {
+		b.Null()
+		return
+	}
+	b.Object(func() {
+		b.Key("commitments")
+		appendHexBytesArray(b, bundle.Commitments)
+		b.Key("proofs")
+		appendHexBytesArray(b, bundle.Proofs)
+		b.Key("blobs")
+		appendHexBytesArray(b, bundle.Blobs)
+	})
+}
+
+// MarshalJSON implements json.Marshaler.
+func (e ExecutionPayloadEnvelope) MarshalJSON() ([]byte, error) {
+	if e.ExecutionPayload == nil {
+		return nil, errors.New("missing required field 'executionPayload' for ExecutionPayloadEnvelope")
+	}
+
+	// Pre-marshal the execution payload using its gencodec MarshalJSON.
+	payload, err := e.ExecutionPayload.MarshalJSON()
+	if err != nil {
+		return nil, err
+	}
+	// Pre-marshal the witness.
+	var witness []byte
+	if e.Witness != nil {
+		witness, err = json.Marshal(e.Witness)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Write the execution payload to the buffer
+	var b jsonw.Buffer
+	b.Object(func() {
+		b.Key("executionPayload")
+		b.RawValue(payload)
+
+		b.Key("blockValue")
+		b.HexBigInt(e.BlockValue)
+
+		b.Key("blobsBundle")
+		marshalBlobsBundle(&b, e.BlobsBundle)
+
+		b.Key("executionRequests")
+		if e.Requests == nil {
+			b.Null()
+		} else {
+			appendHexBytesArray(&b, e.Requests)
+		}
+
+		b.Key("shouldOverrideBuilder")
+		b.Bool(e.Override)
+
+		if e.Witness != nil {
+			b.Key("witness")
+			b.RawValue(witness)
+		}
+	})
+	return b.Output(), nil
+}
+
+func appendHexBytesArray[T ~[]byte](b *jsonw.Buffer, slice []T) {
+	b.Array(func() {
+		for _, elem := range slice {
+			b.HexBytes(elem)
+		}
+	})
+}

beacon/engine/epe_test.go

@@ -0,0 +1,128 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package engine
+
+import (
+	"bytes"
+	"encoding/json"
+	"math/big"
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+)
+
+func makeTestPayload() *ExecutableData {
+	return &ExecutableData{
+		ParentHash:    common.HexToHash("0x01"),
+		FeeRecipient:  common.HexToAddress("0x02"),
+		StateRoot:     common.HexToHash("0x03"),
+		ReceiptsRoot:  common.HexToHash("0x04"),
+		LogsBloom:     make([]byte, 256),
+		Random:        common.HexToHash("0x05"),
+		Number:        100,
+		GasLimit:      1000000,
+		GasUsed:       500000,
+		Timestamp:     1234567890,
+		ExtraData:     []byte("extra"),
+		BaseFeePerGas: big.NewInt(7),
+		BlockHash:     common.HexToHash("0x08"),
+		Transactions:  [][]byte{{0xaa, 0xbb}},
+	}
+}
+
+func TestMarshalJSONRoundtrip(t *testing.T) {
+	witness := hexutil.Bytes{0xde, 0xad}
+	original := ExecutionPayloadEnvelope{
+		ExecutionPayload: makeTestPayload(),
+		BlockValue:       big.NewInt(12345),
+		BlobsBundle: &BlobsBundle{
+			Commitments: []hexutil.Bytes{{0x01, 0x02}},
+			Proofs:      []hexutil.Bytes{{0x03, 0x04}},
+			Blobs:       []hexutil.Bytes{{0x05, 0x06}},
+		},
+		Requests: [][]byte{{0xaa}, {0xbb, 0xcc}},
+		Override: true,
+		Witness:  &witness,
+	}
+
+	data, err := original.MarshalJSON()
+	if err != nil {
+		t.Fatalf("MarshalJSON error: %v", err)
+	}
+
+	var decoded ExecutionPayloadEnvelope
+	if err := json.Unmarshal(data, &decoded); err != nil {
+		t.Fatalf("UnmarshalJSON error: %v", err)
+	}
+
+	if decoded.ExecutionPayload.Number != original.ExecutionPayload.Number {
+		t.Error("ExecutionPayload.Number mismatch")
+	}
+	if decoded.BlockValue.Cmp(original.BlockValue) != 0 {
+		t.Errorf("BlockValue mismatch: got %v, want %v", decoded.BlockValue, original.BlockValue)
+	}
+	if len(decoded.BlobsBundle.Blobs) != len(original.BlobsBundle.Blobs) {
+		t.Error("BlobsBundle.Blobs length mismatch")
+	}
+	if len(decoded.Requests) != len(original.Requests) {
+		t.Error("Requests length mismatch")
+	}
+	if decoded.Override != original.Override {
+		t.Error("Override mismatch")
+	}
+	if !bytes.Equal(*decoded.Witness, *original.Witness) {
+		t.Error("Witness mismatch")
+	}
+}
+
+func TestMarshalJSONNilPayload(t *testing.T) {
+	env := ExecutionPayloadEnvelope{
+		ExecutionPayload: nil,
+		BlockValue:       big.NewInt(1),
+	}
+	_, err := env.MarshalJSON()
+	if err == nil {
+		t.Fatal("expected error for nil ExecutionPayload")
+	}
+}
+
+// TestExecutionPayloadEnvelopeFieldCoverage guards against structural drift.
+// If a field is added to or removed from ExecutionPayloadEnvelope, this test
+// fails, reminding the developer to update MarshalJSON in marshal_epe.go.
+func TestExecutionPayloadEnvelopeFieldCoverage(t *testing.T) {
+	expected := []string{
+		"ExecutionPayload",
+		"BlockValue",
+		"BlobsBundle",
+		"Requests",
+		"Override",
+		"Witness",
+	}
+	typ := reflect.TypeOf(ExecutionPayloadEnvelope{})
+	if typ.NumField() != len(expected) {
+		t.Fatalf("ExecutionPayloadEnvelope has %d fields, expected %d — update MarshalJSON in marshal_epe.go",
+			typ.NumField(), len(expected))
+	}
+	for i, name := range expected {
+		if typ.Field(i).Name != name {
+			t.Errorf("field %d: got %q, want %q — update MarshalJSON in marshal_epe.go",
+				i, typ.Field(i).Name, name)
+		}
+	}
+}

beacon/engine/types.go

@@ -24,6 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/types/bal"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
 )
@@ -59,7 +60,7 @@ var (
 	PayloadV4 PayloadVersion = 0x4
 )
 
-//go:generate go run github.com/fjl/gencodec -type PayloadAttributes -field-override payloadAttributesMarshaling -out gen_blockparams.go
+//go:generate go run github.com/fjl/gencodec -type PayloadAttributes -field-override payloadAttributesMarshaling -out pa_codec.go
 
 // PayloadAttributes describes the environment context in which a block should
 // be built.
@@ -78,28 +79,29 @@ type payloadAttributesMarshaling struct {
 	SlotNumber *hexutil.Uint64
 }
 
-//go:generate go run github.com/fjl/gencodec -type ExecutableData -field-override executableDataMarshaling -out gen_ed.go
+//go:generate go run github.com/fjl/gencodec -type ExecutableData -field-override executableDataMarshaling -out ed_codec.go
 
 // ExecutableData is the data necessary to execute an EL payload.
 type ExecutableData struct {
-	ParentHash    common.Hash         `json:"parentHash"    gencodec:"required"`
-	FeeRecipient  common.Address      `json:"feeRecipient"  gencodec:"required"`
-	StateRoot     common.Hash         `json:"stateRoot"     gencodec:"required"`
-	ReceiptsRoot  common.Hash         `json:"receiptsRoot"  gencodec:"required"`
-	LogsBloom     []byte              `json:"logsBloom"     gencodec:"required"`
-	Random        common.Hash         `json:"prevRandao"    gencodec:"required"`
-	Number        uint64              `json:"blockNumber"   gencodec:"required"`
-	GasLimit      uint64              `json:"gasLimit"      gencodec:"required"`
-	GasUsed       uint64              `json:"gasUsed"       gencodec:"required"`
-	Timestamp     uint64              `json:"timestamp"     gencodec:"required"`
-	ExtraData     []byte              `json:"extraData"     gencodec:"required"`
-	BaseFeePerGas *big.Int            `json:"baseFeePerGas" gencodec:"required"`
-	BlockHash     common.Hash         `json:"blockHash"     gencodec:"required"`
-	Transactions  [][]byte            `json:"transactions"  gencodec:"required"`
-	Withdrawals   []*types.Withdrawal `json:"withdrawals"`
-	BlobGasUsed   *uint64             `json:"blobGasUsed"`
-	ExcessBlobGas *uint64             `json:"excessBlobGas"`
-	SlotNumber    *uint64             `json:"slotNumber,omitempty"`
+	ParentHash      common.Hash          `json:"parentHash"    gencodec:"required"`
+	FeeRecipient    common.Address       `json:"feeRecipient"  gencodec:"required"`
+	StateRoot       common.Hash          `json:"stateRoot"     gencodec:"required"`
+	ReceiptsRoot    common.Hash          `json:"receiptsRoot"  gencodec:"required"`
+	LogsBloom       []byte               `json:"logsBloom"     gencodec:"required"`
+	Random          common.Hash          `json:"prevRandao"    gencodec:"required"`
+	Number          uint64               `json:"blockNumber"   gencodec:"required"`
+	GasLimit        uint64               `json:"gasLimit"      gencodec:"required"`
+	GasUsed         uint64               `json:"gasUsed"       gencodec:"required"`
+	Timestamp       uint64               `json:"timestamp"     gencodec:"required"`
+	ExtraData       []byte               `json:"extraData"     gencodec:"required"`
+	BaseFeePerGas   *big.Int             `json:"baseFeePerGas" gencodec:"required"`
+	BlockHash       common.Hash          `json:"blockHash"     gencodec:"required"`
+	Transactions    [][]byte             `json:"transactions"  gencodec:"required"`
+	Withdrawals     []*types.Withdrawal  `json:"withdrawals"`
+	BlobGasUsed     *uint64              `json:"blobGasUsed"`
+	ExcessBlobGas   *uint64              `json:"excessBlobGas"`
+	SlotNumber      *uint64              `json:"slotNumber,omitempty"`
+	BlockAccessList *bal.BlockAccessList `json:"blockAccessList,omitempty"`
 }
 
 // JSON type overrides for executableData.
@@ -125,7 +127,7 @@ type StatelessPayloadStatusV1 struct {
 	ValidationError *string     `json:"validationError"`
 }
 
-//go:generate go run github.com/fjl/gencodec -type ExecutionPayloadEnvelope -field-override executionPayloadEnvelopeMarshaling -out gen_epe.go
+//go:generate go run github.com/fjl/gencodec -enc=false -type ExecutionPayloadEnvelope -field-override executionPayloadEnvelopeMarshaling -out epe_decode.go
 
 type ExecutionPayloadEnvelope struct {
 	ExecutionPayload *ExecutableData `json:"executionPayload"  gencodec:"required"`
@@ -136,6 +138,12 @@ type ExecutionPayloadEnvelope struct {
 	Witness          *hexutil.Bytes  `json:"witness,omitempty"`
 }
 
+// JSON type overrides for ExecutionPayloadEnvelope.
+type executionPayloadEnvelopeMarshaling struct {
+	BlockValue *hexutil.Big
+	Requests   []hexutil.Bytes
+}
+
 // BlobsBundle includes the marshalled sidecar data. Note this structure is
 // shared by BlobsBundleV1 and BlobsBundleV2 for the sake of simplicity.
 //
@@ -152,16 +160,18 @@ type BlobAndProofV1 struct {
 	Proof hexutil.Bytes `json:"proof"`
 }
 
+// BlobAndProofListV1 is a list of BlobAndProofV1 with a hand-rolled JSON marshaler
+// that avoids the overhead of encoding/json for large blob payloads.
+type BlobAndProofListV1 []*BlobAndProofV1
+
 type BlobAndProofV2 struct {
 	Blob       hexutil.Bytes   `json:"blob"`
 	CellProofs []hexutil.Bytes `json:"proofs"` // proofs MUST contain exactly CELLS_PER_EXT_BLOB cell proofs.
 }
 
-// JSON type overrides for ExecutionPayloadEnvelope.
-type executionPayloadEnvelopeMarshaling struct {
-	BlockValue *hexutil.Big
-	Requests   []hexutil.Bytes
-}
+// BlobAndProofListV2 is a list of BlobAndProofV2 with a hand-rolled JSON marshaler
+// that avoids the overhead of encoding/json for large blob payloads.
+type BlobAndProofListV2 []*BlobAndProofV2
 
 type PayloadStatusV1 struct {
 	Status          string         `json:"status"`
@@ -303,56 +313,66 @@ func ExecutableDataToBlockNoHash(data ExecutableData, versionedHashes []common.H
 		requestsHash = &h
 	}
 
-	header := &types.Header{
-		ParentHash:       data.ParentHash,
-		UncleHash:        types.EmptyUncleHash,
-		Coinbase:         data.FeeRecipient,
-		Root:             data.StateRoot,
-		TxHash:           types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)),
-		ReceiptHash:      data.ReceiptsRoot,
-		Bloom:            types.BytesToBloom(data.LogsBloom),
-		Difficulty:       common.Big0,
-		Number:           new(big.Int).SetUint64(data.Number),
-		GasLimit:         data.GasLimit,
-		GasUsed:          data.GasUsed,
-		Time:             data.Timestamp,
-		BaseFee:          data.BaseFeePerGas,
-		Extra:            data.ExtraData,
-		MixDigest:        data.Random,
-		WithdrawalsHash:  withdrawalsRoot,
-		ExcessBlobGas:    data.ExcessBlobGas,
-		BlobGasUsed:      data.BlobGasUsed,
-		ParentBeaconRoot: beaconRoot,
-		RequestsHash:     requestsHash,
-		SlotNumber:       data.SlotNumber,
+	// If Amsterdam is enabled, data.BlockAccessList is always non-nil,
+	// even for empty blocks with no state transitions.
+	//
+	// If Amsterdam is not enabled yet, blockAccessListHash is expected
+	// to be nil.
+	var blockAccessListHash *common.Hash
+	if data.BlockAccessList != nil {
+		hash := data.BlockAccessList.Hash()
+		blockAccessListHash = &hash
 	}
-	return types.NewBlockWithHeader(header).
-			WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: data.Withdrawals}),
-		nil
+	header := &types.Header{
+		ParentHash:          data.ParentHash,
+		UncleHash:           types.EmptyUncleHash,
+		Coinbase:            data.FeeRecipient,
+		Root:                data.StateRoot,
+		TxHash:              types.DeriveSha(types.Transactions(txs), trie.NewStackTrie(nil)),
+		ReceiptHash:         data.ReceiptsRoot,
+		Bloom:               types.BytesToBloom(data.LogsBloom),
+		Difficulty:          common.Big0,
+		Number:              new(big.Int).SetUint64(data.Number),
+		GasLimit:            data.GasLimit,
+		GasUsed:             data.GasUsed,
+		Time:                data.Timestamp,
+		BaseFee:             data.BaseFeePerGas,
+		Extra:               data.ExtraData,
+		MixDigest:           data.Random,
+		WithdrawalsHash:     withdrawalsRoot,
+		ExcessBlobGas:       data.ExcessBlobGas,
+		BlobGasUsed:         data.BlobGasUsed,
+		ParentBeaconRoot:    beaconRoot,
+		RequestsHash:        requestsHash,
+		SlotNumber:          data.SlotNumber,
+		BlockAccessListHash: blockAccessListHash,
+	}
+	return types.NewBlockWithHeader(header).WithBody(types.Body{Transactions: txs, Uncles: nil, Withdrawals: data.Withdrawals}), nil
 }
 
 // BlockToExecutableData constructs the ExecutableData structure by filling the
 // fields from the given block. It assumes the given block is post-merge block.
 func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types.BlobTxSidecar, requests [][]byte) *ExecutionPayloadEnvelope {
 	data := &ExecutableData{
-		BlockHash:     block.Hash(),
-		ParentHash:    block.ParentHash(),
-		FeeRecipient:  block.Coinbase(),
-		StateRoot:     block.Root(),
-		Number:        block.NumberU64(),
-		GasLimit:      block.GasLimit(),
-		GasUsed:       block.GasUsed(),
-		BaseFeePerGas: block.BaseFee(),
-		Timestamp:     block.Time(),
-		ReceiptsRoot:  block.ReceiptHash(),
-		LogsBloom:     block.Bloom().Bytes(),
-		Transactions:  encodeTransactions(block.Transactions()),
-		Random:        block.MixDigest(),
-		ExtraData:     block.Extra(),
-		Withdrawals:   block.Withdrawals(),
-		BlobGasUsed:   block.BlobGasUsed(),
-		ExcessBlobGas: block.ExcessBlobGas(),
-		SlotNumber:    block.SlotNumber(),
+		BlockHash:       block.Hash(),
+		ParentHash:      block.ParentHash(),
+		FeeRecipient:    block.Coinbase(),
+		StateRoot:       block.Root(),
+		Number:          block.NumberU64(),
+		GasLimit:        block.GasLimit(),
+		GasUsed:         block.GasUsed(),
+		BaseFeePerGas:   block.BaseFee(),
+		Timestamp:       block.Time(),
+		ReceiptsRoot:    block.ReceiptHash(),
+		LogsBloom:       block.Bloom().Bytes(),
+		Transactions:    encodeTransactions(block.Transactions()),
+		Random:          block.MixDigest(),
+		ExtraData:       block.Extra(),
+		Withdrawals:     block.Withdrawals(),
+		BlobGasUsed:     block.BlobGasUsed(),
+		ExcessBlobGas:   block.ExcessBlobGas(),
+		SlotNumber:      block.SlotNumber(),
+		BlockAccessList: block.AccessList(),
 	}
 
 	// Add blobs.

beacon/light/committee_chain.go

@@ -182,6 +182,12 @@ func (s *CommitteeChain) Reset() {
 	s.chainmu.Lock()
 	defer s.chainmu.Unlock()
 
+	s.resetLocked()
+}
+
+// ResetLocked resets the committee chain without locking. The caller should hold
+// the chainmu lock.
+func (s *CommitteeChain) resetLocked() {
 	if err := s.rollback(0); err != nil {
 		log.Error("Error writing batch into chain database", "error", err)
 	}
@@ -201,22 +207,22 @@ func (s *CommitteeChain) CheckpointInit(bootstrap types.BootstrapData) error {
 	}
 	period := bootstrap.Header.SyncPeriod()
 	if err := s.deleteFixedCommitteeRootsFrom(period + 2); err != nil {
-		s.Reset()
+		s.resetLocked()
 		return err
 	}
 	if s.addFixedCommitteeRoot(period, bootstrap.CommitteeRoot) != nil {
-		s.Reset()
+		s.resetLocked()
 		if err := s.addFixedCommitteeRoot(period, bootstrap.CommitteeRoot); err != nil {
-			s.Reset()
+			s.resetLocked()
 			return err
 		}
 	}
 	if err := s.addFixedCommitteeRoot(period+1, common.Hash(bootstrap.CommitteeBranch[0])); err != nil {
-		s.Reset()
+		s.resetLocked()
 		return err
 	}
 	if err := s.addCommittee(period, bootstrap.Committee); err != nil {
-		s.Reset()
+		s.resetLocked()
 		return err
 	}
 	s.changeCounter++

beacon/light/sync/update_sync.go

@@ -98,7 +98,10 @@ func (s *CheckpointInit) Process(requester request.Requester, events []request.E
 			case ssDefault:
 				if resp != nil {
 					if checkpoint := resp.(*types.BootstrapData); checkpoint.Header.Hash() == common.Hash(req.(ReqCheckpointData)) {
-						s.chain.CheckpointInit(*checkpoint)
+						err := s.chain.CheckpointInit(*checkpoint)
+						if err != nil {
+							return
+						}
 						s.initialized = true
 						return
 					}

build/ci.go

@@ -75,7 +75,7 @@ var (
 
 	// Files that end up in the geth-alltools*.zip archive (and the NSIS installer
 	// dev-tools section). Order matches the historical layout produced by ci.go.
-	allToolsBinaries = []string{"abigen", "evm", "geth", "rlpdump", "clef"}
+	allToolsBinaries = []string{"abigen", "evm", "geth", "rlpdump"}
 
 	// Keeper build targets with their configurations
 	keeperTargets = []struct {
@@ -135,10 +135,6 @@ var (
 			BinaryName:  "rlpdump",
 			Description: "Developer utility tool that prints RLP structures.",
 		},
-		{
-			BinaryName:  "clef",
-			Description: "Ethereum account management tool.",
-		},
 	}
 
 	// A debian package is created for all executables listed here.

cmd/clef/README.md

@@ -1,922 +0,0 @@
-# Clef
-
-Clef can be used to sign transactions and data and is meant as a(n eventual) replacement for Geth's account management. This allows DApps to not depend on Geth's account management. When a DApp wants to sign data (or a transaction), it can send the content to Clef, which will then provide the user with context and ask for permission to sign the content. If the user grants the signing request, Clef will send the signature back to the DApp.
-
-This setup allows a DApp to connect to a remote Ethereum node and send transactions that are locally signed. This can help in situations when a DApp is connected to an untrusted remote Ethereum node, because a local one is not available, not synchronized with the chain, or is a node that has no built-in (or limited) account management.
-
-Clef can run as a daemon on the same machine, off a usb-stick like [USB armory](https://inversepath.com/usbarmory), or even a separate VM in a [QubesOS](https://www.qubes-os.org/) type setup.
-
-Check out the
-
-* [CLI tutorial](tutorial.md) for some concrete examples on how Clef works.
-* [Setup docs](docs/setup.md) for information on how to configure Clef on QubesOS or USB Armory.
-* [Data types](datatypes.md) for details on the communication messages between Clef and an external UI.
-
-## Command line flags
-
-Clef accepts the following command line options:
-
-```
-COMMANDS:
-   init    Initialize the signer, generate secret storage
-   attest  Attest that a js-file is to be used
-   setpw   Store a credential for a keystore file
-   delpw   Remove a credential for a keystore file
-   gendoc  Generate documentation about json-rpc format
-   help    Shows a list of commands or help for one command
-
-GLOBAL OPTIONS:
-   --loglevel value        log level to emit to the screen (default: 4)
-   --keystore value        Directory for the keystore (default: "$HOME/.ethereum/keystore")
-   --configdir value       Directory for Clef configuration (default: "$HOME/.clef")
-   --chainid value         Chain id to use for signing (1=mainnet, 17000=Holesky) (default: 1)
-   --lightkdf              Reduce key-derivation RAM & CPU usage at some expense of KDF strength
-   --nousb                 Disables monitoring for and managing USB hardware wallets
-   --pcscdpath value       Path to the smartcard daemon (pcscd) socket file (default: "/run/pcscd/pcscd.comm")
-   --http.addr value       HTTP-RPC server listening interface (default: "localhost")
-   --http.vhosts value     Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. (default: "localhost")
-   --ipcdisable            Disable the IPC-RPC server
-   --ipcpath               Filename for IPC socket/pipe within the datadir (explicit paths escape it)
-   --http                  Enable the HTTP-RPC server
-   --http.port value       HTTP-RPC server listening port (default: 8550)
-   --signersecret value    A file containing the (encrypted) master seed to encrypt Clef data, e.g. keystore credentials and ruleset hash
-   --4bytedb-custom value  File used for writing new 4byte-identifiers submitted via API (default: "./4byte-custom.json")
-   --auditlog value        File used to emit audit logs. Set to "" to disable (default: "audit.log")
-   --rules value           Path to the rule file to auto-authorize requests with
-   --stdio-ui              Use STDIN/STDOUT as a channel for an external UI. This means that an STDIN/STDOUT is used for RPC-communication with a e.g. a graphical user interface, and can be used when Clef is started by an external process.
-   --stdio-ui-test         Mechanism to test interface between Clef and UI. Requires 'stdio-ui'.
-   --advanced              If enabled, issues warnings instead of rejections for suspicious requests. Default off
-   --suppress-bootwarn     If set, does not show the warning during boot
-   --help, -h              show help
-   --version, -v           print the version
-```
-
-Example:
-
-```
-$ clef -keystore /my/keystore -chainid 4
-```
-
-## Security model
-
-The security model of Clef is as follows:
-
-* One critical component (the Clef binary / daemon) is responsible for handling cryptographic operations: signing, private keys, encryption/decryption of keystore files.
-* Clef has a well-defined 'external' API.
-* The 'external' API is considered UNTRUSTED.
-* Clef also communicates with whatever process that invoked the binary, via stdin/stdout.
-  * This channel is considered 'trusted'. Over this channel, approvals and passwords are communicated.
-
-The general flow for signing a transaction using e.g. Geth is as follows:
-![image](sign_flow.png)
-
-In this case, `geth` would be started with `--signer http://localhost:8550` and would relay requests to `eth.sendTransaction`.
-
-## TODOs
-
-Some snags and todos
-
-* [ ] Clef should take a startup param "--no-change", for UIs that do not contain the capability to perform changes to things, only approve/deny. Such a UI should be able to start the signer in a more secure mode by telling it that it only wants approve/deny capabilities.
-* [x] It would be nice if Clef could collect new 4byte-id:s/method selectors, and have a secondary database for those (`4byte_custom.json`). Users could then (optionally) submit their collections for inclusion upstream.
-* [ ] It should be possible to configure Clef to check if an account is indeed known to it, before passing on to the UI. The reason it currently does not, is that it would make it possible to enumerate accounts if it immediately returned "unknown account" (side channel attack).
-* [x] It should be possible to configure Clef to auto-allow listing (certain) accounts, instead of asking every time.
-* [x] Done Upon startup, Clef should spit out some info to the caller (particularly important when executed in `stdio-ui`-mode), invoking methods with the following info:
-  * [x] Version info about the signer
-  * [x] Address of API (HTTP/IPC)
-  * [ ] List of known accounts
-* [ ] Have a default timeout on signing operations, so that if the user has not answered within e.g. 60 seconds, the request is rejected.
-* [ ] `account_signRawTransaction`
-* [ ] `account_bulkSignTransactions([] transactions)` should
-   * only exist if enabled via config/flag
-   * only allow non-data-sending transactions
-   * all txs must use the same `from`-account
-   * let the user confirm, showing
-      * the total amount
-      * the number of unique recipients
-
-* Geth todos
-    - The signer should pass the `Origin` header as call-info to the UI. As of right now, the way that info about the request is put together is a bit of a hack into the HTTP server. This could probably be greatly improved.
-    - Relay: Geth should be started in `geth --signer localhost:8550`.
-    - Currently, the Geth APIs use `common.Address` in the arguments to transaction submission (e.g `to` field). This type is 20 `bytes`, and is incapable of carrying checksum information. The signer uses `common.MixedcaseAddress`, which retains the original input.
-    - The Geth API should switch to use the same type, and relay `to`-account verbatim to the external API.
-* [x] Storage
-    * [x] An encrypted key-value storage should be implemented.
-    * See [rules.md](rules.md) for more info about this.
-* Another potential thing to introduce is pairing.
-  * To prevent spurious requests which users just accept, implement a way to "pair" the caller with the signer (external API).
-  * Thus Geth/cpp would cryptographically handshake and afterwards the caller would be allowed to make signing requests.
-  * This feature would make the addition of rules less dangerous.
-
-* Wallets / accounts. Add API methods for wallets.
-
-## Communication
-
-### External API
-
-Clef listens to HTTP requests on `http.addr`:`http.port` (or to IPC on `ipcpath`), with the same JSON-RPC standard as Geth. The messages are expected to be [JSON-RPC 2.0 standard](https://www.jsonrpc.org/specification).
-
-Some of these calls can require user interaction. Clients must be aware that responses may be delayed significantly or may never be received if a user decides to ignore the confirmation request.
-
-The External API is **untrusted**: it does not accept credentials, nor does it expect that requests have any authority.
-
-### Internal UI API
-
-Clef has one native console-based UI, for operation without any standalone tools. However, there is also an API to communicate with an external UI. To enable that UI, the signer needs to be executed with the `--stdio-ui` option, which allocates `stdin` / `stdout` for the UI API.
-
-An example (insecure) proof-of-concept has been implemented in `pythonsigner.py`.
-
-The model is as follows:
-
-* The user starts the UI app (`pythonsigner.py`).
-* The UI app starts `clef` with `--stdio-ui`, and listens to the
-process output for confirmation-requests.
-* `clef` opens the external HTTP API.
-* When the `signer` receives requests, it sends a JSON-RPC request via `stdout`.
-* The UI app prompts the user accordingly, and responds to `clef`.
-* `clef` signs (or not), and responds to the original request.
-
-## External API
-
-See the [external API changelog](extapi_changelog.md) for information about changes to this API.
-
-### Encoding
-- number: positive integers that are hex encoded
-- data: hex encoded data
-- string: ASCII string
-
-All hex encoded values must be prefixed with `0x`.
-
-### account_new
-
-#### Create new password protected account
-
-The signer will generate a new private key, encrypt it according to [web3 keystore spec](https://ethereum.org/en/developers/docs/data-structures-and-encoding/web3-secret-storage/) and store it in the keystore directory.  
-The client is responsible for creating a backup of the keystore. If the keystore is lost there is no method of retrieving lost accounts.
-
-#### Arguments
-
-None
-
-#### Result
-  - address [string]: account address that is derived from the generated key
-
-#### Sample call
-```json
-{
-  "id": 0,
-  "jsonrpc": "2.0",
-  "method": "account_new",
-  "params": []
-}
-```
-Response
-```json
-{
-  "id": 0,
-  "jsonrpc": "2.0",
-  "result": "0xbea9183f8f4f03d427f6bcea17388bdff1cab133"
-}
-```
-
-### account_list
-
-#### List available accounts
-   List all accounts that this signer currently manages
-
-#### Arguments
-
-None
-
-#### Result
-  - array with account records:
-     - account.address [string]: account address that is derived from the generated key
-
-#### Sample call
-```json
-{
-  "id": 1,
-  "jsonrpc": "2.0",
-  "method": "account_list"
-}
-```
-Response
-```json
-{
-  "id": 1,
-  "jsonrpc": "2.0",
-  "result": [
-    "0xafb2f771f58513609765698f65d3f2f0224a956f",
-    "0xbea9183f8f4f03d427f6bcea17388bdff1cab133"
-  ]
-}
-```
-
-### account_signTransaction
-
-#### Sign transactions
-   Signs a transaction and responds with the signed transaction in RLP-encoded and JSON forms.
-
-#### Arguments
-  1. transaction object:
-     - `from` [address]: account to send the transaction from
-     - `to` [address]: receiver account. If omitted or `0x`, will cause contract creation.
-     - `gas` [number]: maximum amount of gas to burn
-     - `gasPrice` [number]: gas price
-     - `value` [number:optional]: amount of Wei to send with the transaction
-     - `data` [data:optional]:  input data
-     - `nonce` [number]: account nonce
-  2. method signature [string:optional]
-       - The method signature, if present, is to aid decoding the calldata. Should consist of `methodname(paramtype,...)`, e.g. `transfer(uint256,address)`. The signer may use this data to parse the supplied calldata, and show the user. The data, however, is considered totally untrusted, and reliability is not expected.
-
-
-#### Result
-  - raw [data]: signed transaction in RLP encoded form
-  - tx [json]: signed transaction in JSON form
-
-#### Sample call
-```json
-{
-  "id": 2,
-  "jsonrpc": "2.0",
-  "method": "account_signTransaction",
-  "params": [
-    {
-      "from": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
-      "gas": "0x55555",
-      "gasPrice": "0x1234",
-      "input": "0xabcd",
-      "nonce": "0x0",
-      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
-      "value": "0x1234"
-    }
-  ]
-}
-```
-Response
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 2,
-  "result": {
-    "raw": "0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
-    "tx": {
-      "nonce": "0x0",
-      "gasPrice": "0x1234",
-      "gas": "0x55555",
-      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
-      "value": "0x1234",
-      "input": "0xabcd",
-      "v": "0x26",
-      "r": "0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e",
-      "s": "0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
-      "hash": "0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"
-    }
-  }
-}
-```
-#### Sample call with ABI-data
-
-
-```json
-{
-  "id": 67,
-  "jsonrpc": "2.0",
-  "method": "account_signTransaction",
-  "params": [
-    {
-      "from": "0x694267f14675d7e1b9494fd8d72fefe1755710fa",
-      "gas": "0x333",
-      "gasPrice": "0x1",
-      "nonce": "0x0",
-      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
-      "value": "0x0",
-      "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
-    },
-    "safeSend(address)"
-  ]
-}
-```
-Response
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 67,
-  "result": {
-    "raw": "0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
-    "tx": {
-      "nonce": "0x0",
-      "gasPrice": "0x1",
-      "gas": "0x333",
-      "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
-      "value": "0x0",
-      "input": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
-      "v": "0x26",
-      "r": "0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e",
-      "s": "0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
-      "hash": "0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"
-    }
-  }
-}
-```
-
-Bash example:
-```bash
-> curl -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
-
-{"jsonrpc":"2.0","id":67,"result":{"raw":"0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","tx":{"nonce":"0x0","gasPrice":"0x1","gas":"0x333","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0","value":"0x0","input":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012","v":"0x26","r":"0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e","s":"0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663","hash":"0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"}}}
-```
-
-### account_signData
-
-#### Sign data
-   Signs a chunk of data and returns the calculated signature.
-
-#### Arguments
-  - content type [string]: type of signed data
-     - `text/validator`: hex data with a custom validator defined in a contract
-     - `application/clique`: [clique](https://github.com/ethereum/EIPs/issues/225) headers
-     - `text/plain`: simple hex data validated by `account_ecRecover`
-  - account [address]: account to sign with
-  - data [object]: data to sign
-
-#### Result
-  - calculated signature [data]
-
-#### Sample call
-```json
-{
-  "id": 3,
-  "jsonrpc": "2.0",
-  "method": "account_signData",
-  "params": [
-    "data/plain",
-    "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db",
-    "0xaabbccdd"
-  ]
-}
-```
-Response
-
-```json
-{
-  "id": 3,
-  "jsonrpc": "2.0",
-  "result": "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
-}
-```
-
-### account_signTypedData
-
-#### Sign data
-   Signs a chunk of structured data conformant to [EIP-712](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-712.md) and returns the calculated signature.
-
-#### Arguments
-  - account [address]: account to sign with
-  - data [object]: data to sign
-
-#### Result
-  - calculated signature [data]
-
-#### Sample call
-```json
-{
-  "id": 68,
-  "jsonrpc": "2.0",
-  "method": "account_signTypedData",
-  "params": [
-    "0xcd2a3d9f938e13cd947ec05abc7fe734df8dd826",
-    {
-      "types": {
-        "EIP712Domain": [
-          {
-            "name": "name",
-            "type": "string"
-          },
-          {
-            "name": "version",
-            "type": "string"
-          },
-          {
-            "name": "chainId",
-            "type": "uint256"
-          },
-          {
-            "name": "verifyingContract",
-            "type": "address"
-          }
-        ],
-        "Person": [
-          {
-            "name": "name",
-            "type": "string"
-          },
-          {
-            "name": "wallet",
-            "type": "address"
-          }
-        ],
-        "Mail": [
-          {
-            "name": "from",
-            "type": "Person"
-          },
-          {
-            "name": "to",
-            "type": "Person"
-          },
-          {
-            "name": "contents",
-            "type": "string"
-          }
-        ]
-      },
-      "primaryType": "Mail",
-      "domain": {
-        "name": "Ether Mail",
-        "version": "1",
-        "chainId": 1,
-        "verifyingContract": "0xCcCCccccCCCCcCCCCCCcCcCccCcCCCcCcccccccC"
-      },
-      "message": {
-        "from": {
-          "name": "Cow",
-          "wallet": "0xCD2a3d9F938E13CD947Ec05AbC7FE734Df8DD826"
-        },
-        "to": {
-          "name": "Bob",
-          "wallet": "0xbBbBBBBbbBBBbbbBbbBbbbbBBbBbbbbBbBbbBBbB"
-        },
-        "contents": "Hello, Bob!"
-      }
-    }
-  ]
-}
-```
-Response
-
-```json
-{
-    "id": 1,
-    "jsonrpc": "2.0",
-    "result": "0x4355c47d63924e8a72e509b65029052eb6c299d53a04e167c5775fd466751c9d07299936d304c153f6443dfa05f40ff007d72911b6f72307f996231605b915621c"
-}
-```
-
-### account_ecRecover
-
-#### Recover the signing address
-
-Derive the address from the account that was used to sign data with content type `text/plain` and the signature.
-
-#### Arguments
-  - data [data]: data that was signed
-  - signature [data]: the signature to verify
-
-#### Result
-  - derived account [address]
-
-#### Sample call
-```json
-{
-  "id": 4,
-  "jsonrpc": "2.0",
-  "method": "account_ecRecover",
-  "params": [
-    "0xaabbccdd",
-    "0x5b6693f153b48ec1c706ba4169960386dbaa6903e249cc79a8e6ddc434451d417e1e57327872c7f538beeb323c300afa9999a3d4a5de6caf3be0d5ef832b67ef1c"
-  ]
-}
-```
-Response
-
-```json
-{
-  "id": 4,
-  "jsonrpc": "2.0",
-  "result": "0x1923f626bb8dc025849e00f99c25fe2b2f7fb0db"
-}
-```
-
-### account_version
-
-#### Get external API version
-
-Get the version of the external API used by Clef.
-
-#### Arguments
-
-None
-
-#### Result
-
-* external API version [string]
-
-#### Sample call
-```json
-{
-  "id": 0,
-  "jsonrpc": "2.0",
-  "method": "account_version",
-  "params": []
-}
-```
-
-Response
-```json
-{
-    "id": 0,
-    "jsonrpc": "2.0",
-    "result": "6.0.0"
-}
-```
-
-## UI API
-
-These methods needs to be implemented by a UI listener.
-
-By starting the signer with the switch `--stdio-ui-test`, the signer will invoke all known methods, and expect the UI to respond with
-denials. This can be used during development to ensure that the API is (at least somewhat) correctly implemented.
-See `pythonsigner`, which can be invoked via `python3 pythonsigner.py test` to perform the 'denial-handshake-test'.
-
-All methods in this API use object-based parameters, so that there can be no mixup of parameters: each piece of data is accessed by key.
-
-See the [ui API changelog](intapi_changelog.md) for information about changes to this API.
-
-OBS! A slight deviation from `json` standard is in place: every request and response should be confined to a single line.
-Whereas the `json` specification allows for linebreaks, linebreaks __should not__ be used in this communication channel, to make
-things simpler for both parties.
-
-### ApproveTx / `ui_approveTx`
-
-Invoked when there's a transaction for approval.
-
-
-#### Sample call
-
-Here's a method invocation:
-```bash
-
-curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
-```
-Results in the following invocation on the UI:
-```json
-
-{
-  "jsonrpc": "2.0",
-  "id": 1,
-  "method": "ui_approveTx",
-  "params": [
-    {
-      "transaction": {
-        "from": "0x0x694267f14675d7e1b9494fd8d72fefe1755710fa",
-        "to": "0x0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
-        "gas": "0x333",
-        "gasPrice": "0x1",
-        "value": "0x0",
-        "nonce": "0x0",
-        "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
-        "input": null
-      },
-      "call_info": [
-          {
-            "type": "WARNING",
-            "message": "Invalid checksum on to-address"
-          },
-          {
-            "type": "Info",
-            "message": "safeSend(address: 0x0000000000000000000000000000000000000012)"
-          }
-        ],
-      "meta": {
-        "remote": "127.0.0.1:48486",
-        "local": "localhost:8550",
-        "scheme": "HTTP/1.1"
-      }
-    }
-  ]
-}
-
-```
-
-The same method invocation, but with invalid data:
-```bash
-
-curl -i -H "Content-Type: application/json" -X POST --data '{"jsonrpc":"2.0","method":"account_signTransaction","params":[{"from":"0x694267f14675d7e1b9494fd8d72fefe1755710fa","gas":"0x333","gasPrice":"0x1","nonce":"0x0","to":"0x07a565b7ed7d7a678680a4c162885bedbb695fe0", "value":"0x0", "data":"0x4401a6e40000000000000002000000000000000000000000000000000000000000000012"},"safeSend(address)"],"id":67}' http://localhost:8550/
-```
-
-```json
-
-{
-  "jsonrpc": "2.0",
-  "id": 1,
-  "method": "ui_approveTx",
-  "params": [
-    {
-      "transaction": {
-        "from": "0x0x694267f14675d7e1b9494fd8d72fefe1755710fa",
-        "to": "0x0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
-        "gas": "0x333",
-        "gasPrice": "0x1",
-        "value": "0x0",
-        "nonce": "0x0",
-        "data": "0x4401a6e40000000000000002000000000000000000000000000000000000000000000012",
-        "input": null
-      },
-      "call_info": [
-          {
-            "type": "WARNING",
-            "message": "Invalid checksum on to-address"
-          },
-          {
-            "type": "WARNING",
-            "message": "Transaction data did not match ABI-interface: WARNING: Supplied data is stuffed with extra data. \nWant 0000000000000002000000000000000000000000000000000000000000000012\nHave 0000000000000000000000000000000000000000000000000000000000000012\nfor method safeSend(address)"
-          }
-        ],
-      "meta": {
-        "remote": "127.0.0.1:48492",
-        "local": "localhost:8550",
-        "scheme": "HTTP/1.1"
-      }
-    }
-  ]
-}
-
-
-```
-
-One which has missing `to`, but with no `data`:
-
-
-```json
-
-{
-  "jsonrpc": "2.0",
-  "id": 3,
-  "method": "ui_approveTx",
-  "params": [
-    {
-      "transaction": {
-        "from": "",
-        "to": null,
-        "gas": "0x0",
-        "gasPrice": "0x0",
-        "value": "0x0",
-        "nonce": "0x0",
-        "data": null,
-        "input": null
-      },
-      "call_info": [
-          {
-            "type": "CRITICAL",
-            "message": "Tx will create contract with empty code!"
-          }
-        ],
-      "meta": {
-        "remote": "signer binary",
-        "local": "main",
-        "scheme": "in-proc"
-      }
-    }
-  ]
-}
-```
-
-### ApproveListing / `ui_approveListing`
-
-Invoked when a request for account listing has been made.
-
-#### Sample call
-
-```json
-
-{
-  "jsonrpc": "2.0",
-  "id": 5,
-  "method": "ui_approveListing",
-  "params": [
-    {
-      "accounts": [
-        {
-          "url": "keystore:///home/bazonk/.ethereum/keystore/UTC--2017-11-20T14-44-54.089682944Z--123409812340981234098123409812deadbeef42",
-          "address": "0x123409812340981234098123409812deadbeef42"
-        },
-        {
-          "url": "keystore:///home/bazonk/.ethereum/keystore/UTC--2017-11-23T21-59-03.199240693Z--cafebabedeadbeef34098123409812deadbeef42",
-          "address": "0xcafebabedeadbeef34098123409812deadbeef42"
-        }
-      ],
-      "meta": {
-        "remote": "signer binary",
-        "local": "main",
-        "scheme": "in-proc"
-      }
-    }
-  ]
-}
-
-```
-
-
-### ApproveSignData / `ui_approveSignData`
-
-#### Sample call
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 4,
-  "method": "ui_approveSignData",
-  "params": [
-    {
-      "address": "0x123409812340981234098123409812deadbeef42",
-      "raw_data": "0x01020304",
-      "messages": [
-        {
-          "name": "message",
-          "value": "\u0019Ethereum Signed Message:\n4\u0001\u0002\u0003\u0004",
-          "type": "text/plain"
-        }
-      ],
-      "hash": "0x7e3a4e7a9d1744bc5c675c25e1234ca8ed9162bd17f78b9085e48047c15ac310",
-      "meta": {
-        "remote": "signer binary",
-        "local": "main",
-        "scheme": "in-proc"
-      }
-    }
-  ]
-}
-```
-
-### ApproveNewAccount / `ui_approveNewAccount`
-
-Invoked when a request for creating a new account has been made.
-
-#### Sample call
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 4,
-  "method": "ui_approveNewAccount",
-  "params": [
-    {
-      "meta": {
-        "remote": "signer binary",
-        "local": "main",
-        "scheme": "in-proc"
-      }
-    }
-  ]
-}
-```
-
-### ShowInfo / `ui_showInfo`
-
-The UI should show the info (a single message) to the user. Does not expect response.
-
-#### Sample call
-
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 9,
-  "method": "ui_showInfo",
-  "params": [
-    "Tests completed"
-  ]
-}
-
-```
-
-### ShowError / `ui_showError`
-
-The UI should show the error (a single message) to the user. Does not expect response.
-
-```json
-
-{
-  "jsonrpc": "2.0",
-  "id": 2,
-  "method": "ui_showError",
-  "params": [
-    "Something bad happened!"
-  ]
-}
-
-```
-
-### OnApprovedTx / `ui_onApprovedTx`
-
-`OnApprovedTx` is called when a transaction has been approved and signed. The call contains the return value that will be sent to the external caller.  The return value from this method is ignored - the reason for having this callback is to allow the ruleset to keep track of approved transactions.
-
-When implementing rate-limited rules, this callback should be used.
-
-TLDR; Use this method to keep track of signed transactions, instead of using the data in `ApproveTx`.
-
-Example call:
-```json
-
-{
-  "jsonrpc": "2.0",
-  "id": 1,
-  "method": "ui_onApprovedTx",
-  "params": [
-    {
-      "raw": "0xf88380018203339407a565b7ed7d7a678680a4c162885bedbb695fe080a44401a6e4000000000000000000000000000000000000000000000000000000000000001226a0223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20ea02aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
-      "tx": {
-        "nonce": "0x0",
-        "gasPrice": "0x1",
-        "gas": "0x333",
-        "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
-        "value": "0x0",
-        "input": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
-        "v": "0x26",
-        "r": "0x223a7c9bcf5531c99be5ea7082183816eb20cfe0bbc322e97cc5c7f71ab8b20e",
-        "s": "0x2aadee6b34b45bb15bc42d9c09de4a6754e7000908da72d48cc7704971491663",
-        "hash": "0xeba2df809e7a612a0a0d444ccfa5c839624bdc00dd29e3340d46df3870f8a30e"
-      }
-    }
-  ]
-}
-```
-
-### OnSignerStartup / `ui_onSignerStartup`
-
-This method provides the UI with information about what API version the signer uses (both internal and external) as well as build-info and external API,
-in k/v-form.
-
-Example call:
-```json
-
-{
-  "jsonrpc": "2.0",
-  "id": 1,
-  "method": "ui_onSignerStartup",
-  "params": [
-    {
-      "info": {
-        "extapi_http": "http://localhost:8550",
-        "extapi_ipc": null,
-        "extapi_version": "2.0.0",
-        "intapi_version": "1.2.0"
-      }
-    }
-  ]
-}
-
-```
-
-### OnInputRequired / `ui_onInputRequired`
-
-Invoked when Clef requires user input (e.g. a password).
-
-Example call:
-```json
-
-{
-  "jsonrpc": "2.0",
-  "id": 1,
-  "method": "ui_onInputRequired",
-  "params": [
-    {
-      "title": "Account password",
-      "prompt": "Please enter the password for account 0x694267f14675d7e1b9494fd8d72fefe1755710fa",
-      "isPassword": true
-    }
-  ]
-}
-```
-
-
-### Rules for UI apis
-
-A UI should conform to the following rules.
-
-* A UI MUST NOT load any external resources that were not embedded/part of the UI package.
-  * For example, not load icons, stylesheets from the internet
-  * Not load files from the filesystem, unless they reside in the same local directory (e.g. config files)
-* A Graphical UI MUST show the blocky-identicon for ethereum addresses.
-* A UI MUST warn display appropriate warning if the destination-account is formatted with invalid checksum.
-* A UI MUST NOT open any ports or services
-  * The signer opens the public port
-* A UI SHOULD verify the permissions on the signer binary, and refuse to execute or warn if permissions allow non-user write.
-* A UI SHOULD inform the user about the `SHA256` or `MD5` hash of the binary being executed
-* A UI SHOULD NOT maintain a secondary storage of data, e.g. list of accounts
-  * The signer provides accounts
-* A UI SHOULD, to the best extent possible, use static linking / bundling, so that required libraries are bundled
-along with the UI.
-
-
-### UI Implementations
-
-There are a couple of implementation for a UI. We'll try to keep this list up to date.
-
-| Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters|
-| ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- |
-| QtSigner| https://github.com/holiman/qtsigner/ | Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: |  :+1: (partially)|
-| GtkSigner| https://github.com/holiman/gtksigner | Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: |  :x: |
-| Frame | https://github.com/floating/frame/commits/go-signer | Electron-based| :x:| :x:| :x:| :x:| ?| :x: |  :x: |
-| Clef UI| https://github.com/ethereum/clef-ui | Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: |  :+1: (approve tx only)|

cmd/clef/consolecmd_test.go

@@ -1,121 +0,0 @@
-// Copyright 2022 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-)
-
-// TestImportRaw tests clef --importraw
-func TestImportRaw(t *testing.T) {
-	t.Parallel()
-	keyPath := filepath.Join(t.TempDir(), fmt.Sprintf("%v-tempkey.test", t.Name()))
-	os.WriteFile(keyPath, []byte("0102030405060708090a0102030405060708090a0102030405060708090a0102"), 0777)
-
-	t.Run("happy-path", func(t *testing.T) {
-		t.Parallel()
-		// Run clef importraw
-		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "importraw", keyPath)
-		clef.input("myverylongpassword").input("myverylongpassword")
-		if out := string(clef.Output()); !strings.Contains(out,
-			"Key imported:\n  Address 0x9160DC9105f7De5dC5E7f3d97ef11DA47269BdA6") {
-			t.Logf("Output\n%v", out)
-			t.Error("Failure")
-		}
-	})
-	// tests clef --importraw with mismatched passwords.
-	t.Run("pw-mismatch", func(t *testing.T) {
-		t.Parallel()
-		// Run clef importraw
-		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "importraw", keyPath)
-		clef.input("myverylongpassword1").input("myverylongpassword2").WaitExit()
-		if have, want := clef.StderrText(), "Passwords do not match\n"; have != want {
-			t.Errorf("have %q, want %q", have, want)
-		}
-	})
-	// tests clef --importraw with a too short password.
-	t.Run("short-pw", func(t *testing.T) {
-		t.Parallel()
-		// Run clef importraw
-		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "importraw", keyPath)
-		clef.input("shorty").input("shorty").WaitExit()
-		if have, want := clef.StderrText(),
-			"password requirements not met: password too short (<10 characters)\n"; have != want {
-			t.Errorf("have %q, want %q", have, want)
-		}
-	})
-}
-
-// TestListAccounts tests clef --list-accounts
-func TestListAccounts(t *testing.T) {
-	t.Parallel()
-	keyPath := filepath.Join(t.TempDir(), fmt.Sprintf("%v-tempkey.test", t.Name()))
-	os.WriteFile(keyPath, []byte("0102030405060708090a0102030405060708090a0102030405060708090a0102"), 0777)
-
-	t.Run("no-accounts", func(t *testing.T) {
-		t.Parallel()
-		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "list-accounts")
-		if out := string(clef.Output()); !strings.Contains(out, "The keystore is empty.") {
-			t.Logf("Output\n%v", out)
-			t.Error("Failure")
-		}
-	})
-	t.Run("one-account", func(t *testing.T) {
-		t.Parallel()
-		// First, we need to import
-		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "importraw", keyPath)
-		clef.input("myverylongpassword").input("myverylongpassword").WaitExit()
-		// Secondly, do a listing, using the same datadir
-		clef = runWithKeystore(t, clef.Datadir, "--suppress-bootwarn", "--lightkdf", "list-accounts")
-		if out := string(clef.Output()); !strings.Contains(out, "0x9160DC9105f7De5dC5E7f3d97ef11DA47269BdA6 (keystore:") {
-			t.Logf("Output\n%v", out)
-			t.Error("Failure")
-		}
-	})
-}
-
-// TestListWallets tests clef --list-wallets
-func TestListWallets(t *testing.T) {
-	t.Parallel()
-	keyPath := filepath.Join(t.TempDir(), fmt.Sprintf("%v-tempkey.test", t.Name()))
-	os.WriteFile(keyPath, []byte("0102030405060708090a0102030405060708090a0102030405060708090a0102"), 0777)
-
-	t.Run("no-accounts", func(t *testing.T) {
-		t.Parallel()
-		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "list-wallets")
-		if out := string(clef.Output()); !strings.Contains(out, "There are no wallets.") {
-			t.Logf("Output\n%v", out)
-			t.Error("Failure")
-		}
-	})
-	t.Run("one-account", func(t *testing.T) {
-		t.Parallel()
-		// First, we need to import
-		clef := runClef(t, "--suppress-bootwarn", "--lightkdf", "importraw", keyPath)
-		clef.input("myverylongpassword").input("myverylongpassword").WaitExit()
-		// Secondly, do a listing, using the same datadir
-		clef = runWithKeystore(t, clef.Datadir, "--suppress-bootwarn", "--lightkdf", "list-wallets")
-		if out := string(clef.Output()); !strings.Contains(out, "Account 0: 0x9160DC9105f7De5dC5E7f3d97ef11DA47269BdA6") {
-			t.Logf("Output\n%v", out)
-			t.Error("Failure")
-		}
-	})
-}

cmd/clef/datatypes.md

@@ -1,224 +0,0 @@
-## UI Client interface
-
-These data types are defined in the channel between clef and the UI
-### SignDataRequest
-
-SignDataRequest contains information about a pending request to sign some data. The data to be signed can be of various types, defined by content-type. Clef has done most of the work in canonicalizing and making sense of the data, and it's up to the UI to present the user with the contents of the `message`
-
-Example:
-```json
-{
-  "content_type": "text/plain",
-  "address": "0xDEADbEeF000000000000000000000000DeaDbeEf",
-  "raw_data": "GUV0aGVyZXVtIFNpZ25lZCBNZXNzYWdlOgoxMWhlbGxvIHdvcmxk",
-  "messages": [
-    {
-      "name": "message",
-      "value": "\u0019Ethereum Signed Message:\n11hello world",
-      "type": "text/plain"
-    }
-  ],
-  "hash": "0xd9eba16ed0ecae432b71fe008c98cc872bb4cc214d3220a36f365326cf807d68",
-  "meta": {
-    "remote": "localhost:9999",
-    "local": "localhost:8545",
-    "scheme": "http",
-    "User-Agent": "Firefox 3.2",
-    "Origin": "www.malicious.ru"
-  }
-}
-```
-### SignDataResponse - approve
-
-Response to SignDataRequest
-
-Example:
-```json
-{
-  "approved": true
-}
-```
-### SignDataResponse - deny
-
-Response to SignDataRequest
-
-Example:
-```json
-{
-  "approved": false
-}
-```
-### SignTxRequest
-
-SignTxRequest contains information about a pending request to sign a transaction. Aside from the transaction itself, there is also a `call_info`-struct. That struct contains messages of various types, that the user should be informed of.
-
-As in any request, it's important to consider that the `meta` info also contains untrusted data.
-
-The `transaction` (on input into clef) can have either `data` or `input` -- if both are set, they must be identical, otherwise an error is generated. However, Clef will always use `data` when passing this struct on (if Clef does otherwise, please file a ticket)
-
-Example:
-```json
-{
-  "transaction": {
-    "from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
-    "to": null,
-    "gas": "0x3e8",
-    "gasPrice": "0x5",
-    "value": "0x6",
-    "nonce": "0x1",
-    "data": "0x01020304"
-  },
-  "call_info": [
-    {
-      "type": "Warning",
-      "message": "Something looks odd, show this message as a warning"
-    },
-    {
-      "type": "Info",
-      "message": "User should see this as well"
-    }
-  ],
-  "meta": {
-    "remote": "localhost:9999",
-    "local": "localhost:8545",
-    "scheme": "http",
-    "User-Agent": "Firefox 3.2",
-    "Origin": "www.malicious.ru"
-  }
-}
-```
-### SignTxResponse - approve
-
-Response to request to sign a transaction. This response needs to contain the `transaction`, because the UI is free to make modifications to the transaction.
-
-Example:
-```json
-{
-  "transaction": {
-    "from": "0xDEADbEeF000000000000000000000000DeaDbeEf",
-    "to": null,
-    "gas": "0x3e8",
-    "gasPrice": "0x5",
-    "value": "0x6",
-    "nonce": "0x4",
-    "data": "0x04030201"
-  },
-  "approved": true
-}
-```
-### SignTxResponse - deny
-
-Response to SignTxRequest. When denying a request, there's no need to provide the transaction in return
-
-Example:
-```json
-{
-  "transaction": {
-    "from": "0x",
-    "to": null,
-    "gas": "0x0",
-    "gasPrice": "0x0",
-    "value": "0x0",
-    "nonce": "0x0",
-    "data": null
-  },
-  "approved": false
-}
-```
-### OnApproved - SignTransactionResult
-
-SignTransactionResult is used in the call `clef` -> `OnApprovedTx(result)`
-
-This occurs _after_ successful completion of the entire signing procedure, but right before the signed transaction is passed to the external caller. This method (and data) can be used by the UI to signal to the user that the transaction was signed, but it is primarily useful for ruleset implementations.
-
-A ruleset that implements a rate limitation needs to know what transactions are sent out to the external interface. By hooking into this methods, the ruleset can maintain track of that count.
-
-**OBS:** Note that if an attacker can restore your `clef` data to a previous point in time (e.g through a backup), the attacker can reset such windows, even if he/she is unable to decrypt the content.
-
-The `OnApproved` method cannot be responded to, it's purely informative
-
-Example:
-```json
-{
-  "raw": "0xf85d640101948a8eafb1cf62bfbeb1741769dae1a9dd47996192018026a0716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293a04e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
-  "tx": {
-    "nonce": "0x64",
-    "gasPrice": "0x1",
-    "gas": "0x1",
-    "to": "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192",
-    "value": "0x1",
-    "input": "0x",
-    "v": "0x26",
-    "r": "0x716bd90515acb1e68e5ac5867aa11a1e65399c3349d479f5fb698554ebc6f293",
-    "s": "0x4e8a4ebfff434e971e0ef12c5bf3a881b06fd04fc3f8b8a7291fb67a26a1d4ed",
-    "hash": "0x662f6d772692dd692f1b5e8baa77a9ff95bbd909362df3fc3d301aafebde5441"
-  }
-}
-```
-### UserInputRequest
-
-Sent when clef needs the user to provide data. If 'password' is true, the input field should be treated accordingly (echo-free)
-
-Example:
-```json
-{
-  "prompt": "The question to ask the user",
-  "title": "The title here",
-  "isPassword": true
-}
-```
-### UserInputResponse
-
-Response to UserInputRequest
-
-Example:
-```json
-{
-  "text": "The textual response from user"
-}
-```
-### ListRequest
-
-Sent when a request has been made to list addresses. The UI is provided with the full `account`s, including local directory names. Note: this information is not passed back to the external caller, who only sees the `address`es.
-
-Example:
-```json
-{
-  "accounts": [
-    {
-      "address": "0xdeadbeef000000000000000000000000deadbeef",
-      "url": "keystore:///path/to/keyfile/a"
-    },
-    {
-      "address": "0x1111111122222222222233333333334444444444",
-      "url": "keystore:///path/to/keyfile/b"
-    }
-  ],
-  "meta": {
-    "remote": "localhost:9999",
-    "local": "localhost:8545",
-    "scheme": "http",
-    "User-Agent": "Firefox 3.2",
-    "Origin": "www.malicious.ru"
-  }
-}
-```
-### ListResponse
-
-Response to list request. The response contains a list of all addresses to show to the caller. Note: the UI is free to respond with any address the caller, regardless of whether it exists or not
-
-Example:
-```json
-{
-  "accounts": [
-    {
-      "address": "0x0000000000000000000000000000000000000000",
-      "url": ".. ignored .."
-    },
-    {
-      "address": "0xffffffffffffffffffffffffffffffffffffffff",
-      "url": ""
-    }
-  ]
-}
-```

cmd/clef/docs/qubes/qubes-client.py

@@ -1,23 +0,0 @@
-"""
-This implements a dispatcher which listens to localhost:8550, and proxies
-requests via qrexec to the service qubes.EthSign on a target domain
-"""
-
-import http.server
-import socketserver,subprocess
-
-PORT=8550
-TARGET_DOMAIN= 'debian-work'
-
-class Dispatcher(http.server.BaseHTTPRequestHandler):
-    def do_POST(self):
-        post_data = self.rfile.read(int(self.headers['Content-Length']))
-        p = subprocess.Popen(['/usr/bin/qrexec-client-vm',TARGET_DOMAIN,'qubes.Clefsign'],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        output = p.communicate(post_data)[0]
-        self.wfile.write(output)
-
-
-with socketserver.TCPServer(("",PORT), Dispatcher) as httpd:
-    print("Serving at port", PORT)
-    httpd.serve_forever()
-

cmd/clef/docs/qubes/qubes.Clefsign

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-SIGNER_BIN="/home/user/tools/clef/clef"
-SIGNER_CMD="/home/user/tools/gtksigner/gtkui.py -s $SIGNER_BIN"
-
-# Start clef if not already started
-if [ ! -S /home/user/.clef/clef.ipc ]; then
-	$SIGNER_CMD &
-	sleep 1
-fi
-
-# Should be started by now
-if [ -S /home/user/.clef/clef.ipc ]; then
-    # Post incoming request to HTTP channel
-	curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8550 2>/dev/null
-fi

cmd/clef/docs/setup.md

@@ -1,198 +0,0 @@
-# Setting up Clef
-
-This document describes how Clef can be used in a more secure manner than executing it from your everyday laptop, 
-in order to ensure that the keys remain safe in the event that your computer should get compromised. 
-
-## Qubes OS
-
-
-### Background 
-
-The Qubes operating system is based around virtual machines (qubes), where a set of virtual machines are configured, typically for 
-different purposes such as:
-
-- personal
-   - Your personal email, browsing etc
-- work
-  - Work email etc
-- vault
-  - a VM without network access, where gpg-keys and/or keepass credentials are stored. 
-
-A couple of dedicated virtual machines handle externalities:
-
-- sys-net provides networking to all other (network-enabled) machines
-- sys-firewall handles firewall rules
-- sys-usb handles USB devices, and can map usb-devices to certain qubes.
-
-The goal of this document is to describe how we can set up clef to provide secure transaction
-signing from a `vault` vm, to another networked qube which runs Dapps.
-
-### Setup
-
-There are two ways that this can be achieved: integrated via Qubes or integrated via networking. 
-
-
-#### 1. Qubes Integrated
-
-Qubes provides a facility for inter-qubes communication via `qrexec`. A qube can request to make a cross-qube RPC request 
-to another qube. The OS then asks the user if the call is permitted. 
-
-![Example](qubes/qrexec-example.png)
-
-A policy-file can be created to allow such interaction. On the `target` domain, a service is invoked which can read the
-`stdin` from the `client` qube. 
-
-This is how [Split GPG](https://www.qubes-os.org/doc/split-gpg/) is implemented. We can set up Clef the same way:
-
-##### Server
-
-![Clef via qrexec](qubes/clef_qubes_qrexec.png)
-
-On the `target` qubes, we need to define the RPC service.
-
-[qubes.Clefsign](qubes/qubes.Clefsign):
-
-```bash
-#!/bin/bash
-
-SIGNER_BIN="/home/user/tools/clef/clef"
-SIGNER_CMD="/home/user/tools/gtksigner/gtkui.py -s $SIGNER_BIN"
-
-# Start clef if not already started
-if [ ! -S /home/user/.clef/clef.ipc ]; then
-	$SIGNER_CMD &
-	sleep 1
-fi
-
-# Should be started by now
-if [ -S /home/user/.clef/clef.ipc ]; then
-    # Post incoming request to HTTP channel
-	curl -H "Content-Type: application/json" -X POST -d @- http://localhost:8550 2>/dev/null
-fi
-
-```
-This RPC service is not complete (see notes about HTTP headers below), but works as a proof-of-concept. 
-It will forward the data received on `stdin` (forwarded by the OS) to Clef's HTTP channel.  
-
-It would have been possible to send data directly to the `/home/user/.clef/.clef.ipc` 
-socket via e.g `nc -U /home/user/.clef/clef.ipc`, but the reason for sending the request 
-data over `HTTP` instead of `IPC` is that we want the ability to forward `HTTP` headers.
-
-To enable the service:
-
-``` bash
-sudo cp qubes.Clefsign /etc/qubes-rpc/
-sudo chmod +x /etc/qubes-rpc/ qubes.Clefsign
-```
-
-This setup uses [gtksigner](https://github.com/holiman/gtksigner), which is a very minimal GTK-based UI that works well 
-with minimal requirements. 
-
-##### Client
-
-
-On the `client` qube, we need to create a listener which will receive the request from the Dapp, and proxy it. 
-
-
-[qubes-client.py](qubes/qubes-client.py):
-
-```python
-
-"""
-This implements a dispatcher which listens to localhost:8550, and proxies
-requests via qrexec to the service qubes.EthSign on a target domain
-"""
-
-import http.server
-import socketserver,subprocess
-
-PORT=8550
-TARGET_DOMAIN= 'debian-work'
-
-class Dispatcher(http.server.BaseHTTPRequestHandler):
-    def do_POST(self):
-        post_data = self.rfile.read(int(self.headers['Content-Length']))
-        p = subprocess.Popen(['/usr/bin/qrexec-client-vm',TARGET_DOMAIN,'qubes.Clefsign'],stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        output = p.communicate(post_data)[0]
-        self.wfile.write(output)
-
-
-with socketserver.TCPServer(("",PORT), Dispatcher) as httpd:
-    print("Serving at port", PORT)
-    httpd.serve_forever()
-
-
-```
-
-#### Testing
-
-To test the flow, if we have set up `debian-work` as the `target`, we can do
- 
-```bash
-$ cat newaccnt.json 
-{ "id": 0, "jsonrpc": "2.0","method": "account_new","params": []}
-
-$ cat newaccnt.json| qrexec-client-vm debian-work qubes.Clefsign
-```
-
-A dialog should pop up first to allow the IPC call:
-
-![one](qubes/qubes_newaccount-1.png)
-
-Followed by a GTK-dialog to approve the operation:
-
-![two](qubes/qubes_newaccount-2.png)
-
-To test the full flow, we use the client wrapper. Start it on the `client` qube:
-```
-[user@work qubes]$ python3 qubes-client.py 
-```
-
-Make the request over http (`client` qube):
-```
-[user@work clef]$ cat newaccnt.json | curl -X POST -d @- http://localhost:8550
-```
-And it should show the same popups again. 
-
-##### Pros and cons
-
-The benefits of this setup are:
-
-- This is the qubes-os intended model for inter-qube communication,
-- and thus benefits from qubes-os dialogs and policies for user approval
-
-However, it comes with a couple of drawbacks:
-
-- The `qubes-gpg-client` must forward the http request via RPC to the `target` qube. When doing so, the proxy
-  will either drop important headers, or replace them. 
-  - The `Host` header is most likely `localhost` 
-  - The `Origin` header must be forwarded
-  - Information about the remote ip must be added as a `X-Forwarded-For`. However, Clef cannot always trust an `XFF` header, 
-  since malicious clients may lie about `XFF` in order to fool the http server into believing it comes from another address.
-- Even with a policy in place to allow RPC calls between `caller` and `target`, there will be several popups:
-  - One qubes-specific where the user specifies the `target` vm
-  - One clef-specific to approve the transaction
-  
-
-#### 2. Network integrated
-
-The second way to set up Clef on a qubes system is to allow networking, and have Clef listen to a port which is accessible
-from other qubes. 
-
-![Clef via http](qubes/clef_qubes_http.png)
-
-
-
-
-## USBArmory
-
-The [USB armory](https://inversepath.com/usbarmory) is an open source hardware design with an 800 MHz ARM processor. It is a pocket-size
-computer. When inserted into a laptop, it identifies itself as a USB network interface, basically adding another network
-to your computer. Over this new network interface, you can SSH into the device. 
-
-Running Clef off a USB armory means that you can use the armory as a very versatile offline computer, which only
-ever connects to a local network between your computer and the device itself.
-
-Needless to say, while this model should be fairly secure against remote attacks, an attacker with physical access
-to the USB Armory would trivially be able to extract the contents of the device filesystem. 
-

cmd/clef/extapi_changelog.md

@@ -1,104 +0,0 @@
-## Changelog for external API
-
-The API uses [semantic versioning](https://semver.org/).
-
-TL;DR: Given a version number MAJOR.MINOR.PATCH, increment the:
-
-* MAJOR version when you make incompatible API changes,
-* MINOR version when you add functionality in a backwards-compatible manner, and
-* PATCH version when you make backwards-compatible bug fixes.
-
-Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.
-
-### 6.1.0
-
-The API-method `account_signGnosisSafeTx` was added. This method takes two parameters, 
-`[address, safeTx]`. The latter, `safeTx`, can be copy-pasted from the gnosis relay. For example: 
-
-```
-{
-  "jsonrpc": "2.0",
-  "method": "account_signGnosisSafeTx",
-  "params": ["0xfd1c4226bfD1c436672092F4eCbfC270145b7256",
-    {
-      "safe": "0x25a6c4BBd32B2424A9c99aEB0584Ad12045382B3",
-      "to": "0xB372a646f7F05Cc1785018dBDA7EBc734a2A20E2",
-      "value": "20000000000000000",
-      "data": null,
-      "operation": 0,
-      "gasToken": "0x0000000000000000000000000000000000000000",
-      "safeTxGas": 27845,
-      "baseGas": 0,
-      "gasPrice": "0",
-      "refundReceiver": "0x0000000000000000000000000000000000000000",
-      "nonce": 2,
-      "executionDate": null,
-      "submissionDate": "2020-09-15T21:54:49.617634Z",
-      "modified": "2020-09-15T21:54:49.617634Z",
-      "blockNumber": null,
-      "transactionHash": null,
-      "safeTxHash": "0x2edfbd5bc113ff18c0631595db32eb17182872d88d9bf8ee4d8c2dd5db6d95e2",
-      "executor": null,
-      "isExecuted": false,
-      "isSuccessful": null,
-      "ethGasPrice": null,
-      "gasUsed": null,
-      "fee": null,
-      "origin": null,
-      "dataDecoded": null,
-      "confirmationsRequired": null,
-      "confirmations": [
-        {
-          "owner": "0xAd2e180019FCa9e55CADe76E4487F126Fd08DA34",
-          "submissionDate": "2020-09-15T21:54:49.663299Z",
-          "transactionHash": null,
-          "confirmationType": "CONFIRMATION",
-          "signature": "0x95a7250bb645f831c86defc847350e7faff815b2fb586282568e96cc859e39315876db20a2eed5f7a0412906ec5ab57652a6f645ad4833f345bda059b9da2b821c",
-          "signatureType": "EOA"
-        }
-      ],
-      "signatures": null
-    }
-  ],
-  "id": 67
-}
-```
-
-Not all fields are required, though. This method is really just a UX helper, which massages the 
-input to conform to the `EIP-712` [specification](https://docs.safe.global/core-api/transaction-service-reference/gnosis) 
-for the Gnosis Safe, and making the output be directly importable to by a relay service. 
-
-
-### 6.0.0
-
-* `New` was changed to deliver only an address, not the full `Account` data
-* `Export` was moved from External API to the UI Server API
-
-#### 5.0.0
-
-* The external `account_EcRecover`-method was reimplemented.
-* The external method `account_sign(address, data)` was replaced with `account_signData(contentType, address, data)`.
-The addition of `contentType` makes it possible to use the method for different types of objects, such as:
-  * signing data with an intended validator (not yet implemented)
-  * signing clique headers,
-  * signing plain personal messages,
-* The external method `account_signTypedData` implements [EIP-712](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-712.md) and makes it possible to sign typed data.
-
-#### 4.0.0
-
-* The external `account_Ecrecover`-method was removed.
-* The external `account_Import`-method was removed.
-
-#### 3.0.0
-
-* The external `account_List`-method was changed to not expose `url`, which contained info about the local filesystem. It now returns only a list of addresses.
-
-#### 2.0.0
-
-* Commit `73abaf04b1372fa4c43201fb1b8019fe6b0a6f8d`, move `from` into `transaction` object in `signTransaction`. This
-makes the `accounts_signTransaction` identical to the old `eth_signTransaction`.
-
-
-#### 1.0.0
-
-Initial release.

cmd/clef/intapi_changelog.md

@@ -1,191 +0,0 @@
-## Changelog for internal API (ui-api)
-
-The API uses [semantic versioning](https://semver.org/).
-
-TL;DR: Given a version number MAJOR.MINOR.PATCH, increment the:
-
-* MAJOR version when you make incompatible API changes,
-* MINOR version when you add functionality in a backwards-compatible manner, and
-* PATCH version when you make backwards-compatible bug fixes.
-
-Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format.
-
-### 7.0.1 
-
-Added `clef_New` to the internal API callable from a UI.
-
-> `New` creates a new password protected Account. The private key is protected with
-> the given password. Users are responsible to backup the private key that is stored
-> in the keystore location that was specified when this API was created.
-> This method is the same as New on the external API, the difference being that
-> this implementation does not ask for confirmation, since it's initiated by
-> the user
-
-### 7.0.0
-
-- The `message` field was renamed to `messages` in all data signing request methods to better reflect that it's a list, not a value.
-- The `storage.Put` and `storage.Get` methods in the rule execution engine were lower-cased to `storage.put` and `storage.get` to be consistent with JavaScript call conventions.
-
-### 6.0.0
-
-Removed `password` from responses to operations which require them. This is for two reasons,
-
-- Consistency between how rulesets operate and how manual processing works. A rule can `Approve` but require the actual password to be stored in the clef storage.
-With this change, the same stored password can be used even if rulesets are not enabled, but storage is.
-- It also removes the usability-shortcut that a UI might otherwise want to implement; remembering passwords. Since we now will not require the
-password on every `Approve`, there's no need for the UI to cache it locally.
-  - In a future update, we'll likely add `clef_storePassword` to the internal API, so the user can store it via his UI (currently only CLI works).
-
-Affected datatypes:
-- `SignTxResponse`
-- `SignDataResponse`
-- `NewAccountResponse`
-
-If `clef` requires a password, the `OnInputRequired` will be used to collect it.
-
-
-### 5.0.0
-
-Changed the namespace format to adhere to the legacy ethereum format: `name_methodName`. Changes:
-
-* `ApproveTx` -> `ui_approveTx`
-* `ApproveSignData` -> `ui_approveSignData`
-* `ApproveExport` -> `removed`
-* `ApproveImport`  -> `removed`
-* `ApproveListing`  -> `ui_approveListing`
-* `ApproveNewAccount`  -> `ui_approveNewAccount`
-* `ShowError` -> `ui_showError`
-* `ShowInfo` -> `ui_showInfo`
-* `OnApprovedTx` -> `ui_onApprovedTx`
-* `OnSignerStartup` -> `ui_onSignerStartup`
-* `OnInputRequired` -> `ui_onInputRequired`
-
-
-### 4.0.0
-
-* Bidirectional communication implemented, so the UI can query `clef` via the stdin/stdout RPC channel. Methods implemented are:
-  - `clef_listWallets`
-  - `clef_listAccounts`
-  - `clef_listWallets`
-  - `clef_deriveAccount`
-  - `clef_importRawKey`
-  - `clef_openWallet`
-  - `clef_chainId`
-  - `clef_setChainId`
-  - `clef_export`
-  - `clef_import`
-
-* The type `Account` was modified (the json-field `type` was removed), to consist of
-
-```go
-type Account struct {
-	Address common.Address `json:"address"` // Ethereum account address derived from the key
-	URL     URL            `json:"url"`     // Optional resource locator within a backend
-}
-```
-
-
-### 3.2.0
-
-* Make `ShowError`, `OnApprovedTx`, `OnSignerStartup` be json-rpc [notifications](https://www.jsonrpc.org/specification#notification):
-
-> A Notification is a Request object without an "id" member. A Request object that is a Notification signifies the Client's lack of interest in the corresponding Response object, and as such no Response object needs to be returned to the client. The Server MUST NOT reply to a Notification, including those that are within a batch request.
->
->  Notifications are not confirmable by definition, since they do not have a Response object to be returned. As such, the Client would not be aware of any errors (like e.g. "Invalid params","Internal error"
-### 3.1.0
-
-* Add `ContentType` `string` to `SignDataRequest` to accommodate the latest [EIP-191](https://eips.ethereum.org/EIPS/eip-191) and [EIP-712](https://eips.ethereum.org/EIPS/eip-712) implementations.
-
-### 3.0.0
-
-* Make use of `OnInputRequired(info UserInputRequest)` for obtaining master password during startup
-
-### 2.1.0
-
-* Add `OnInputRequired(info UserInputRequest)` to internal API. This method is used when Clef needs user input, e.g. passwords.
-
-The following structures are used:
-
-```go
-UserInputRequest struct {
-	Prompt     string `json:"prompt"`
-	Title      string `json:"title"`
-	IsPassword bool   `json:"isPassword"`
-}
-UserInputResponse struct {
-	Text string `json:"text"`
-}
-```
-
-### 2.0.0
-
-* Modify how `call_info` on a transaction is conveyed. New format:
-
-```
-{
-  "jsonrpc": "2.0",
-  "id": 2,
-  "method": "ApproveTx",
-  "params": [
-    {
-      "transaction": {
-        "from": "0x82A2A876D39022B3019932D30Cd9c97ad5616813",
-        "to": "0x07a565b7ed7d7a678680a4c162885bedbb695fe0",
-        "gas": "0x333",
-        "gasPrice": "0x123",
-        "value": "0x10",
-        "nonce": "0x0",
-        "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012",
-        "input": null
-      },
-      "call_info": [
-        {
-          "type": "WARNING",
-          "message": "Invalid checksum on to-address"
-        },
-        {
-          "type": "WARNING",
-          "message": "Tx contains data, but provided ABI signature could not be matched: Did not match: test (0 matches)"
-        }
-      ],
-      "meta": {
-        "remote": "127.0.0.1:54286",
-        "local": "localhost:8550",
-        "scheme": "HTTP/1.1"
-      }
-    }
-  ]
-}
-```
-
-#### 1.2.0
-
-* Add `OnStartup` method, to provide the UI with information about what API version
-the signer uses (both internal and external) as well as build-info and external api.
-
-Example call:
-```json
-{
-  "jsonrpc": "2.0",
-  "id": 1,
-  "method": "OnSignerStartup",
-  "params": [
-    {
-      "info": {
-        "extapi_http": "http://localhost:8550",
-        "extapi_ipc": null,
-        "extapi_version": "2.0.0",
-        "intapi_version": "1.2.0"
-      }
-    }
-  ]
-}
-```
-
-#### 1.1.0
-
-* Add `OnApproved` method
-
-#### 1.0.0
-
-Initial release.

cmd/clef/pythonsigner.py

@@ -1,315 +0,0 @@
-import sys
-import subprocess
-
-from tinyrpc.transports import ServerTransport
-from tinyrpc.protocols.jsonrpc import JSONRPCProtocol
-from tinyrpc.dispatch import public, RPCDispatcher
-from tinyrpc.server import RPCServer
-
-"""
-This is a POC example of how to write a custom UI for Clef.
-The UI starts the clef process with the '--stdio-ui' option
-and communicates with clef using standard input / output.
-
-The standard input/output is a relatively secure way to communicate,
-as it does not require opening any ports or IPC files. Needless to say,
-it does not protect against memory inspection mechanisms
-where an attacker can access process memory.
-
-To make this work install all the requirements:
-
-  pip install -r requirements.txt
-"""
-
-try:
-    import urllib.parse as urlparse
-except ImportError:
-    import urllib as urlparse
-
-
-class StdIOTransport(ServerTransport):
-    """Uses std input/output for RPC"""
-
-    def receive_message(self):
-        return None, urlparse.unquote(sys.stdin.readline())
-
-    def send_reply(self, context, reply):
-        print(reply)
-
-
-class PipeTransport(ServerTransport):
-    """Uses std a pipe for RPC"""
-
-    def __init__(self, input, output):
-        self.input = input
-        self.output = output
-
-    def receive_message(self):
-        data = self.input.readline()
-        print(">> {}".format(data))
-        return None, urlparse.unquote(data)
-
-    def send_reply(self, context, reply):
-        reply = str(reply, "utf-8")
-        print("<< {}".format(reply))
-        self.output.write("{}\n".format(reply))
-
-
-def sanitize(txt, limit=100):
-    return txt[:limit].encode("unicode_escape").decode("utf-8")
-
-
-def metaString(meta):
-    """
-    "meta":{"remote":"clef binary","local":"main","scheme":"in-proc","User-Agent":"","Origin":""}
-    """  # noqa: E501
-    message = (
-        "\tRequest context:\n"
-        "\t\t{remote} -> {scheme} -> {local}\n"
-        "\tAdditional HTTP header data, provided by the external caller:\n"
-        "\t\tUser-Agent: {user_agent}\n"
-        "\t\tOrigin: {origin}\n"
-    )
-    return message.format(
-        remote=meta.get("remote", "<missing>"),
-        scheme=meta.get("scheme", "<missing>"),
-        local=meta.get("local", "<missing>"),
-        user_agent=sanitize(meta.get("User-Agent"), 200),
-        origin=sanitize(meta.get("Origin"), 100),
-    )
-
-
-class StdIOHandler:
-    def __init__(self):
-        pass
-
-    @public
-    def approveTx(self, req):
-        """
-        Example request:
-
-        {"jsonrpc":"2.0","id":20,"method":"ui_approveTx","params":[{"transaction":{"from":"0xDEADbEeF000000000000000000000000DeaDbeEf","to":"0xDEADbEeF000000000000000000000000DeaDbeEf","gas":"0x3e8","gasPrice":"0x5","maxFeePerGas":null,"maxPriorityFeePerGas":null,"value":"0x6","nonce":"0x1","data":"0x"},"call_info":null,"meta":{"remote":"clef binary","local":"main","scheme":"in-proc","User-Agent":"","Origin":""}}]}
-
-        :param transaction: transaction info
-        :param call_info: info about the call, e.g. if ABI info could not be
-        :param meta: metadata about the request, e.g. where the call comes from
-        :return:
-        """  # noqa: E501
-        message = (
-            "Sign transaction request:\n"
-            "\t{meta_string}\n"
-            "\n"
-            "\tFrom: {from_}\n"
-            "\tTo: {to}\n"
-            "\n"
-            "\tAuto-rejecting request"
-        )
-        meta = req.get("meta", {})
-        transaction = req.get("transaction")
-        sys.stdout.write(
-            message.format(
-                meta_string=metaString(meta),
-                from_=transaction.get("from", "<missing>"),
-                to=transaction.get("to", "<missing>"),
-            )
-        )
-        return {
-            "approved": False,
-        }
-
-    @public
-    def approveSignData(self, req):
-        """
-        Example request:
-
-        {"jsonrpc":"2.0","id":8,"method":"ui_approveSignData","params":[{"content_type":"application/x-clique-header","address":"0x0011223344556677889900112233445566778899","raw_data":"+QIRoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAlAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAuQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIIFOYIFOYIFOoIFOoIFOppFeHRyYSBkYXRhIEV4dHJhIGRhdGEgRXh0cqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIgAAAAAAAAAAA==","messages":[{"name":"Clique header","value":"clique header 1337 [0x44381ab449d77774874aca34634cb53bc21bd22aef2d3d4cf40e51176cb585ec]","type":"clique"}],"call_info":null,"hash":"0xa47ab61438a12a06c81420e308c2b7aae44e9cd837a5df70dd021421c0f58643","meta":{"remote":"clef binary","local":"main","scheme":"in-proc","User-Agent":"","Origin":""}}]}
-        """  # noqa: E501
-        message = (
-            "Sign data request:\n"
-            "\t{meta_string}\n"
-            "\n"
-            "\tContent-type: {content_type}\n"
-            "\tAddress: {address}\n"
-            "\tHash: {hash_}\n"
-            "\n"
-            "\tAuto-rejecting request\n"
-        )
-        meta = req.get("meta", {})
-        sys.stdout.write(
-            message.format(
-                meta_string=metaString(meta),
-                content_type=req.get("content_type"),
-                address=req.get("address"),
-                hash_=req.get("hash"),
-            )
-        )
-
-        return {
-            "approved": False,
-            "password": None,
-        }
-
-    @public
-    def approveNewAccount(self, req):
-        """
-        Example request:
-
-        {"jsonrpc":"2.0","id":25,"method":"ui_approveNewAccount","params":[{"meta":{"remote":"clef binary","local":"main","scheme":"in-proc","User-Agent":"","Origin":""}}]}
-        """  # noqa: E501
-        message = (
-            "Create new account request:\n"
-            "\t{meta_string}\n"
-            "\n"
-            "\tAuto-rejecting request\n"
-        )
-        meta = req.get("meta", {})
-        sys.stdout.write(message.format(meta_string=metaString(meta)))
-        return {
-            "approved": False,
-        }
-
-    @public
-    def showError(self, req):
-        """
-        Example request:
-
-        {"jsonrpc":"2.0","method":"ui_showError","params":[{"text":"If you see this message, enter 'yes' to the next question"}]}
-
-        :param message: to display
-        :return:nothing
-        """  # noqa: E501
-        message = (
-            "## Error\n{text}\n"
-            "Press enter to continue\n"
-        )
-        text = req.get("text")
-        sys.stdout.write(message.format(text=text))
-        input()
-        return
-
-    @public
-    def showInfo(self, req):
-        """
-        Example request:
-
-        {"jsonrpc":"2.0","method":"ui_showInfo","params":[{"text":"If you see this message, enter 'yes' to next question"}]}
-
-        :param message: to display
-        :return:nothing
-        """  # noqa: E501
-        message = (
-            "## Info\n{text}\n"
-            "Press enter to continue\n"
-        )
-        text = req.get("text")
-        sys.stdout.write(message.format(text=text))
-        input()
-        return
-
-    @public
-    def onSignerStartup(self, req):
-        """
-        Example request:
-
-        {"jsonrpc":"2.0", "method":"ui_onSignerStartup", "params":[{"info":{"extapi_http":"n/a","extapi_ipc":"/home/user/.clef/clef.ipc","extapi_version":"6.1.0","intapi_version":"7.0.1"}}]}
-        """  # noqa: E501
-        message = (
-            "\n"
-            "\t\tExt api url: {extapi_http}\n"
-            "\t\tInt api ipc: {extapi_ipc}\n"
-            "\t\tExt api ver: {extapi_version}\n"
-            "\t\tInt api ver: {intapi_version}\n"
-        )
-        info = req.get("info")
-        sys.stdout.write(
-            message.format(
-                extapi_http=info.get("extapi_http"),
-                extapi_ipc=info.get("extapi_ipc"),
-                extapi_version=info.get("extapi_version"),
-                intapi_version=info.get("intapi_version"),
-            )
-        )
-
-    @public
-    def approveListing(self, req):
-        """
-        Example request:
-
-        {"jsonrpc":"2.0","id":23,"method":"ui_approveListing","params":[{"accounts":[{"address":...
-        """  # noqa: E501
-        message = (
-            "\n"
-            "## Account listing request\n"
-            "\t{meta_string}\n"
-            "\tDo you want to allow listing the following accounts?\n"
-            "\t-{addrs}\n"
-            "\n"
-            "->Auto-answering No\n"
-        )
-        meta = req.get("meta", {})
-        accounts = req.get("accounts", [])
-        addrs = [x.get("address") for x in accounts]
-        sys.stdout.write(
-            message.format(
-                addrs="\n\t-".join(addrs),
-                meta_string=metaString(meta)
-            )
-        )
-        return {}
-
-    @public
-    def onInputRequired(self, req):
-        """
-        Example request:
-
-        {"jsonrpc":"2.0","id":1,"method":"ui_onInputRequired","params":[{"title":"Master Password","prompt":"Please enter the password to decrypt the master seed","isPassword":true}]}
-
-        :param message: to display
-        :return:nothing
-        """  # noqa: E501
-        message = (
-            "\n"
-            "## {title}\n"
-            "\t{prompt}\n"
-            "\n"
-            "> "
-        )
-        sys.stdout.write(
-            message.format(
-                title=req.get("title"),
-                prompt=req.get("prompt")
-            )
-        )
-        isPassword = req.get("isPassword")
-        if not isPassword:
-            return {"text": input()}
-
-        return ""
-
-
-def main(args):
-    cmd = ["clef", "--stdio-ui"]
-    if len(args) > 0 and args[0] == "test":
-        cmd.extend(["--stdio-ui-test"])
-    print("cmd: {}".format(" ".join(cmd)))
-
-    dispatcher = RPCDispatcher()
-    dispatcher.register_instance(StdIOHandler(), "ui_")
-
-    # line buffered
-    p = subprocess.Popen(
-        cmd,
-        bufsize=1,
-        universal_newlines=True,
-        stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-    )
-
-    rpc_server = RPCServer(
-        PipeTransport(p.stdout, p.stdin), JSONRPCProtocol(), dispatcher
-    )
-    rpc_server.serve_forever()
-
-
-if __name__ == "__main__":
-    main(sys.argv[1:])

cmd/clef/requirements.txt

@@ -1 +0,0 @@
-tinyrpc==1.1.4

cmd/clef/rules.md

@@ -1,234 +0,0 @@
-# Rules
-
-The `signer` binary contains a ruleset engine, implemented with [OttoVM](https://github.com/robertkrimen/otto)
-
-It enables use cases like the following:
-
-* I want to auto-approve transactions with contract `CasinoDapp`, with up to `0.05 ether` in value to maximum `1 ether` per 24h period
-* I want to auto-approve transaction to contract `EthAlarmClock` with `data`=`0xdeadbeef`, if `value=0`, `gas < 44k` and `gasPrice < 40Gwei`
-
-The two main features that are required for this to work well are:
-
-1. Rule Implementation: how to create, manage, and interpret rules in a flexible but secure manner
-2. Credential management and credentials; how to provide auto-unlock without exposing keys unnecessarily.
-
-The section below deals with both of them
-
-## Rule Implementation
-
-A ruleset file is implemented as a `js` file. Under the hood, the ruleset engine is a `SignerUI`, implementing the same methods as the `json-rpc` methods
-defined in the UI protocol. Example:
-
-```js
-function asBig(str) {
-	if (str.slice(0, 2) == "0x") {
-		return new BigNumber(str.slice(2), 16)
-	}
-	return new BigNumber(str)
-}
-
-// Approve transactions to a certain contract if the value is below a certain limit
-function ApproveTx(req) {
-	var limit = new BigNumber("0xb1a2bc2ec50000")
-	var value = asBig(req.transaction.value);
-
-	if (req.transaction.to.toLowerCase() == "0xae967917c465db8578ca9024c205720b1a3651a9" && value.lt(limit)) {
-		return "Approve"
-	}
-	// If we return "Reject", it will be rejected.
-	// By not returning anything, it will be passed to the next UI, for manual processing
-}
-
-// Approve listings if request made from IPC
-function ApproveListing(req){
-    if (req.metadata.scheme == "ipc"){ return "Approve"}
-}
-```
-
-Whenever the external API is called (and the ruleset is enabled), the `signer` calls the UI, which is an instance of a ruleset-engine. The ruleset-engine
-invokes the corresponding method. In doing so, there are three possible outcomes:
-
-1. JS returns "Approve"
-  * Auto-approve request
-2. JS returns "Reject"
-  * Auto-reject request
-3. Error occurs, or something else is returned
-  * Pass on to `next` ui: the regular UI channel.
-
-A more advanced example can be found below, "Example 1: ruleset for a rate-limited window", using `storage` to `Put` and `Get` `string`s by key.
-
-* At the time of writing, storage only exists as an ephemeral unencrypted implementation, to be used during testing.
-
-### Things to note
-
-The Otto vm has a few [caveats](https://github.com/robertkrimen/otto):
-
-* "use strict" will parse, but does nothing.
-* The regular expression engine (re2/regexp) is not fully compatible with the ECMA5 specification.
-* Otto targets ES5. ES6 features (eg: Typed Arrays) are not supported.
-
-Additionally, a few more have been added
-
-* The rule execution cannot load external javascript files.
-* The only preloaded library is [`bignumber.js`](https://github.com/MikeMcl/bignumber.js) version `2.0.3`. This one is fairly old, and is not aligned with the documentation at the GitHub repository.
-* Each invocation is made in a fresh virtual machine. This means that you cannot store data in global variables between invocations. This is a deliberate choice -- if you want to store data, use the disk-backed `storage`, since rules should not rely on ephemeral data.
-* Javascript API parameters are _always_ an object. This is also a design choice, to ensure that parameters are accessed by _key_ and not by order. This is to prevent mistakes due to missing parameters or parameter changes.
-* The JS engine has access to `storage` and `console`.
-
-#### Security considerations
-
-##### Security of ruleset
-
-Some security precautions can be made, such as:
-
-* Never load `ruleset.js` unless the file is `readonly` (`r-??-??-?`). If the user wishes to modify the ruleset, he must make it writeable and then set back to readonly.
-  * This is to prevent attacks where files are dropped on the users disk.
-* Since we're going to have to have some form of secure storage (not defined in this section), we could also store the `sha3` of the `ruleset.js` file in there.
-  * If the user wishes to modify the ruleset, he'd then have to perform e.g. `signer --attest /path/to/ruleset --credential <creds>`
-
-##### Security of implementation
-
-The drawback of this very flexible solution is that the `signer` needs to contain a javascript engine. This is pretty simple to implement since it's already
-implemented for `geth`. There are no known security vulnerabilities in it, nor have we had any security problems with it so far.
-
-The javascript engine would be an added attack surface; but if the validation of `rulesets` is made good (with hash-based attestation), the actual javascript cannot be considered
-an attack surface -- if an attacker can control the ruleset, a much simpler attack would be to implement an "always-approve" rule instead of exploiting the js vm. The only benefit
-to be gained from attacking the actual `signer` process from the `js` side would be if it could somehow extract cryptographic keys from memory.
-
-##### Security in usability
-
-Javascript is flexible, but also easy to get wrong, especially when users assume that `js` can handle large integers natively. Typical errors
-include trying to multiply `gasCost` with `gas` without using `bigint`:s.
-
-It's unclear whether any other DSL could be more secure; since there's always the possibility of erroneously implementing a rule.
-
-
-## Credential management
-
-The ability to auto-approve transactions means that the signer needs to have the necessary credentials to decrypt keyfiles. These passwords are hereafter called `ksp` (keystore pass).
-
-### Example implementation
-
-Upon startup of the signer, the signer is given a switch: `--seed <path/to/masterseed>`
-The `seed` contains a blob of bytes, which is the master seed for the `signer`.
-
-The `signer` uses the `seed` to:
-
-* Generate the `path` where the settings are stored.
-  * `./settings/1df094eb-c2b1-4689-90dd-790046d38025/vault.dat`
-  * `./settings/1df094eb-c2b1-4689-90dd-790046d38025/rules.js`
-* Generate the encryption password for `vault.dat`.
-
-The `vault.dat` would be an encrypted container storing the following information:
-
-* `ksp` entries
-* `sha256` hash of `rules.js`
-* Information about pair:ed callers (not yet specified)
-
-### Security considerations
-
-This would leave it up to the user to ensure that the `path/to/masterseed` is handled securely. It's difficult to get around this, although one could
-imagine leveraging OS-level keychains where supported. The setup is however, in general, similar to how ssh-keys are stored in `.ssh/`.
-
-
-# Implementation status
-
-This is now implemented (with ephemeral non-encrypted storage for now, so not yet enabled).
-
-## Example 1: ruleset for a rate-limited window
-
-
-```js
-function big(str) {
-	if (str.slice(0, 2) == "0x") {
-		return new BigNumber(str.slice(2), 16)
-	}
-	return new BigNumber(str)
-}
-
-// Time window: 1 week
-var window = 1000* 3600*24*7;
-
-// Limit: 1 ether
-var limit = new BigNumber("1e18");
-
-function isLimitOk(transaction) {
-	var value = big(transaction.value)
-	// Start of our window function
-	var windowstart = new Date().getTime() - window;
-
-	var txs = [];
-	var stored = storage.get('txs');
-
-	if (stored != "") {
-		txs = JSON.parse(stored)
-	}
-	// First, remove all that has passed out of the time window
-	var newtxs = txs.filter(function(tx){return tx.tstamp > windowstart});
-	console.log(txs, newtxs.length);
-
-	// Secondly, aggregate the current sum
-	sum = new BigNumber(0)
-
-	sum = newtxs.reduce(function(agg, tx){ return big(tx.value).plus(agg)}, sum);
-	console.log("ApproveTx > Sum so far", sum);
-	console.log("ApproveTx > Requested", value.toNumber());
-
-	// Would we exceed the weekly limit ?
-	return sum.plus(value).lt(limit)
-
-}
-function ApproveTx(r) {
-	if (isLimitOk(r.transaction)) {
-		return "Approve"
-	}
-	return "Nope"
-}
-
-/**
-* OnApprovedTx(str) is called when a transaction has been approved and signed. The parameter
-	* 'response_str' contains the return value that will be sent to the external caller.
-* The return value from this method is ignore - the reason for having this callback is to allow the
-* ruleset to keep track of approved transactions.
-*
-* When implementing rate-limited rules, this callback should be used.
-* If a rule responds with neither 'Approve' nor 'Reject' - the tx goes to manual processing. If the user
-* then accepts the transaction, this method will be called.
-*
-* TLDR; Use this method to keep track of signed transactions, instead of using the data in ApproveTx.
-*/
-function OnApprovedTx(resp) {
-	var value = big(resp.tx.value)
-	var txs = []
-	// Load stored transactions
-	var stored = storage.get('txs');
-	if (stored != "") {
-		txs = JSON.parse(stored)
-	}
-	// Add this to the storage
-	txs.push({tstamp: new Date().getTime(), value: value});
-	storage.put("txs", JSON.stringify(txs));
-}
-```
-
-## Example 2: allow destination
-
-```js
-function ApproveTx(r) {
-	if (r.transaction.from.toLowerCase() == "0x0000000000000000000000000000000000001337") {
-		return "Approve"
-	}
-	if (r.transaction.from.toLowerCase() == "0x000000000000000000000000000000000000dead") {
-		return "Reject"
-	}
-	// Otherwise goes to manual processing
-}
-```
-
-## Example 3: Allow listing
-
-```js
-function ApproveListing() {
-	return "Approve"
-}
-```

cmd/clef/run_test.go

@@ -1,103 +0,0 @@
-// Copyright 2022 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/internal/cmdtest"
-	"github.com/ethereum/go-ethereum/internal/reexec"
-)
-
-const registeredName = "clef-test"
-
-type testproc struct {
-	*cmdtest.TestCmd
-
-	// template variables for expect
-	Datadir   string
-	Etherbase string
-}
-
-func init() {
-	reexec.Register(registeredName, func() {
-		if err := app.Run(os.Args); err != nil {
-			fmt.Fprintln(os.Stderr, err)
-			os.Exit(1)
-		}
-		os.Exit(0)
-	})
-}
-
-func TestMain(m *testing.M) {
-	// check if we have been reexec'd
-	if reexec.Init() {
-		return
-	}
-	os.Exit(m.Run())
-}
-
-// runClef spawns clef with the given command line args and adds keystore arg.
-// This method creates a temporary  keystore folder which will be removed after
-// the test exits.
-func runClef(t *testing.T, args ...string) *testproc {
-	ddir := t.TempDir()
-	return runWithKeystore(t, ddir, args...)
-}
-
-// runWithKeystore spawns clef with the given command line args and adds keystore arg.
-// This method does _not_ create the keystore folder, but it _does_ add the arg
-// to the args.
-func runWithKeystore(t *testing.T, keystore string, args ...string) *testproc {
-	args = append([]string{"--keystore", keystore}, args...)
-	tt := &testproc{Datadir: keystore}
-	tt.TestCmd = cmdtest.NewTestCmd(t, tt)
-	// Boot "clef". This actually runs the test binary but the TestMain
-	// function will prevent any tests from running.
-	tt.Run(registeredName, args...)
-	return tt
-}
-
-func (proc *testproc) input(text string) *testproc {
-	proc.TestCmd.InputLine(text)
-	return proc
-}
-
-/*
-// waitForEndpoint waits for the rpc endpoint to appear, or
-// aborts after 3 seconds.
-func (proc *testproc) waitForEndpoint(t *testing.T) *testproc {
-	t.Helper()
-	timeout := 3 * time.Second
-	ipc := filepath.Join(proc.Datadir, "clef.ipc")
-
-	start := time.Now()
-	for time.Since(start) < timeout {
-		if _, err := os.Stat(ipc); !errors.Is(err, os.ErrNotExist) {
-			t.Logf("endpoint %v opened", ipc)
-			return proc
-		}
-		time.Sleep(200 * time.Millisecond)
-	}
-	t.Logf("stderr: \n%v", proc.StderrText())
-	t.Logf("stdout: \n%v", proc.Output())
-	t.Fatal("endpoint", ipc, "did not open within", timeout)
-	return proc
-}
-*/

cmd/clef/testdata/sign_1559_missing_field_exp_fail.json

@@ -1,16 +0,0 @@
-{
-  "jsonrpc": "2.0",
-  "method": "account_signTransaction",
-  "params": [
-    {
-      "from": "0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192",
-      "to": "0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192",
-      "gas": "0x333",
-      "maxFeePerGas": "0x123",
-      "nonce": "0x0",
-      "value": "0x10",
-      "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
-    }
-  ],
-  "id": 67
-}

cmd/clef/testdata/sign_1559_missing_maxfeepergas_exp_fail.json

@@ -1,16 +0,0 @@
-{
-  "jsonrpc": "2.0",
-  "method": "account_signTransaction",
-  "params": [
-    {
-      "from": "0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192",
-      "to": "0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192",
-      "gas": "0x333",
-      "maxPriorityFeePerGas": "0x123",
-      "nonce": "0x0",
-      "value": "0x10",
-      "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
-    }
-  ],
-  "id": 67
-}

cmd/clef/testdata/sign_1559_tx.json

@@ -1,17 +0,0 @@
-{
-  "jsonrpc": "2.0",
-  "method": "account_signTransaction",
-  "params": [
-    {
-      "from": "0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192",
-      "to": "0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192",
-      "gas": "0x333",
-      "maxPriorityFeePerGas": "0x123",
-      "maxFeePerGas": "0x123",
-      "nonce": "0x0",
-      "value": "0x10",
-      "data": "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
-    }
-  ],
-  "id": 67
-}

cmd/clef/testdata/sign_bad_checksum_exp_fail.json

@@ -1,17 +0,0 @@
-{
-  "jsonrpc": "2.0",
-  "method": "account_signTransaction",
-  "params": [
-    {
-	"from":"0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192",
-	"to":"0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192",
-      "gas": "0x333",
-      "gasPrice": "0x123",
-      "nonce": "0x0",
-      "value": "0x10",
-      "data":
-        "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
-    }
-  ],
-  "id": 67
-}

cmd/clef/testdata/sign_normal_exp_ok.json

@@ -1,17 +0,0 @@
-{
-  "jsonrpc": "2.0",
-  "method": "account_signTransaction",
-  "params": [
-    {
-	"from":"0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192",
-	"to":"0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192",
-      "gas": "0x333",
-      "gasPrice": "0x123",
-      "nonce": "0x0",
-      "value": "0x10",
-      "data":
-        "0x4401a6e40000000000000000000000000000000000000000000000000000000000000012"
-    }
-  ],
-  "id": 67
-}

cmd/clef/tests/testsigner.js

@@ -1,89 +0,0 @@
-// Copyright 2019 The go-ethereum Authors
-// This file is part of go-ethereum.
-//
-// go-ethereum is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// go-ethereum is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
-
-// This file is a test-utility for testing clef-functionality
-//
-// Start clef with
-//
-// build/bin/clef --4bytedb=./cmd/clef/4byte.json --rpc
-//
-// Start geth with
-//
-// build/bin/geth --nodiscover --maxpeers 0 --signer http://localhost:8550 console --preload=cmd/clef/tests/testsigner.js
-//
-// and in the console simply invoke
-//
-// > test()
-//
-// You can reload the file via `reload()`
-
-function reload(){
-	loadScript("./cmd/clef/tests/testsigner.js");
-}
-
-function init(){
-    if (typeof accts == 'undefined' || accts.length == 0){
-        accts = eth.accounts
-        console.log("Got accounts ", accts);
-    }
-}
-init()
-function testTx(){
-    if( accts && accts.length > 0) {
-        var a = accts[0]
-        var txdata = eth.signTransaction({from: a, to: a, value: 1, nonce: 1, gas: 1, gasPrice: 1})
-        var v = parseInt(txdata.tx.v)
-        console.log("V value: ", v)
-        if (v == 37 || v == 38){
-            console.log("Mainnet 155-protected chainid was used")
-        }
-        if (v == 27 || v == 28){
-            throw new Error("Mainnet chainid was used, but without replay protection!")
-        }
-    }
-}
-function testSignText(){
-    if( accts && accts.length > 0){
-        var a = accts[0]
-        var r = eth.sign(a, "0x68656c6c6f20776f726c64"); //hello world
-        console.log("signing response",  r)
-    }
-}
-function testClique(){
-    if( accts && accts.length > 0){
-        var a = accts[0]
-        var r = debug.testSignCliqueBlock(a, 0); // Sign genesis
-        console.log("signing response",  r)
-        if( a != r){
-            throw new Error("Requested signing by "+a+ " but got sealer "+r)
-        }
-    }
-}
-
-function test(){
-    var tests = [
-        testTx,
-        testSignText,
-        testClique,
-    ]
-    for( i in tests){
-        try{
-            tests[i]()
-        }catch(err){
-            console.log(err)
-        }
-    }
- }

cmd/clef/tutorial.md

@@ -1,353 +0,0 @@
-## Initializing Clef
-
-First things first, Clef needs to store some data itself. Since that data might be sensitive (passwords, signing rules, accounts), Clef's entire storage is encrypted. To support encrypting data, the first step is to initialize Clef with a random master seed, itself too encrypted with your chosen password:
-
-```text
-$ clef init
-
-WARNING!
-
-Clef is an account management tool. It may, like any software, contain bugs.
-
-Please take care to
-- backup your keystore files,
-- verify that the keystore(s) can be opened with your password.
-
-Clef is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
-without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
-PURPOSE. See the GNU General Public License for more details.
-
-Enter 'ok' to proceed:
-> ok
-
-The master seed of clef will be locked with a password.
-Please specify a password. Do not forget this password!
-Password:
-Repeat password:
-
-A master seed has been generated into /home/martin/.clef/masterseed.json
-
-This is required to be able to store credentials, such as:
-* Passwords for keystores (used by rule engine)
-* Storage for JavaScript auto-signing rules
-* Hash of JavaScript rule-file
-
-You should treat 'masterseed.json' with utmost secrecy and make a backup of it!
-* The password is necessary but not enough, you need to back up the master seed too!
-* The master seed does not contain your accounts, those need to be backed up separately!
-```
-
-*For readability purposes, we'll remove the WARNING printout, user confirmation and the unlocking of the master seed in the rest of this document.*
-
-## Remote interactions
-
-Clef is capable of managing both key-file based accounts as well as hardware wallets. To evaluate clef, we're going to point it to our Rinkeby testnet keystore and specify the Rinkeby chain ID for signing (Clef doesn't have a backing chain, so it doesn't know what network it runs on).
-
-```text
-$ clef --keystore ~/.ethereum/rinkeby/keystore --chainid 4
-
-INFO [07-01|11:00:46.385] Starting signer                          chainid=4 keystore=$HOME/.ethereum/rinkeby/keystore light-kdf=false advanced=false
-DEBUG[07-01|11:00:46.389] FS scan times                            list=3.521941ms set=9.017µs diff=4.112µs
-DEBUG[07-01|11:00:46.391] Ledger support enabled
-DEBUG[07-01|11:00:46.391] Trezor support enabled via HID
-DEBUG[07-01|11:00:46.391] Trezor support enabled via WebUSB
-INFO [07-01|11:00:46.391] Audit logs configured                    file=audit.log
-DEBUG[07-01|11:00:46.392] IPC registered                           namespace=account
-INFO [07-01|11:00:46.392] IPC endpoint opened                      url=$HOME/.clef/clef.ipc
-------- Signer info -------
-* intapi_version : 7.0.0
-* extapi_version : 6.0.0
-* extapi_http : n/a
-* extapi_ipc : $HOME/.clef/clef.ipc
-```
-
-By default, Clef starts up in CLI (Command Line Interface) mode. Arbitrary remote processes may *request* account interactions (e.g. sign a transaction), which the user will need to individually *confirm*.
-
-To test this out, we can *request* Clef to list all account via its *External API endpoint*:
-
-```text
-echo '{"id": 1, "jsonrpc": "2.0", "method": "account_list"}' | nc -U ~/.clef/clef.ipc
-```
-
-This will prompt the user within the Clef CLI to confirm or deny the request:
-
-```text
--------- List Account request--------------
-A request has been made to list all accounts.
-You can select which accounts the caller can see
-  [x] 0xD9C9Cd5f6779558b6e0eD4e6Acf6b1947E7fA1F3
-    URL: keystore://$HOME/.ethereum/rinkeby/keystore/UTC--2017-04-14T15-15-00.327614556Z--d9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3
-  [x] 0x086278A6C067775F71d6B2BB1856Db6E28c30418
-    URL: keystore://$HOME/.ethereum/rinkeby/keystore/UTC--2018-02-06T22-53-11.211657239Z--086278a6c067775f71d6b2bb1856db6e28c30418
--------------------------------------------
-Request context:
-	NA -> NA -> NA
-
-Additional HTTP header data, provided by the external caller:
-	User-Agent:
-	Origin:
-Approve? [y/N]:
->
-```
-
-Depending on whether we approve or deny the request, the original NetCat process will get:
-
-```text
-{"jsonrpc":"2.0","id":1,"result":["0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3","0x086278a6c067775f71d6b2bb1856db6e28c30418"]}
-
-or
-
-{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"Request denied"}}
-```
-
-Apart from listing accounts, you can also *request* creating a new account; signing transactions and data; and recovering signatures. You can find the available methods in the Clef [External API Spec](https://github.com/ethereum/go-ethereum/tree/master/cmd/clef#external-api-1) and the [External API Changelog](https://github.com/ethereum/go-ethereum/blob/master/cmd/clef/extapi_changelog.md).
-
-*Note, the number of things you can do from the External API is deliberately small, since we want to limit the power of remote calls by as much as possible! Clef has an [Internal API](https://github.com/ethereum/go-ethereum/tree/master/cmd/clef#ui-api-1) too for the UI (User Interface) which is much richer and can support custom interfaces on top. But that's out of scope here.*
-
-## Automatic rules
-
-For most users, manually confirming every transaction is the way to go. However, there are cases when it makes sense to set up some rules which permit Clef to sign a transaction without prompting the user. One such example would be running a signer on Rinkeby or other PoA networks.
-
-For starters, we can create a rule file that automatically permits anyone to list our available accounts without user confirmation. The rule file is a tiny JavaScript snippet that you can program however you want:
-
-```js
-function ApproveListing() {
-    return "Approve"
-}
-```
-
-Of course, Clef isn't going to just accept and run arbitrary scripts you give it, that would be dangerous if someone changes your rule file! Instead, you need to explicitly *attest* the rule file, which entails injecting its hash into Clef's secure store.
-
-```text
-$ sha256sum rules.js
-645b58e4f945e24d0221714ff29f6aa8e860382ced43490529db1695f5fcc71c  rules.js
-
-$ clef attest 645b58e4f945e24d0221714ff29f6aa8e860382ced43490529db1695f5fcc71c
-Decrypt master seed of clef
-Password:
-INFO [07-01|13:25:03.290] Ruleset attestation updated              sha256=645b58e4f945e24d0221714ff29f6aa8e860382ced43490529db1695f5fcc71c
-```
-
-At this point, we can start Clef with the rule file:
-
-```text
-$ clef --keystore ~/.ethereum/rinkeby/keystore --chainid 4 --rules rules.js
-
-INFO [07-01|13:39:49.726] Rule engine configured                   file=rules.js
-INFO [07-01|13:39:49.726] Starting signer                          chainid=4 keystore=$HOME/.ethereum/rinkeby/keystore light-kdf=false advanced=false
-DEBUG[07-01|13:39:49.726] FS scan times                            list=35.15µs set=4.251µs diff=2.766µs
-DEBUG[07-01|13:39:49.727] Ledger support enabled
-DEBUG[07-01|13:39:49.727] Trezor support enabled via HID
-DEBUG[07-01|13:39:49.727] Trezor support enabled via WebUSB
-INFO [07-01|13:39:49.728] Audit logs configured                    file=audit.log
-DEBUG[07-01|13:39:49.728] IPC registered                           namespace=account
-INFO [07-01|13:39:49.728] IPC endpoint opened                      url=$HOME/.clef/clef.ipc
-------- Signer info -------
-* intapi_version : 7.0.0
-* extapi_version : 6.0.0
-* extapi_http : n/a
-* extapi_ipc : $HOME/.clef/clef.ipc
-```
-
-Any account listing *request* will now be auto-approved by the rule file:
-
-```text
-$ echo '{"id": 1, "jsonrpc": "2.0", "method": "account_list"}' | nc -U ~/.clef/clef.ipc
-{"jsonrpc":"2.0","id":1,"result":["0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3","0x086278a6c067775f71d6b2bb1856db6e28c30418"]}
-```
-
-## Under the hood
-
-While doing the operations above, these files have been created:
-
-```text
-$ ls -laR ~/.clef/
-
-$HOME/.clef/:
-total 24
-drwxr-x--x   3 user user  4096 Jul  1 13:45 .
-drwxr-xr-x 102 user user 12288 Jul  1 13:39 ..
-drwx------   2 user user  4096 Jul  1 13:25 02f90c0603f4f2f60188
--r--------   1 user user   868 Jun 28 13:55 masterseed.json
-
-$HOME/.clef/02f90c0603f4f2f60188:
-total 12
-drwx------ 2 user user 4096 Jul  1 13:25 .
-drwxr-x--x 3 user user 4096 Jul  1 13:45 ..
--rw------- 1 user user  159 Jul  1 13:25 config.json
-
-$ cat ~/.clef/02f90c0603f4f2f60188/config.json
-{"ruleset_sha256":{"iv":"SWWEtnl+R+I+wfG7","c":"I3fjmwmamxVcfGax7D0MdUOL29/rBWcs73WBILmYK0o1CrX7wSMc3y37KsmtlZUAjp0oItYq01Ow8VGUOzilG91tDHInB5YHNtm/YkufEbo="}}
-```
-
-In `$HOME/.clef`, the `masterseed.json` file was created, containing the master seed. This seed was then used to derive a few other things:
-
-- **Vault location**: in this case `02f90c0603f4f2f60188`.
-   - If you use a different master seed, a different vault location will be used that does not conflict with each other (e.g. `clef --signersecret /path/to/file`). This allows you to run multiple instances of Clef, each with its own rules (e.g. mainnet + testnet).
-- **`config.json`**: the encrypted key/value storage for configuration data, currently only containing the key `ruleset_sha256`, the attested hash of the automatic rules to use.
-
-## Advanced rules
-
-In order to make more useful rules - like signing transactions - the signer needs access to the passwords needed to unlock keys from the keystore. You can inject an unlock password via `clef setpw`.
-
-```text
-$ clef setpw 0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3
-
-Please enter a password to store for this address:
-Password:
-Repeat password:
-
-Decrypt master seed of clef
-Password:
-INFO [07-01|14:05:56.031] Credential store updated                 key=0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3
-```
-
-Now let's update the rules to make use of the new credentials:
-
-```js
-function ApproveListing() {
-    return "Approve"
-}
-
-function ApproveSignData(req) {
-    if (req.address.toLowerCase() == "0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3") {
-        if (req.messages[0].value.indexOf("bazonk") >= 0) {
-            return "Approve"
-        }
-        return "Reject"
-    }
-    // Otherwise goes to manual processing
-}
-```
-
-In this example:
-
-- Any requests to sign data with the account `0xd9c9...` will be:
-    - Auto-approved if the message contains `bazonk`,
-    - Auto-rejected if the message does not contain `bazonk`,
-- Any other requests will be passed along for manual confirmation.
-
-*Note, to make this example work, please use you own accounts. You can create a new account either via Clef or the traditional account CLI tools. If the latter was chosen, make sure both Clef and Geth use the same keystore by specifying `--keystore path/to/your/keystore` when running Clef.*
-
-Attest the new rule file so that Clef will accept loading it:
-
-```text
-$ sha256sum rules.js
-f163a1738b649259bb9b369c593fdc4c6b6f86cc87e343c3ba58faee03c2a178  rules.js
-
-$ clef attest f163a1738b649259bb9b369c593fdc4c6b6f86cc87e343c3ba58faee03c2a178
-Decrypt master seed of clef
-Password:
-INFO [07-01|14:11:28.509] Ruleset attestation updated              sha256=f163a1738b649259bb9b369c593fdc4c6b6f86cc87e343c3ba58faee03c2a178
-```
-
-Restart Clef with the new rules in place:
-
-```
-$ clef --keystore ~/.ethereum/rinkeby/keystore --chainid 4 --rules rules.js
-
-INFO [07-01|14:12:41.636] Rule engine configured                   file=rules.js
-INFO [07-01|14:12:41.636] Starting signer                          chainid=4 keystore=$HOME/.ethereum/rinkeby/keystore light-kdf=false advanced=false
-DEBUG[07-01|14:12:41.636] FS scan times                            list=46.722µs set=4.47µs diff=2.157µs
-DEBUG[07-01|14:12:41.637] Ledger support enabled
-DEBUG[07-01|14:12:41.637] Trezor support enabled via HID
-DEBUG[07-01|14:12:41.638] Trezor support enabled via WebUSB
-INFO [07-01|14:12:41.638] Audit logs configured                    file=audit.log
-DEBUG[07-01|14:12:41.638] IPC registered                           namespace=account
-INFO [07-01|14:12:41.638] IPC endpoint opened                      url=$HOME/.clef/clef.ipc
-------- Signer info -------
-* intapi_version : 7.0.0
-* extapi_version : 6.0.0
-* extapi_http : n/a
-* extapi_ipc : $HOME/.clef/clef.ipc
-```
-
-Then test signing, once with `bazonk` and once without:
-
-```
-$ echo '{"id": 1, "jsonrpc":"2.0", "method":"account_signData", "params":["data/plain", "0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3", "0x202062617a6f6e6b2062617a2067617a0a"]}' | nc -U ~/.clef/clef.ipc
-{"jsonrpc":"2.0","id":1,"result":"0x4f93e3457027f6be99b06b3392d0ebc60615ba448bb7544687ef1248dea4f5317f789002df783979c417d969836b6fda3710f5bffb296b4d51c8aaae6e2ac4831c"}
-
-$ echo '{"id": 1, "jsonrpc":"2.0", "method":"account_signData", "params":["data/plain", "0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3", "0x2020626f6e6b2062617a2067617a0a"]}' | nc -U ~/.clef/clef.ipc
-{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"Request denied"}}
-```
-
-Meanwhile, in the Clef output log you can see:
-```text
-INFO [02-21|14:42:41] Op approved
-INFO [02-21|14:42:56] Op rejected
-```
-
-The signer also stores all traffic over the external API in a log file. The last 4 lines shows the two requests and their responses:
-
-```text
-$ tail -n 4 audit.log
-t=2019-07-01T15:52:14+0300 lvl=info msg=SignData   api=signer type=request  metadata="{\"remote\":\"NA\",\"local\":\"NA\",\"scheme\":\"NA\",\"User-Agent\":\"\",\"Origin\":\"\"}" addr="0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3 [chksum INVALID]" data=0x202062617a6f6e6b2062617a2067617a0a content-type=data/plain
-t=2019-07-01T15:52:14+0300 lvl=info msg=SignData   api=signer type=response data=4f93e3457027f6be99b06b3392d0ebc60615ba448bb7544687ef1248dea4f5317f789002df783979c417d969836b6fda3710f5bffb296b4d51c8aaae6e2ac4831c error=nil
-t=2019-07-01T15:52:23+0300 lvl=info msg=SignData   api=signer type=request  metadata="{\"remote\":\"NA\",\"local\":\"NA\",\"scheme\":\"NA\",\"User-Agent\":\"\",\"Origin\":\"\"}" addr="0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3 [chksum INVALID]" data=0x2020626f6e6b2062617a2067617a0a     content-type=data/plain
-t=2019-07-01T15:52:23+0300 lvl=info msg=SignData   api=signer type=response data=                                     error="Request denied"
-```
-
-For more details on writing automatic rules, please see the [rules spec](https://github.com/ethereum/go-ethereum/blob/master/cmd/clef/rules.md).
-
-## Geth integration
-
-Of course, as awesome as Clef is, it's not feasible to interact with it via JSON RPC by hand. Long term, we're hoping to convince the general Ethereum community to support Clef as a general signer (it's only 3-5 methods), thus allowing your favorite DApp, Metamask, MyCrypto, etc to request signatures directly.
-
-Until then however, we're trying to pave the way via Geth. Geth v1.9.0 has built in support via `--signer <API endpoint>` for using a local or remote Clef instance as an account backend!
-
-We can try this by running Clef with our previous rules on Rinkeby (for now it's a good idea to allow auto-listing accounts, since Geth likes to retrieve them once in a while).
-
-```text
-$ clef --keystore ~/.ethereum/rinkeby/keystore --chainid 4 --rules rules.js
-```
-
-In a different window we can start Geth, list our accounts, even list our wallets to see where the accounts originate from:
-
-```text
-$ geth --rinkeby --signer=~/.clef/clef.ipc console
-
-> eth.accounts
-["0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3", "0x086278a6c067775f71d6b2bb1856db6e28c30418"]
-
-> personal.listWallets
-[{
-    accounts: [{
-        address: "0xd9c9cd5f6779558b6e0ed4e6acf6b1947e7fa1f3",
-        url: "extapi://$HOME/.clef/clef.ipc"
-    }, {
-        address: "0x086278a6c067775f71d6b2bb1856db6e28c30418",
-        url: "extapi://$HOME/.clef/clef.ipc"
-    }],
-    status: "ok [version=6.0.0]",
-    url: "extapi://$HOME/.clef/clef.ipc"
-}]
-
-> eth.sendTransaction({from: eth.accounts[0], to: eth.accounts[0]})
-```
-
-Lastly, when we requested a transaction to be sent, Clef prompted us in the original window to approve it:
-
-```text
---------- Transaction request-------------
-to:       0xD9C9Cd5f6779558b6e0eD4e6Acf6b1947E7fA1F3
-from:     0xD9C9Cd5f6779558b6e0eD4e6Acf6b1947E7fA1F3 [chksum ok]
-value:    0 wei
-gas:      0x5208 (21000)
-gasprice: 1000000000 wei
-nonce:    0x2366 (9062)
-
-Request context:
-	NA -> NA -> NA
-
-Additional HTTP header data, provided by the external caller:
-	User-Agent:
-	Origin:
--------------------------------------------
-Approve? [y/N]:
-> y
-```
-
-:boom:
-
-*Note, if you enable the external signer backend in Geth, all other account management is disabled. This is because long term we want to remove account management from Geth.*

cmd/era/main.go

@@ -183,11 +183,11 @@ func open(ctx *cli.Context, epoch uint64) (era.Era, error) {
 	return openByPath(path)
 }
 
-// openByPath tries to open a single file as either eraE or era1 based on extension,
+// openByPath tries to open a single file as either Ere or Era1 based on extension,
 // falling back to the other reader if needed.
 func openByPath(path string) (era.Era, error) {
 	switch strings.ToLower(filepath.Ext(path)) {
-	case ".erae":
+	case ".ere":
 		if e, err := execdb.Open(path); err != nil {
 			return nil, err
 		} else {
@@ -229,7 +229,7 @@ func verify(ctx *cli.Context) error {
 
 	// Build the verification list respecting the rule:
 	// era1: must have accumulator, always verify
-	// erae: verify only if accumulator exists (pre-merge)
+	// ere:  verify only if accumulator exists (pre-merge / transition)
 
 	// Build list of files to verify.
 	verify := make([]string, 0, len(entries))
@@ -251,15 +251,15 @@ func verify(ctx *cli.Context) error {
 			}
 			verify = append(verify, path)
 
-		case ".erae":
+		case ".ere":
 			e, err := execdb.Open(path)
 			if err != nil {
-				return fmt.Errorf("error opening erae file %s: %w", name, err)
+				return fmt.Errorf("error opening ere file %s: %w", name, err)
 			}
 			_, accErr := e.Accumulator()
 			e.Close()
 			if accErr == nil {
-				verify = append(verify, path) // pre-merge only
+				verify = append(verify, path) // pre-merge / transition only
 			}
 		default:
 			return fmt.Errorf("unsupported era file: %s", name)

cmd/evm/internal/t8ntool/block.go

@@ -56,6 +56,8 @@ type header struct {
 	BlobGasUsed           *uint64           `json:"blobGasUsed"   rlp:"optional"`
 	ExcessBlobGas         *uint64           `json:"excessBlobGas"   rlp:"optional"`
 	ParentBeaconBlockRoot *common.Hash      `json:"parentBeaconBlockRoot" rlp:"optional"`
+	RequestsHash          *common.Hash      `json:"requestsHash" rlp:"optional"`
+	BlockAccessListHash   *common.Hash      `json:"blockAccessListHash" rlp:"optional"`
 	SlotNumber            *uint64           `json:"slotNumber" rlp:"optional"`
 }
 
@@ -119,26 +121,28 @@ func (c *cliqueInput) UnmarshalJSON(input []byte) error {
 // ToBlock converts i into a *types.Block
 func (i *bbInput) ToBlock() *types.Block {
 	header := &types.Header{
-		ParentHash:       i.Header.ParentHash,
-		UncleHash:        types.EmptyUncleHash,
-		Coinbase:         common.Address{},
-		Root:             i.Header.Root,
-		TxHash:           types.EmptyTxsHash,
-		ReceiptHash:      types.EmptyReceiptsHash,
-		Bloom:            i.Header.Bloom,
-		Difficulty:       common.Big0,
-		Number:           i.Header.Number,
-		GasLimit:         i.Header.GasLimit,
-		GasUsed:          i.Header.GasUsed,
-		Time:             i.Header.Time,
-		Extra:            i.Header.Extra,
-		MixDigest:        i.Header.MixDigest,
-		BaseFee:          i.Header.BaseFee,
-		WithdrawalsHash:  i.Header.WithdrawalsHash,
-		BlobGasUsed:      i.Header.BlobGasUsed,
-		ExcessBlobGas:    i.Header.ExcessBlobGas,
-		ParentBeaconRoot: i.Header.ParentBeaconBlockRoot,
-		SlotNumber:       i.Header.SlotNumber,
+		ParentHash:          i.Header.ParentHash,
+		UncleHash:           types.EmptyUncleHash,
+		Coinbase:            common.Address{},
+		Root:                i.Header.Root,
+		TxHash:              types.EmptyTxsHash,
+		ReceiptHash:         types.EmptyReceiptsHash,
+		Bloom:               i.Header.Bloom,
+		Difficulty:          common.Big0,
+		Number:              i.Header.Number,
+		GasLimit:            i.Header.GasLimit,
+		GasUsed:             i.Header.GasUsed,
+		Time:                i.Header.Time,
+		Extra:               i.Header.Extra,
+		MixDigest:           i.Header.MixDigest,
+		BaseFee:             i.Header.BaseFee,
+		WithdrawalsHash:     i.Header.WithdrawalsHash,
+		BlobGasUsed:         i.Header.BlobGasUsed,
+		ExcessBlobGas:       i.Header.ExcessBlobGas,
+		ParentBeaconRoot:    i.Header.ParentBeaconBlockRoot,
+		RequestsHash:        i.Header.RequestsHash,
+		BlockAccessListHash: i.Header.BlockAccessListHash,
+		SlotNumber:          i.Header.SlotNumber,
 	}
 
 	// Fill optional values.

cmd/evm/internal/t8ntool/execution.go

@@ -35,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/tracing"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/types/bal"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto/keccak"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -75,6 +76,9 @@ type ExecutionResult struct {
 	CurrentBlobGasUsed   *math.HexOrDecimal64  `json:"blobGasUsed,omitempty"`
 	RequestsHash         *common.Hash          `json:"requestsHash,omitempty"`
 	Requests             [][]byte              `json:"requests"`
+
+	BlockAccessList     hexutil.Bytes `json:"blockAccessList,omitempty"`
+	BlockAccessListHash *common.Hash  `json:"blockAccessListHash,omitempty"`
 }
 
 type executionResultMarshaling struct {
@@ -152,8 +156,10 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		return h
 	}
 	var (
-		isEIP4762 = chainConfig.IsUBT(big.NewInt(int64(pre.Env.Number)), pre.Env.Timestamp)
-		statedb   *state.StateDB
+		statedb *state.StateDB
+
+		isEIP4762   = chainConfig.IsUBT(big.NewInt(int64(pre.Env.Number)), pre.Env.Timestamp)
+		isAmsterdam = chainConfig.IsAmsterdam(big.NewInt(int64(pre.Env.Number)), pre.Env.Timestamp)
 	)
 	if pre.AllocPath != "" {
 		var err error
@@ -172,6 +178,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		includedTxs types.Transactions
 		blobGasUsed = uint64(0)
 		receipts    = make(types.Receipts, 0)
+
+		// TODO return blockAccessList as a part of result
+		blockAccessList = bal.NewConstructionBlockAccessList()
 	)
 	vmContext := vm.BlockContext{
 		CanTransfer: core.CanTransfer,
@@ -183,6 +192,9 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		GasLimit:    pre.Env.GasLimit,
 		GetHash:     getHash,
 	}
+	if pre.Env.SlotNumber != nil {
+		vmContext.SlotNum = *pre.Env.SlotNumber
+	}
 	// If currentBaseFee is defined, add it to the vmContext.
 	if pre.Env.BaseFee != nil {
 		vmContext.BaseFee = new(big.Int).Set(pre.Env.BaseFee)
@@ -231,14 +243,14 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 	}
 	evm := vm.NewEVM(vmContext, statedb, chainConfig, vmConfig)
 	if beaconRoot := pre.Env.ParentBeaconBlockRoot; beaconRoot != nil {
-		core.ProcessBeaconBlockRoot(*beaconRoot, evm)
+		core.ProcessBeaconBlockRoot(*beaconRoot, evm, blockAccessList)
 	}
 	if pre.Env.BlockHashes != nil && chainConfig.IsPrague(new(big.Int).SetUint64(pre.Env.Number), pre.Env.Timestamp) {
 		var (
 			prevNumber = pre.Env.Number - 1
 			prevHash   = pre.Env.BlockHashes[math.HexOrDecimal64(prevNumber)]
 		)
-		core.ProcessParentBlockHash(prevHash, evm)
+		core.ProcessParentBlockHash(prevHash, evm, blockAccessList)
 	}
 	for i := 0; txIt.Next(); i++ {
 		tx, err := txIt.Tx()
@@ -271,11 +283,12 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 			}
 		}
 		statedb.SetTxContext(tx.Hash(), len(receipts), uint32(len(receipts)+1))
+
 		var (
 			snapshot = statedb.Snapshot()
 			gp       = gaspool.Snapshot()
 		)
-		receipt, err := core.ApplyTransactionWithEVM(msg, gaspool, statedb, vmContext.BlockNumber, blockHash, pre.Env.Timestamp, tx, evm)
+		receipt, bal, err := core.ApplyTransactionWithEVM(msg, gaspool, statedb, vmContext.BlockNumber, blockHash, pre.Env.Timestamp, tx, evm)
 		if err != nil {
 			statedb.RevertToSnapshot(snapshot)
 			log.Info("rejected tx", "index", i, "hash", tx.Hash(), "from", msg.From, "error", err)
@@ -292,10 +305,11 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		}
 		blobGasUsed += txBlobGas
 		receipts = append(receipts, receipt)
+		blockAccessList.Merge(bal)
 	}
-
 	statedb.IntermediateRoot(chainConfig.IsEIP158(vmContext.BlockNumber))
 
+	// TODO(rjl493456442) call engine.Finalize() instead
 	// Add mining reward? (-1 means rewards are disabled)
 	if miningReward >= 0 {
 		// Add mining reward. The mining reward may be `0`, which only makes a difference in the cases
@@ -324,11 +338,22 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 	for _, w := range pre.Env.Withdrawals {
 		// Amount is in gwei, turn into wei
 		amount := new(big.Int).Mul(new(big.Int).SetUint64(w.Amount), big.NewInt(params.GWei))
-		statedb.AddBalance(w.Address, uint256.MustFromBig(amount), tracing.BalanceIncreaseWithdrawal)
+		prev := statedb.AddBalance(w.Address, uint256.MustFromBig(amount), tracing.BalanceIncreaseWithdrawal)
 
 		if isEIP4762 {
 			statedb.AccessEvents().AddAccount(w.Address, true, stdmath.MaxUint64)
 		}
+		if isAmsterdam {
+			if w.Amount == 0 {
+				// Zero amount withdrawal, account is accessed potential
+				// without state changes.
+				blockAccessList.AccountRead(w.Address)
+			} else {
+				// Non-zero amount withdrawal, account is accessed with
+				// a balance change.
+				blockAccessList.BalanceChange(uint32(len(receipts)+1), w.Address, new(uint256.Int).Add(&prev, uint256.MustFromBig(amount)))
+			}
+		}
 	}
 
 	// Gather the execution-layer triggered requests.
@@ -336,10 +361,12 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 	for _, receipt := range receipts {
 		allLogs = append(allLogs, receipt.Logs...)
 	}
-	requests, err := core.PostExecution(context.Background(), chainConfig, vmContext.BlockNumber, vmContext.Time, allLogs, evm, uint32(len(receipts)+1))
+	requests, bal, err := core.PostExecution(context.Background(), chainConfig, vmContext.BlockNumber, vmContext.Time, allLogs, evm, uint32(len(receipts)+1))
 	if err != nil {
 		return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("failed to process post-execution: %v", err))
 	}
+	blockAccessList.Merge(bal)
+
 	// Commit block
 	root, err := statedb.Commit(vmContext.BlockNumber.Uint64(), chainConfig.IsEIP158(vmContext.BlockNumber), chainConfig.IsCancun(vmContext.BlockNumber, vmContext.Time))
 	if err != nil {
@@ -371,6 +398,16 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 		execRs.RequestsHash = &h
 		execRs.Requests = requests
 	}
+	if isAmsterdam {
+		encoded := blockAccessList.ToEncodingObj()
+		balRLP, err := rlp.EncodeToBytes(encoded)
+		if err != nil {
+			return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not encode BAL: %v", err))
+		}
+		balHash := encoded.Hash()
+		execRs.BlockAccessListHash = &balHash
+		execRs.BlockAccessList = balRLP
+	}
 
 	// Re-create statedb instance with new root for MPT mode
 	statedb, err = state.New(root, statedb.Database())
@@ -381,9 +418,24 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig,
 	return statedb, execRs, body, nil
 }
 
+// newPrestateTrieDBConfig returns the triedb config used to construct the
+// prestate. UBT mode requires the path-based backend; the legacy hash-based
+// backend cannot decode UBT-encoded nodes.
+func newPrestateTrieDBConfig(isBintrie bool) *triedb.Config {
+	if isBintrie {
+		cfg := *triedb.UBTDefaults
+		cfg.Preimages = true
+		return &cfg
+	}
+	return &triedb.Config{Preimages: true}
+}
+
 func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, isBintrie bool) *state.StateDB {
-	tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true, IsUBT: isBintrie})
+	tdb := triedb.NewDatabase(db, newPrestateTrieDBConfig(isBintrie))
 	sdb := state.NewDatabase(tdb, nil)
+	if isBintrie {
+		sdb.(*state.UBTDatabase).EnableAllocRecording()
+	}
 
 	root := types.EmptyRootHash
 	if isBintrie {
@@ -421,8 +473,11 @@ func MakePreState(db ethdb.Database, accounts types.GenesisAlloc, isBintrie bool
 // MakePreStateStreaming is like MakePreState, but decodes the alloc from disk
 // one account at a time so the full map is never held in memory.
 func MakePreStateStreaming(db ethdb.Database, allocPath string, isBintrie bool) (*state.StateDB, error) {
-	tdb := triedb.NewDatabase(db, &triedb.Config{Preimages: true, IsUBT: isBintrie})
+	tdb := triedb.NewDatabase(db, newPrestateTrieDBConfig(isBintrie))
 	sdb := state.NewDatabase(tdb, nil)
+	if isBintrie {
+		sdb.(*state.UBTDatabase).EnableAllocRecording()
+	}
 
 	root := types.EmptyRootHash
 	if isBintrie {

cmd/evm/internal/t8ntool/gen_execresult.go

@@ -32,6 +32,8 @@ func (e ExecutionResult) MarshalJSON() ([]byte, error) {
 		CurrentBlobGasUsed   *math.HexOrDecimal64  `json:"blobGasUsed,omitempty"`
 		RequestsHash         *common.Hash          `json:"requestsHash,omitempty"`
 		Requests             []hexutil.Bytes       `json:"requests"`
+		BlockAccessList      hexutil.Bytes         `json:"blockAccessList,omitempty"`
+		BlockAccessListHash  *common.Hash          `json:"blockAccessListHash,omitempty"`
 	}
 	var enc ExecutionResult
 	enc.StateRoot = e.StateRoot
@@ -54,6 +56,8 @@ func (e ExecutionResult) MarshalJSON() ([]byte, error) {
 			enc.Requests[k] = v
 		}
 	}
+	enc.BlockAccessList = e.BlockAccessList
+	enc.BlockAccessListHash = e.BlockAccessListHash
 	return json.Marshal(&enc)
 }
 
@@ -75,6 +79,8 @@ func (e *ExecutionResult) UnmarshalJSON(input []byte) error {
 		CurrentBlobGasUsed   *math.HexOrDecimal64  `json:"blobGasUsed,omitempty"`
 		RequestsHash         *common.Hash          `json:"requestsHash,omitempty"`
 		Requests             []hexutil.Bytes       `json:"requests"`
+		BlockAccessList      *hexutil.Bytes        `json:"blockAccessList,omitempty"`
+		BlockAccessListHash  *common.Hash          `json:"blockAccessListHash,omitempty"`
 	}
 	var dec ExecutionResult
 	if err := json.Unmarshal(input, &dec); err != nil {
@@ -130,5 +136,11 @@ func (e *ExecutionResult) UnmarshalJSON(input []byte) error {
 			e.Requests[k] = v
 		}
 	}
+	if dec.BlockAccessList != nil {
+		e.BlockAccessList = *dec.BlockAccessList
+	}
+	if dec.BlockAccessListHash != nil {
+		e.BlockAccessListHash = dec.BlockAccessListHash
+	}
 	return nil
 }

cmd/evm/internal/t8ntool/gen_header.go

@@ -38,6 +38,8 @@ func (h header) MarshalJSON() ([]byte, error) {
 		BlobGasUsed           *math.HexOrDecimal64  `json:"blobGasUsed"   rlp:"optional"`
 		ExcessBlobGas         *math.HexOrDecimal64  `json:"excessBlobGas"   rlp:"optional"`
 		ParentBeaconBlockRoot *common.Hash          `json:"parentBeaconBlockRoot" rlp:"optional"`
+		RequestsHash          *common.Hash          `json:"requestsHash" rlp:"optional"`
+		BlockAccessListHash   *common.Hash          `json:"blockAccessListHash" rlp:"optional"`
 		SlotNumber            *math.HexOrDecimal64  `json:"slotNumber" rlp:"optional"`
 	}
 	var enc header
@@ -61,6 +63,8 @@ func (h header) MarshalJSON() ([]byte, error) {
 	enc.BlobGasUsed = (*math.HexOrDecimal64)(h.BlobGasUsed)
 	enc.ExcessBlobGas = (*math.HexOrDecimal64)(h.ExcessBlobGas)
 	enc.ParentBeaconBlockRoot = h.ParentBeaconBlockRoot
+	enc.RequestsHash = h.RequestsHash
+	enc.BlockAccessListHash = h.BlockAccessListHash
 	enc.SlotNumber = (*math.HexOrDecimal64)(h.SlotNumber)
 	return json.Marshal(&enc)
 }
@@ -88,6 +92,8 @@ func (h *header) UnmarshalJSON(input []byte) error {
 		BlobGasUsed           *math.HexOrDecimal64  `json:"blobGasUsed"   rlp:"optional"`
 		ExcessBlobGas         *math.HexOrDecimal64  `json:"excessBlobGas"   rlp:"optional"`
 		ParentBeaconBlockRoot *common.Hash          `json:"parentBeaconBlockRoot" rlp:"optional"`
+		RequestsHash          *common.Hash          `json:"requestsHash" rlp:"optional"`
+		BlockAccessListHash   *common.Hash          `json:"blockAccessListHash" rlp:"optional"`
 		SlotNumber            *math.HexOrDecimal64  `json:"slotNumber" rlp:"optional"`
 	}
 	var dec header
@@ -158,6 +164,12 @@ func (h *header) UnmarshalJSON(input []byte) error {
 	if dec.ParentBeaconBlockRoot != nil {
 		h.ParentBeaconBlockRoot = dec.ParentBeaconBlockRoot
 	}
+	if dec.RequestsHash != nil {
+		h.RequestsHash = dec.RequestsHash
+	}
+	if dec.BlockAccessListHash != nil {
+		h.BlockAccessListHash = dec.BlockAccessListHash
+	}
 	if dec.SlotNumber != nil {
 		h.SlotNumber = (*uint64)(dec.SlotNumber)
 	}

cmd/evm/internal/t8ntool/transition.go

@@ -30,6 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/overlay"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/tracing"
@@ -243,14 +244,55 @@ func Transition(ctx *cli.Context) error {
 		collector = make(Alloc)
 		s.DumpToCollector(collector, nil)
 	default:
-		btleaves = make(map[common.Hash]hexutil.Bytes)
-		if err := s.DumpBinTrieLeaves(btleaves); err != nil {
-			return err
+		udb, ok := s.Database().(*state.UBTDatabase)
+		if !ok {
+			return NewError(ErrorEVM, errors.New("expected UBTDatabase in binary trie mode"))
+		}
+		rec := udb.AllocRecorder()
+		if rec == nil {
+			return NewError(ErrorEVM, errors.New("UBT alloc recorder was not enabled"))
+		}
+		collector = Alloc(rec.Alloc())
+		if err := mergeUnmigratedBaseAlloc(udb, s.IntermediateRoot(false), collector); err != nil {
+			return NewError(ErrorEVM, fmt.Errorf("failed to merge base MPT alloc: %v", err))
 		}
 	}
 	return dispatchOutput(ctx, baseDir, result, collector, allocOutput, body, btleaves)
 }
 
+func mergeUnmigratedBaseAlloc(udb *state.UBTDatabase, currentRoot common.Hash, dst Alloc) error {
+	ts := overlay.LoadTransitionState(udb.TrieDB().Disk(), currentRoot, true)
+	if !ts.InTransition() {
+		return nil
+	}
+	if ts.BaseRoot == (common.Hash{}) || ts.BaseRoot == types.EmptyRootHash {
+		return nil
+	}
+	mptDB := state.NewMPTDatabase(udb.TrieDB(), nil)
+	sdb, err := state.New(ts.BaseRoot, mptDB)
+	if err != nil {
+		return fmt.Errorf("open base MPT at %x: %w", ts.BaseRoot, err)
+	}
+	if _, err := sdb.DumpToCollector(mergeAlloc(dst), nil); err != nil {
+		return fmt.Errorf("walk base MPT at %x: %w", ts.BaseRoot, err)
+	}
+	return nil
+}
+
+type mergeAlloc Alloc
+
+func (m mergeAlloc) OnRoot(common.Hash) {}
+
+func (m mergeAlloc) OnAccount(addr *common.Address, da state.DumpAccount) {
+	if addr == nil {
+		return
+	}
+	if _, exists := m[*addr]; exists {
+		return
+	}
+	m[*addr] = dumpAccountToTypesAccount(da)
+}
+
 // writeStreamedAlloc writes the post-state alloc to path one account at a
 // time, producing the same JSON shape as saveFile on an Alloc map.
 func writeStreamedAlloc(path string, s *state.StateDB) error {

cmd/geth/chaincmd.go

@@ -101,7 +101,6 @@ if one is set.  Otherwise it prints the genesis from the datadir.`,
 			utils.NoCompactionFlag,
 			utils.LogSlowBlockFlag,
 			utils.MetricsEnabledFlag,
-			utils.MetricsEnabledExpensiveFlag,
 			utils.MetricsHTTPFlag,
 			utils.MetricsPortFlag,
 			utils.MetricsEnableInfluxDBFlag,
@@ -116,7 +115,6 @@ if one is set.  Otherwise it prints the genesis from the datadir.`,
 			utils.MetricsInfluxDBBucketFlag,
 			utils.MetricsInfluxDBOrganizationFlag,
 			utils.StateSizeTrackingFlag,
-			utils.TxLookupLimitFlag,
 			utils.VMTraceFlag,
 			utils.VMTraceJsonConfigFlag,
 			utils.TransactionHistoryFlag,
@@ -157,7 +155,7 @@ be gzipped.`,
 		Name:      "import-history",
 		Usage:     "Import an Era archive",
 		ArgsUsage: "<dir>",
-		Flags:     slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag, utils.EraFormatFlag}, utils.DatabaseFlags, utils.NetworkFlags),
+		Flags:     slices.Concat([]cli.Flag{utils.TransactionHistoryFlag, utils.EraFormatFlag}, utils.DatabaseFlags, utils.NetworkFlags),
 		Description: `
 The import-history command will import blocks and their corresponding receipts
 from Era archives.
@@ -528,15 +526,15 @@ func importHistory(ctx *cli.Context) error {
 
 	var (
 		format = ctx.String(utils.EraFormatFlag.Name)
-		from   func(era.ReadAtSeekCloser) (era.Era, error)
+		from   func(f era.ReadAtSeekCloser) (era.Era, error)
 	)
 	switch format {
 	case "era1", "era":
 		from = onedb.From
-	case "erae":
+	case "ere":
 		from = execdb.From
 	default:
-		return fmt.Errorf("unknown --era.format %q (expected 'era1' or 'erae')", format)
+		return fmt.Errorf("unknown --era.format %q (expected 'era1' or 'ere')", format)
 	}
 	if err := utils.ImportHistory(chain, dir, network, from); err != nil {
 		return err
@@ -582,11 +580,11 @@ func exportHistory(ctx *cli.Context) error {
 	case "era1", "era":
 		newBuilder = func(w io.Writer) era.Builder { return onedb.NewBuilder(w) }
 		filename = func(network string, epoch int, root common.Hash) string { return onedb.Filename(network, epoch, root) }
-	case "erae":
+	case "ere":
 		newBuilder = func(w io.Writer) era.Builder { return execdb.NewBuilder(w) }
 		filename = func(network string, epoch int, root common.Hash) string { return execdb.Filename(network, epoch, root) }
 	default:
-		return fmt.Errorf("unknown archive format %q (use 'era1' or 'erae')", format)
+		return fmt.Errorf("unknown archive format %q (use 'era1' or 'ere')", format)
 	}
 	if err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), newBuilder, filename); err != nil {
 		utils.Fatalf("Export error: %v\n", err)
@@ -744,7 +742,7 @@ func pruneHistory(ctx *cli.Context) error {
 	)
 
 	// Check the current freezer tail to see if pruning is needed/possible.
-	freezerTail, _ := chaindb.Tail()
+	freezerTail, _ := chaindb.Tail(rawdb.ChainFreezerBlockDataGroup)
 	if freezerTail > 0 {
 		if freezerTail == targetBlock {
 			log.Info("Database already pruned to target block", "tail", freezerTail)
@@ -776,7 +774,7 @@ func pruneHistory(ctx *cli.Context) error {
 	log.Info("Starting history pruning", "head", currentHeader.Number, "target", targetBlock, "targetHash", targetBlockHash.Hex())
 	start := time.Now()
 	rawdb.PruneTransactionIndex(chaindb, targetBlock)
-	if _, err := chaindb.TruncateTail(targetBlock); err != nil {
+	if _, err := chaindb.TruncateTail(rawdb.ChainFreezerBlockDataGroup, targetBlock); err != nil {
 		return fmt.Errorf("failed to truncate ancient data: %v", err)
 	}
 	log.Info("History pruning completed", "tail", targetBlock, "elapsed", common.PrettyDuration(time.Since(start)))

cmd/geth/config.go

@@ -354,9 +354,6 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) {
 	if ctx.IsSet(utils.MetricsEnabledFlag.Name) {
 		cfg.Metrics.Enabled = ctx.Bool(utils.MetricsEnabledFlag.Name)
 	}
-	if ctx.IsSet(utils.MetricsEnabledExpensiveFlag.Name) {
-		log.Warn("Expensive metrics are collected by default, please remove this flag", "flag", utils.MetricsEnabledExpensiveFlag.Name)
-	}
 	if ctx.IsSet(utils.MetricsHTTPFlag.Name) {
 		cfg.Metrics.HTTP = ctx.String(utils.MetricsHTTPFlag.Name)
 	}

cmd/geth/consolecmd_test.go

@@ -51,10 +51,9 @@ func runMinimalGeth(t *testing.T, args ...string) *testgeth {
 // then terminated by closing the input stream.
 func TestConsoleWelcome(t *testing.T) {
 	t.Parallel()
-	coinbase := "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182"
 
 	// Start a geth console, make sure it's cleaned up and terminate the console
-	geth := runMinimalGeth(t, "--miner.etherbase", coinbase, "console")
+	geth := runMinimalGeth(t, "console")
 
 	// Gather all the infos the welcome message needs to contain
 	geth.SetTemplateFunc("goos", func() string { return runtime.GOOS })
@@ -98,7 +97,7 @@ func TestAttachWelcome(t *testing.T) {
 	p := trulyRandInt(1024, 65533) // Yeah, sometimes this will fail, sorry :P
 	httpPort = strconv.Itoa(p)
 	wsPort = strconv.Itoa(p + 1)
-	geth := runMinimalGeth(t, "--miner.etherbase", "0x8605cdbbdb6d264aa742e77020dcbc58fcdce182",
+	geth := runMinimalGeth(t,
 		"--ipcpath", ipc,
 		"--http", "--http.port", httpPort,
 		"--ws", "--ws.port", wsPort)

cmd/geth/main.go

@@ -49,13 +49,11 @@ var (
 	// flags that configure the node
 	nodeFlags = slices.Concat([]cli.Flag{
 		utils.IdentityFlag,
-		utils.UnlockedAccountFlag,
 		utils.PasswordFileFlag,
 		utils.BootnodesFlag,
 		utils.MinFreeDiskSpaceFlag,
 		utils.KeyStoreDirFlag,
 		utils.ExternalSignerFlag,
-		utils.NoUSBFlag, // deprecated
 		utils.USBFlag,
 		utils.SmartCardDaemonPathFlag,
 		utils.OverrideOsaka,
@@ -63,7 +61,6 @@ var (
 		utils.OverrideBPO2,
 		utils.OverrideUBT,
 		utils.OverrideGenesisFlag,
-		utils.EnablePersonal, // deprecated
 		utils.TxPoolLocalsFlag,
 		utils.TxPoolNoLocalsFlag,
 		utils.TxPoolJournalFlag,
@@ -83,7 +80,6 @@ var (
 		utils.ExitWhenSyncedFlag,
 		utils.GCModeFlag,
 		utils.SnapshotFlag,
-		utils.TxLookupLimitFlag, // deprecated
 		utils.TransactionHistoryFlag,
 		utils.ChainHistoryFlag,
 		utils.LogHistoryFlag,
@@ -95,12 +91,9 @@ var (
 		utils.BinTrieGroupDepthFlag,
 		utils.LightKDFFlag,
 		utils.EthRequiredBlocksFlag,
-		utils.LegacyWhitelistFlag, // deprecated
 		utils.CacheFlag,
 		utils.CacheDatabaseFlag,
 		utils.CacheTrieFlag,
-		utils.CacheTrieJournalFlag,   // deprecated
-		utils.CacheTrieRejournalFlag, // deprecated
 		utils.CacheGCFlag,
 		utils.CacheSnapshotFlag,
 		utils.CacheNoPrefetchFlag,
@@ -112,20 +105,16 @@ var (
 		utils.DiscoveryPortFlag,
 		utils.MaxPeersFlag,
 		utils.MaxPendingPeersFlag,
-		utils.MiningEnabledFlag, // deprecated
 		utils.MinerGasLimitFlag,
 		utils.MinerGasPriceFlag,
-		utils.MinerEtherbaseFlag, // deprecated
 		utils.MinerExtraDataFlag,
 		utils.MinerMaxBlobsFlag,
 		utils.MinerRecommitIntervalFlag,
 		utils.MinerPendingFeeRecipientFlag,
-		utils.MinerNewPayloadTimeoutFlag, // deprecated
 		utils.NATFlag,
 		utils.NoDiscoverFlag,
 		utils.DiscoveryV4Flag,
 		utils.DiscoveryV5Flag,
-		utils.LegacyDiscoveryV5Flag, // deprecated
 		utils.NetrestrictFlag,
 		utils.NodeKeyFileFlag,
 		utils.NodeKeyHexFlag,
@@ -145,8 +134,6 @@ var (
 		utils.GpoMaxGasPriceFlag,
 		utils.GpoIgnoreGasPriceFlag,
 		configFileFlag,
-		utils.LogDebugFlag,
-		utils.LogBacktraceAtFlag,
 		utils.BeaconApiFlag,
 		utils.BeaconApiHeaderFlag,
 		utils.BeaconThresholdFlag,
@@ -182,7 +169,6 @@ var (
 		utils.WSPathPrefixFlag,
 		utils.IPCDisabledFlag,
 		utils.IPCPathFlag,
-		utils.InsecureUnlockAllowedFlag,
 		utils.RPCGlobalGasCapFlag,
 		utils.RPCGlobalEVMTimeoutFlag,
 		utils.RPCGlobalTxFeeCapFlag,
@@ -204,7 +190,6 @@ var (
 
 	metricsFlags = []cli.Flag{
 		utils.MetricsEnabledFlag,
-		utils.MetricsEnabledExpensiveFlag,
 		utils.MetricsHTTPFlag,
 		utils.MetricsPortFlag,
 		utils.MetricsEnableInfluxDBFlag,
@@ -340,10 +325,6 @@ func startNode(ctx *cli.Context, stack *node.Node, isConsole bool) {
 	// Start up the node itself
 	utils.StartNode(ctx, stack, isConsole)
 
-	if ctx.IsSet(utils.UnlockedAccountFlag.Name) {
-		log.Warn(`The "unlock" flag has been deprecated and has no effect`)
-	}
-
 	// Register wallet event handlers to open and auto-derive wallets
 	events := make(chan accounts.WalletEvent, 16)
 	stack.AccountManager().Subscribe(events)

cmd/geth/run_test.go

@@ -32,8 +32,7 @@ type testgeth struct {
 	*cmdtest.TestCmd
 
 	// template variables for expect
-	Datadir   string
-	Etherbase string
+	Datadir string
 }
 
 func init() {
@@ -75,10 +74,6 @@ func runGeth(t *testing.T, args ...string) *testgeth {
 			if i < len(args)-1 {
 				tt.Datadir = args[i+1]
 			}
-		case "--miner.etherbase":
-			if i < len(args)-1 {
-				tt.Etherbase = args[i+1]
-			}
 		}
 	}
 	if tt.Datadir == "" {

cmd/geth/snapshot.go

@@ -22,11 +22,18 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/http"
+	_ "net/http/pprof"
 	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
 	"slices"
 	"sort"
+	"syscall"
 	"time"
 
+	pebbleimpl "github.com/cockroachdb/pebble"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
@@ -36,6 +43,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state/snapshot"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb/pebble"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
@@ -80,6 +88,33 @@ geth snapshot verify-state <state-root>
 will traverse the whole accounts and storages set based on the specified
 snapshot and recalculate the root hash of state for verification.
 In other words, this command does the snapshot to trie conversion.
+`,
+			},
+			{
+				Name:      "generate-trie",
+				Usage:     "Benchmark triedb.GenerateTrie against a hard-linked checkpoint of the chaindata",
+				ArgsUsage: "[<root>]",
+				Action:    benchGenerateTrie,
+				Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags, []cli.Flag{
+					&cli.StringFlag{
+						Name:  "checkpoint",
+						Usage: "Directory for the pebble checkpoint (default: <chaindata-parent>/.gentrie-bench-<ts>)",
+					},
+					&cli.BoolFlag{
+						Name:  "keep",
+						Usage: "Keep the checkpoint directory after the run (debugging)",
+					},
+					&cli.BoolFlag{
+						Name:  "pprof",
+						Usage: "Serve pprof profiles on localhost:6060 (block + mutex profiles enabled)",
+					},
+				}),
+				Description: `
+geth snapshot generate-trie [<root>]
+
+Runs triedb.GenerateTrie against a hard-linked pebble checkpoint of the
+chaindata. Checkpoint is removed on exit unless --keep is set. Defaults 
+to the snapshot root if <root> is not given.
 `,
 			},
 			{
@@ -289,6 +324,157 @@ func verifyState(ctx *cli.Context) error {
 	}
 }
 
+// benchGenerateTrie runs triedb.GenerateTrie against a hard-linked checkpoint
+// of the chaindata so the source datadir is never written to.
+func benchGenerateTrie(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	if ctx.Bool("pprof") {
+		runtime.SetBlockProfileRate(1)
+		runtime.SetMutexProfileFraction(1)
+		go func() {
+			log.Info("pprof listening", "addr", ":6060")
+			if err := http.ListenAndServe(":6060", nil); err != nil {
+				log.Warn("pprof server stopped", "err", err)
+			}
+		}()
+	}
+
+	// Resolve source chaindata path (handles network-specific subdirs).
+	srcDir := stack.ResolvePath("chaindata")
+	if fi, err := os.Stat(srcDir); err != nil {
+		return fmt.Errorf("chaindata not found at %s: %w", srcDir, err)
+	} else if !fi.IsDir() {
+		return fmt.Errorf("%s is not a directory", srcDir)
+	}
+
+	// Default to snapshot root, not head: that's what GenerateTrie actually
+	// reconstructs from flat state. On a fully-synced node they match.
+	var root common.Hash
+	if ctx.NArg() == 1 {
+		r, err := parseRoot(ctx.Args().First())
+		if err != nil {
+			return fmt.Errorf("parse root: %w", err)
+		}
+		root = r
+	} else {
+		chaindb := utils.MakeChainDatabase(ctx, stack, true)
+		snapRoot := rawdb.ReadSnapshotRoot(chaindb)
+		head := rawdb.ReadHeadBlock(chaindb)
+		chaindb.Close()
+		switch {
+		case snapRoot != (common.Hash{}):
+			root = snapRoot
+			log.Info("using snapshot root", "root", root)
+		case head != nil:
+			root = head.Root()
+			log.Info("using head block root", "number", head.Number(), "root", root)
+		default:
+			return errors.New("no snapshot or head block found; pass <root> explicitly")
+		}
+	}
+
+	// Default checkpoint sits next to chaindata so hard links work.
+	ckpt := ctx.String("checkpoint")
+	if ckpt == "" {
+		ts := time.Now().Format("20060102-150405")
+		ckpt = filepath.Join(filepath.Dir(srcDir), fmt.Sprintf(".gentrie-bench-%s", ts))
+	}
+	if _, err := os.Stat(ckpt); err == nil {
+		return fmt.Errorf("checkpoint dir %s already exists; remove it or pass --checkpoint to a fresh path", ckpt)
+	}
+
+	log.Info("creating pebble checkpoint", "src", srcDir, "dst", ckpt)
+	checkpointStart := time.Now()
+	if err := makeCheckpoint(srcDir, ckpt); err != nil {
+		return fmt.Errorf("checkpoint failed: %w", err)
+	}
+	log.Info("checkpoint created", "elapsed", time.Since(checkpointStart))
+
+	// Clean up the checkpoint on exit, including Ctrl-C.
+	keep := ctx.Bool("keep")
+	cleanup := func() {
+		if keep {
+			log.Info("keeping checkpoint", "path", ckpt)
+			return
+		}
+		log.Info("removing checkpoint", "path", ckpt)
+		if err := os.RemoveAll(ckpt); err != nil {
+			log.Error("failed to remove checkpoint", "err", err)
+		}
+	}
+	defer cleanup()
+
+	cancelCh := make(chan struct{})
+	sigCh := make(chan os.Signal, 1)
+	signal.Notify(sigCh, os.Interrupt, syscall.SIGTERM)
+	defer signal.Stop(sigCh)
+	go func() {
+		<-sigCh
+		log.Warn("interrupt received; cancelling GenerateTrie")
+		close(cancelCh)
+	}()
+
+	// Open the checkpoint writable. Reuse source ancient. Checkpoint only
+	// hard-links the pebble SSTs (not the freezer), and GenerateTrie never
+	// writes to ancient, so sharing it is safe.
+	srcAncient := stack.ResolveAncient("chaindata", "")
+	kv, err := pebble.New(ckpt, 4096, 1024, "gentrie-bench", false)
+	if err != nil {
+		return fmt.Errorf("open checkpoint: %w", err)
+	}
+	chaindb, err := rawdb.Open(kv, rawdb.OpenOptions{
+		Ancient:          srcAncient,
+		MetricsNamespace: "gentrie-bench",
+	})
+	if err != nil {
+		kv.Close()
+		return fmt.Errorf("rawdb.Open checkpoint: %w", err)
+	}
+	defer chaindb.Close()
+
+	// Pick up the trie scheme already in use (path or hash).
+	triedbInst := utils.MakeTrieDatabase(ctx, stack, chaindb, false, true, false)
+	scheme := triedbInst.Scheme()
+	triedbInst.Close()
+
+	log.Info("running GenerateTrie", "scheme", scheme, "root", root)
+	runStart := time.Now()
+	stats, err := triedb.GenerateTrie(chaindb, scheme, root, cancelCh)
+	elapsed := time.Since(runStart)
+
+	status := "root matched"
+	if err != nil {
+		status = fmt.Sprintf("failed (%s)", err)
+		log.Error("GenerateTrie failed", "elapsed", elapsed, "err", err)
+	}
+
+	fmt.Printf("\n=== generate-trie benchmark ===\n")
+	fmt.Printf("scheme:    %s\n", scheme)
+	fmt.Printf("root:      %s\n", root.Hex())
+	fmt.Printf("status:    %s\n", status)
+	fmt.Printf("accounts:  %d (%d updated)\n", stats.Scanned, stats.Updated)
+	fmt.Printf("wall time: %s\n", elapsed)
+	return err
+}
+
+// makeCheckpoint opens srcDir as a pebble database and writes a hard-linked
+// checkpoint to dstDir. Source is closed on return.
+//
+// Opens read-write so pebble can finalize its startup (WAL replay, fresh
+// OPTIONS file) before checkpointing. Read-only mode skips that step, and
+// Checkpoint then fails trying to hard-link the missing OPTIONS file. The
+// read-write open does no more than a normal geth startup would.
+func makeCheckpoint(srcDir, dstDir string) error {
+	db, err := pebbleimpl.Open(srcDir, &pebbleimpl.Options{})
+	if err != nil {
+		return fmt.Errorf("open source pebble: %w", err)
+	}
+	defer db.Close()
+	return db.Checkpoint(dstDir)
+}
+
 // checkDanglingStorage iterates the snap storage data, and verifies that all
 // storage also has corresponding account data.
 func checkDanglingStorage(ctx *cli.Context) error {

cmd/keeper/go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/golang/snappy v1.0.0 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
 	github.com/holiman/uint256 v1.3.2 // indirect
+	github.com/klauspost/compress v1.17.8 // indirect
 	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
 	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.1 // indirect
@@ -35,12 +36,12 @@ require (
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
-	go.opentelemetry.io/otel v1.40.0 // indirect
-	go.opentelemetry.io/otel/metric v1.40.0 // indirect
-	go.opentelemetry.io/otel/trace v1.40.0 // indirect
-	golang.org/x/crypto v0.47.0 // indirect
+	go.opentelemetry.io/otel v1.41.0 // indirect
+	go.opentelemetry.io/otel/metric v1.41.0 // indirect
+	go.opentelemetry.io/otel/trace v1.41.0 // indirect
+	golang.org/x/crypto v0.48.0 // indirect
 	golang.org/x/sync v0.19.0 // indirect
-	golang.org/x/sys v0.40.0 // indirect
+	golang.org/x/sys v0.41.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 

cmd/keeper/go.sum

@@ -119,16 +119,18 @@ github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+F
 github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
-go.opentelemetry.io/otel v1.40.0 h1:oA5YeOcpRTXq6NN7frwmwFR0Cn3RhTVZvXsP4duvCms=
-go.opentelemetry.io/otel v1.40.0/go.mod h1:IMb+uXZUKkMXdPddhwAHm6UfOwJyh4ct1ybIlV14J0g=
-go.opentelemetry.io/otel/metric v1.40.0 h1:rcZe317KPftE2rstWIBitCdVp89A2HqjkxR3c11+p9g=
-go.opentelemetry.io/otel/metric v1.40.0/go.mod h1:ib/crwQH7N3r5kfiBZQbwrTge743UDc7DTFVZrrXnqc=
-go.opentelemetry.io/otel/sdk v1.40.0 h1:KHW/jUzgo6wsPh9At46+h4upjtccTmuZCFAc9OJ71f8=
-go.opentelemetry.io/otel/sdk v1.40.0/go.mod h1:Ph7EFdYvxq72Y8Li9q8KebuYUr2KoeyHx0DRMKrYBUE=
-go.opentelemetry.io/otel/trace v1.40.0 h1:WA4etStDttCSYuhwvEa8OP8I5EWu24lkOzp+ZYblVjw=
-go.opentelemetry.io/otel/trace v1.40.0/go.mod h1:zeAhriXecNGP/s2SEG3+Y8X9ujcJOTqQ5RgdEJcawiA=
-golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
-golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
+go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c=
+go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE=
+go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ=
+go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps=
+go.opentelemetry.io/otel/sdk v1.41.0 h1:YPIEXKmiAwkGl3Gu1huk1aYWwtpRLeskpV+wPisxBp8=
+go.opentelemetry.io/otel/sdk v1.41.0/go.mod h1:ahFdU0G5y8IxglBf0QBJXgSe7agzjE4GiTJ6HT9ud90=
+go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0=
+go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
+golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df h1:UA2aFVmmsIlefxMk29Dp2juaUSth8Pyn3Tq5Y5mJGME=
 golang.org/x/exp v0.0.0-20230626212559-97b1e661b5df/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
@@ -137,10 +139,10 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
-golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
-golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
 google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

cmd/utils/cmd.go

@@ -242,11 +242,21 @@ func ImportChain(chain *core.BlockChain, fn string) error {
 }
 
 func readList(filename string) ([]string, error) {
-	b, err := os.ReadFile(filename)
+	f, err := os.Open(filename)
 	if err != nil {
 		return nil, err
 	}
-	return strings.Split(string(b), "\n"), nil
+	defer f.Close()
+
+	var lines []string
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		lines = append(lines, scanner.Text())
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	return lines, nil
 }
 
 // ImportHistory imports Era1 files containing historical block information,

cmd/utils/flags.go

@@ -1110,7 +1110,7 @@ Please note that --` + MetricsHTTPFlag.Name + ` must be set to start the server.
 	// Era flags are a group of flags related to the era archive format.
 	EraFormatFlag = &cli.StringFlag{
 		Name:  "era.format",
-		Usage: "Archive format: 'era1' or 'erae'",
+		Usage: "Archive format: 'era1' or 'ere'",
 	}
 )
 
@@ -1432,9 +1432,6 @@ func MakeDatabaseHandles(max int) int {
 
 // setEtherbase retrieves the etherbase from the directly specified command line flags.
 func setEtherbase(ctx *cli.Context, cfg *ethconfig.Config) {
-	if ctx.IsSet(MinerEtherbaseFlag.Name) {
-		log.Warn("Option --miner.etherbase is deprecated as the etherbase is set by the consensus client post-merge")
-	}
 	if !ctx.IsSet(MinerPendingFeeRecipientFlag.Name) {
 		return
 	}
@@ -1507,9 +1504,6 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	if ctx.IsSet(JWTSecretFlag.Name) {
 		cfg.JWTSecret = ctx.String(JWTSecretFlag.Name)
 	}
-	if ctx.IsSet(EnablePersonal.Name) {
-		log.Warn(fmt.Sprintf("Option --%s is deprecated. The 'personal' RPC namespace has been removed.", EnablePersonal.Name))
-	}
 
 	if ctx.IsSet(ExternalSignerFlag.Name) {
 		cfg.ExternalSigner = ctx.String(ExternalSignerFlag.Name)
@@ -1524,15 +1518,9 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 	if ctx.IsSet(LightKDFFlag.Name) {
 		cfg.UseLightweightKDF = ctx.Bool(LightKDFFlag.Name)
 	}
-	if ctx.IsSet(NoUSBFlag.Name) || cfg.NoUSB {
-		log.Warn("Option --nousb is deprecated and USB is deactivated by default. Use --usb to enable")
-	}
 	if ctx.IsSet(USBFlag.Name) {
 		cfg.USB = ctx.Bool(USBFlag.Name)
 	}
-	if ctx.IsSet(InsecureUnlockAllowedFlag.Name) {
-		log.Warn(fmt.Sprintf("Option --%s is deprecated and has no effect", InsecureUnlockAllowedFlag.Name))
-	}
 	if ctx.IsSet(DBEngineFlag.Name) {
 		dbEngine := ctx.String(DBEngineFlag.Name)
 		if dbEngine != "leveldb" && dbEngine != "pebble" {
@@ -1541,13 +1529,6 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) {
 		log.Info(fmt.Sprintf("Using %s as db engine", dbEngine))
 		cfg.DBEngine = dbEngine
 	}
-	// deprecation notice for log debug flags (TODO: find a more appropriate place to put these?)
-	if ctx.IsSet(LogBacktraceAtFlag.Name) {
-		log.Warn("Option --log.backtrace flag is deprecated")
-	}
-	if ctx.IsSet(LogDebugFlag.Name) {
-		log.Warn("Option --log.debug flag is deprecated")
-	}
 }
 
 func setSmartCard(ctx *cli.Context, cfg *node.Config) {
@@ -1685,9 +1666,6 @@ func setBlobPool(ctx *cli.Context, cfg *blobpool.Config) {
 }
 
 func setMiner(ctx *cli.Context, cfg *miner.Config) {
-	if ctx.Bool(MiningEnabledFlag.Name) {
-		log.Warn("The flag --mine is deprecated and will be removed")
-	}
 	if ctx.IsSet(MinerExtraDataFlag.Name) {
 		cfg.ExtraData = []byte(ctx.String(MinerExtraDataFlag.Name))
 	}
@@ -1700,10 +1678,6 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) {
 	if ctx.IsSet(MinerRecommitIntervalFlag.Name) {
 		cfg.Recommit = ctx.Duration(MinerRecommitIntervalFlag.Name)
 	}
-	if ctx.IsSet(MinerNewPayloadTimeoutFlag.Name) {
-		log.Warn("The flag --miner.newpayload-timeout is deprecated and will be removed, please use --miner.recommit")
-		cfg.Recommit = ctx.Duration(MinerNewPayloadTimeoutFlag.Name)
-	}
 	if ctx.IsSet(MinerMaxBlobsFlag.Name) {
 		cfg.MaxBlobsPerBlock = ctx.Int(MinerMaxBlobsFlag.Name)
 	}
@@ -1712,12 +1686,7 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) {
 func setRequiredBlocks(ctx *cli.Context, cfg *ethconfig.Config) {
 	requiredBlocks := ctx.String(EthRequiredBlocksFlag.Name)
 	if requiredBlocks == "" {
-		if ctx.IsSet(LegacyWhitelistFlag.Name) {
-			log.Warn("The flag --whitelist is deprecated and will be removed, please use --eth.requiredblocks")
-			requiredBlocks = ctx.String(LegacyWhitelistFlag.Name)
-		} else {
-			return
-		}
+		return
 	}
 	cfg.RequiredBlocks = make(map[uint64]common.Hash)
 	for _, entry := range strings.Split(requiredBlocks, ",") {
@@ -1837,11 +1806,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	}
 	if ctx.IsSet(TransactionHistoryFlag.Name) {
 		cfg.TransactionHistory = ctx.Uint64(TransactionHistoryFlag.Name)
-	} else if ctx.IsSet(TxLookupLimitFlag.Name) {
-		log.Warn("The flag --txlookuplimit is deprecated and will be removed, please use --history.transactions")
-		cfg.TransactionHistory = ctx.Uint64(TxLookupLimitFlag.Name)
 	}
-	if ctx.String(GCModeFlag.Name) == "archive" {
+	if cfg.NoPruning {
 		if cfg.TransactionHistory != 0 {
 			cfg.TransactionHistory = 0
 			log.Warn("Disabled transaction unindexing for archive node")
@@ -1936,8 +1902,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 			cfg.EthDiscoveryURLs = SplitAndTrim(urls)
 		}
 	}
-	if ctx.Bool(StateSizeTrackingFlag.Name) {
-		cfg.EnableStateSizeTracking = true
+	if ctx.IsSet(StateSizeTrackingFlag.Name) {
+		cfg.EnableStateSizeTracking = ctx.Bool(StateSizeTrackingFlag.Name)
 	}
 	// Override any default configs for hard coded networks.
 	switch {

cmd/utils/flags_legacy.go

@@ -19,8 +19,6 @@ package utils
 import (
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/eth/ethconfig"
-	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/urfave/cli/v2"
 )
 
@@ -32,127 +30,7 @@ var ShowDeprecated = &cli.Command{
 	Description: "Show flags that have been deprecated and will soon be removed",
 }
 
-var DeprecatedFlags = []cli.Flag{
-	NoUSBFlag,
-	LegacyWhitelistFlag,
-	CacheTrieJournalFlag,
-	CacheTrieRejournalFlag,
-	LegacyDiscoveryV5Flag,
-	TxLookupLimitFlag,
-	LogBacktraceAtFlag,
-	LogDebugFlag,
-	MinerNewPayloadTimeoutFlag,
-	MinerEtherbaseFlag,
-	MiningEnabledFlag,
-	MetricsEnabledExpensiveFlag,
-	EnablePersonal,
-	UnlockedAccountFlag,
-	InsecureUnlockAllowedFlag,
-}
-
-var (
-	// Deprecated May 2020, shown in aliased flags section
-	NoUSBFlag = &cli.BoolFlag{
-		Name:     "nousb",
-		Hidden:   true,
-		Usage:    "Disables monitoring for and managing USB hardware wallets (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
-	// Deprecated March 2022
-	LegacyWhitelistFlag = &cli.StringFlag{
-		Name:     "whitelist",
-		Hidden:   true,
-		Usage:    "Comma separated block number-to-hash mappings to enforce (<number>=<hash>) (deprecated in favor of --eth.requiredblocks)",
-		Category: flags.DeprecatedCategory,
-	}
-	// Deprecated July 2023
-	CacheTrieJournalFlag = &cli.StringFlag{
-		Name:     "cache.trie.journal",
-		Hidden:   true,
-		Usage:    "Disk journal directory for trie cache to survive node restarts",
-		Category: flags.DeprecatedCategory,
-	}
-	CacheTrieRejournalFlag = &cli.DurationFlag{
-		Name:     "cache.trie.rejournal",
-		Hidden:   true,
-		Usage:    "Time interval to regenerate the trie cache journal",
-		Category: flags.DeprecatedCategory,
-	}
-	LegacyDiscoveryV5Flag = &cli.BoolFlag{
-		Name:     "v5disc",
-		Hidden:   true,
-		Usage:    "Enables the experimental RLPx V5 (Topic Discovery) mechanism (deprecated, use --discv5 instead)",
-		Category: flags.DeprecatedCategory,
-	}
-	// Deprecated August 2023
-	TxLookupLimitFlag = &cli.Uint64Flag{
-		Name:     "txlookuplimit",
-		Hidden:   true,
-		Usage:    "Number of recent blocks to maintain transactions index for (default = about one year, 0 = entire chain) (deprecated, use history.transactions instead)",
-		Value:    ethconfig.Defaults.TransactionHistory,
-		Category: flags.DeprecatedCategory,
-	}
-	// Deprecated November 2023
-	LogBacktraceAtFlag = &cli.StringFlag{
-		Name:     "log.backtrace",
-		Hidden:   true,
-		Usage:    "Request a stack trace at a specific logging statement (deprecated)",
-		Value:    "",
-		Category: flags.DeprecatedCategory,
-	}
-	LogDebugFlag = &cli.BoolFlag{
-		Name:     "log.debug",
-		Hidden:   true,
-		Usage:    "Prepends log messages with call-site location (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
-	// Deprecated February 2024
-	MinerNewPayloadTimeoutFlag = &cli.DurationFlag{
-		Name:     "miner.newpayload-timeout",
-		Hidden:   true,
-		Usage:    "Specify the maximum time allowance for creating a new payload (deprecated)",
-		Value:    ethconfig.Defaults.Miner.Recommit,
-		Category: flags.DeprecatedCategory,
-	}
-	MinerEtherbaseFlag = &cli.StringFlag{
-		Name:     "miner.etherbase",
-		Hidden:   true,
-		Usage:    "0x prefixed public address for block mining rewards (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
-	MiningEnabledFlag = &cli.BoolFlag{
-		Name:     "mine",
-		Hidden:   true,
-		Usage:    "Enable mining (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
-	MetricsEnabledExpensiveFlag = &cli.BoolFlag{
-		Name:     "metrics.expensive",
-		Hidden:   true,
-		Usage:    "Enable expensive metrics collection and reporting (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
-	// Deprecated Oct 2024
-	EnablePersonal = &cli.BoolFlag{
-		Name:     "rpc.enabledeprecatedpersonal",
-		Hidden:   true,
-		Usage:    "This used to enable the 'personal' namespace.",
-		Category: flags.DeprecatedCategory,
-	}
-	UnlockedAccountFlag = &cli.StringFlag{
-		Name:     "unlock",
-		Hidden:   true,
-		Usage:    "Comma separated list of accounts to unlock (deprecated)",
-		Value:    "",
-		Category: flags.DeprecatedCategory,
-	}
-	InsecureUnlockAllowedFlag = &cli.BoolFlag{
-		Name:     "allow-insecure-unlock",
-		Hidden:   true,
-		Usage:    "Allow insecure account unlocking when account-related RPCs are exposed by http (deprecated)",
-		Category: flags.DeprecatedCategory,
-	}
-)
+var DeprecatedFlags = []cli.Flag{}
 
 // showDeprecated displays deprecated flags that will be soon removed from the codebase.
 func showDeprecated(*cli.Context) error {

cmd/utils/history_test.go

@@ -53,7 +53,7 @@ func TestHistoryImportAndExport(t *testing.T) {
 		from     func(f era.ReadAtSeekCloser) (era.Era, error)
 	}{
 		{"era1", onedb.NewBuilder, onedb.Filename, onedb.From},
-		{"erae", execdb.NewBuilder, execdb.Filename, execdb.From},
+		{"ere", execdb.NewBuilder, execdb.Filename, execdb.From},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			var (
@@ -106,10 +106,14 @@ func TestHistoryImportAndExport(t *testing.T) {
 			}
 
 			// Read checksums.
-			b, err := os.ReadFile(filepath.Join(dir, "checksums.txt"))
+			checksumsFile := filepath.Join(dir, "checksums.txt")
+			b, err := os.ReadFile(checksumsFile)
 			if err != nil {
 				t.Fatalf("failed to read checksums: %v", err)
 			}
+
+			// Add a trailing newline to ensure checksum handling is defensive.
+			_ = os.WriteFile(checksumsFile, append(b, '\n'), 0644)
 			checksums := strings.Split(string(b), "\n")
 
 			// Verify each Era.

consensus/beacon/consensus.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
 	"github.com/ethereum/go-ethereum/core/tracing"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/types/bal"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/holiman/uint256"
@@ -342,9 +343,9 @@ func (beacon *Beacon) Prepare(chain consensus.ChainHeaderReader, header *types.H
 }
 
 // Finalize implements consensus.Engine and processes withdrawals on top.
-func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state vm.StateDB, body *types.Body) {
+func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state vm.StateDB, body *types.Body, blockAccessIndex uint32, bal *bal.ConstructionBlockAccessList) {
 	if !beacon.IsPoSHeader(header) {
-		beacon.ethone.Finalize(chain, header, state, body)
+		beacon.ethone.Finalize(chain, header, state, body, blockAccessIndex, bal)
 		return
 	}
 	// Withdrawals processing.
@@ -352,7 +353,20 @@ func (beacon *Beacon) Finalize(chain consensus.ChainHeaderReader, header *types.
 		// Convert amount from gwei to wei.
 		amount := new(uint256.Int).SetUint64(w.Amount)
 		amount = amount.Mul(amount, uint256.NewInt(params.GWei))
-		state.AddBalance(w.Address, amount, tracing.BalanceIncreaseWithdrawal)
+		prev := state.AddBalance(w.Address, amount, tracing.BalanceIncreaseWithdrawal)
+
+		// Populate the block-level accessList if Amsterdam is enabled
+		if chain.Config().IsAmsterdam(header.Number, header.Time) {
+			if w.Amount == 0 {
+				// Zero amount withdrawal, account is accessed potential
+				// without state changes.
+				bal.AccountRead(w.Address)
+			} else {
+				// Non-zero amount withdrawal, account is accessed with
+				// a balance change.
+				bal.BalanceChange(blockAccessIndex, w.Address, new(uint256.Int).Add(&prev, amount))
+			}
+		}
 	}
 	// No block reward which is issued by consensus layer instead.
 }

consensus/clique/clique.go

@@ -34,6 +34,7 @@ import (
 	"github.com/ethereum/go-ethereum/consensus/misc"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/types/bal"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/crypto/keccak"
@@ -573,7 +574,7 @@ func (c *Clique) Prepare(chain consensus.ChainHeaderReader, header *types.Header
 
 // Finalize implements consensus.Engine. There is no post-transaction
 // consensus rules in clique, do nothing here.
-func (c *Clique) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state vm.StateDB, body *types.Body) {
+func (c *Clique) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state vm.StateDB, body *types.Body, blockAccessIndex uint32, bal *bal.ConstructionBlockAccessList) {
 	// No block rewards in PoA, so the state remains as is
 }
 

consensus/consensus.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/types/bal"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/params"
 )
@@ -79,12 +80,12 @@ type Engine interface {
 	// rules of a particular engine. The changes are executed inline.
 	Prepare(chain ChainHeaderReader, header *types.Header) error
 
-	// Finalize runs any post-transaction state modifications (e.g. block rewards
-	// or process withdrawals) but does not assemble the block.
+	// Finalize runs any post-transaction consensus-specific state modifications
+	// (e.g. block rewards or process withdrawals) but does not assemble the block.
 	//
 	// Note: The state database might be updated to reflect any consensus rules
 	// that happen at finalization (e.g. block rewards).
-	Finalize(chain ChainHeaderReader, header *types.Header, state vm.StateDB, body *types.Body)
+	Finalize(chain ChainHeaderReader, header *types.Header, state vm.StateDB, body *types.Body, blockAccessIndex uint32, bal *bal.ConstructionBlockAccessList)
 
 	// Seal generates a new sealing request for the given input block and pushes
 	// the result into the given channel.

consensus/ethash/consensus.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
 	"github.com/ethereum/go-ethereum/core/tracing"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/types/bal"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto/keccak"
 	"github.com/ethereum/go-ethereum/params"
@@ -504,7 +505,7 @@ func (ethash *Ethash) Prepare(chain consensus.ChainHeaderReader, header *types.H
 }
 
 // Finalize implements consensus.Engine, accumulating the block and uncle rewards.
-func (ethash *Ethash) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state vm.StateDB, body *types.Body) {
+func (ethash *Ethash) Finalize(chain consensus.ChainHeaderReader, header *types.Header, state vm.StateDB, body *types.Body, blockAccessIndex uint32, bal *bal.ConstructionBlockAccessList) {
 	// Accumulate any block and uncle rewards
 	accumulateRewards(chain.Config(), state, header, body.Uncles)
 }

core/bintrie_witness_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/tracing"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/types/bal"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
@@ -202,7 +203,7 @@ func TestProcessParentBlockHash(t *testing.T) {
 			}
 			vmContext := NewEVMBlockContext(header, nil, new(common.Address))
 			evm := vm.NewEVM(vmContext, statedb, chainConfig, vm.Config{})
-			ProcessParentBlockHash(header.ParentHash, evm)
+			ProcessParentBlockHash(header.ParentHash, evm, bal.NewConstructionBlockAccessList())
 		}
 		// Read block hashes for block 0 .. num-1
 		for i := 0; i < num; i++ {

core/block_validator.go

@@ -111,6 +111,28 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error {
 		}
 	}
 
+	// Block access list hash must be present in header after the
+	// Amsterdam hard fork.
+	if v.config.IsAmsterdam(block.Number(), block.Time()) {
+		if block.Header().BlockAccessListHash == nil {
+			return errors.New("block access list hash not set in header")
+		}
+		// If the block does not include an access list, compute it locally during
+		// execution and validate it against the access list hash in the header.
+		//
+		// If the block includes an attached access list, validate it directly here.
+		if block.AccessList() != nil {
+			computed := block.AccessList().Hash()
+			if *block.Header().BlockAccessListHash != computed {
+				return fmt.Errorf("access list hash mismatch, computed: %x, remote: %x", computed, *block.Header().BlockAccessListHash)
+			} else if err := block.AccessList().Validate(block.GasLimit(), len(block.Transactions())); err != nil {
+				return fmt.Errorf("invalid block access list: %v", err)
+			}
+		}
+	} else if block.Header().BlockAccessListHash != nil || block.AccessList() != nil {
+		return errors.New("block had access list before Amsterdam")
+	}
+
 	// Ancestor block must be known.
 	if !v.bc.HasBlockAndState(block.ParentHash(), block.NumberU64()-1) {
 		if !v.bc.HasBlock(block.ParentHash(), block.NumberU64()-1) {
@@ -160,6 +182,23 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
 	} else if res.Requests != nil {
 		return errors.New("block has requests before prague fork")
 	}
+	// Verify Block-level accessList once Amsterdam is enabled
+	if v.config.IsAmsterdam(block.Number(), block.Time()) {
+		if res.Bal == nil {
+			return errors.New("block access list is not available in amsterdam")
+		}
+		if block.Header().BlockAccessListHash == nil {
+			return errors.New("block access list hash not set in header")
+		}
+		enc := res.Bal.ToEncodingObj()
+		local, remote := enc.Hash(), *block.Header().BlockAccessListHash
+		if local != remote {
+			return fmt.Errorf("access list hash mismatch, local: %x, remote: %x", local, remote)
+		}
+		if err := enc.Validate(block.GasLimit(), len(block.Transactions())); err != nil {
+			return fmt.Errorf("invalid block access list: %v", err)
+		}
+	}
 	// Validate the state root against the received state root and throw
 	// an error if they don't match.
 	if root := statedb.IntermediateRoot(v.config.IsEIP158(header.Number)); header.Root != root {

core/blockchain.go

@@ -84,11 +84,15 @@ var (
 	accountCacheMissMeter = metrics.NewRegisteredMeter("chain/account/reads/cache/process/miss", nil)
 	storageCacheHitMeter  = metrics.NewRegisteredMeter("chain/storage/reads/cache/process/hit", nil)
 	storageCacheMissMeter = metrics.NewRegisteredMeter("chain/storage/reads/cache/process/miss", nil)
+	codeCacheHitMeter     = metrics.NewRegisteredMeter("chain/code/reads/cache/process/hit", nil)
+	codeCacheMissMeter    = metrics.NewRegisteredMeter("chain/code/reads/cache/process/miss", nil)
 
 	accountCacheHitPrefetchMeter  = metrics.NewRegisteredMeter("chain/account/reads/cache/prefetch/hit", nil)
 	accountCacheMissPrefetchMeter = metrics.NewRegisteredMeter("chain/account/reads/cache/prefetch/miss", nil)
 	storageCacheHitPrefetchMeter  = metrics.NewRegisteredMeter("chain/storage/reads/cache/prefetch/hit", nil)
 	storageCacheMissPrefetchMeter = metrics.NewRegisteredMeter("chain/storage/reads/cache/prefetch/miss", nil)
+	codeCacheHitPrefetchMeter     = metrics.NewRegisteredMeter("chain/code/reads/cache/prefetch/hit", nil)
+	codeCacheMissPrefetchMeter    = metrics.NewRegisteredMeter("chain/code/reads/cache/prefetch/miss", nil)
 
 	accountReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/account/single/reads", nil)
 	storageReadSingleTimer = metrics.NewRegisteredResettingTimer("chain/storage/single/reads", nil)
@@ -326,6 +330,7 @@ type BlockChain struct {
 	flushInterval atomic.Int64                     // Time interval (processing time) after which to flush a state
 	triedb        *triedb.Database                 // The database handler for maintaining trie nodes.
 	codedb        *state.CodeDB                    // The database handler for maintaining contract codes.
+	jumpDestCache vm.JumpDestCache                 // Shared JUMPDEST analysis cache for block processing
 	txIndexer     *txIndexer                       // Transaction indexer, might be nil if not enabled
 
 	hc               *HeaderChain
@@ -408,6 +413,7 @@ func NewBlockChain(db ethdb.Database, genesis *Genesis, engine consensus.Engine,
 		db:                 db,
 		triedb:             triedb,
 		codedb:             state.NewCodeDB(db),
+		jumpDestCache:      NewJumpDestCache(),
 		triegc:             prque.New[int64, common.Hash](nil),
 		chainmu:            syncx.NewClosableMutex(),
 		bodyCache:          lru.NewCache[common.Hash, *types.Body](bodyCacheLimit),
@@ -716,7 +722,7 @@ func (bc *BlockChain) loadLastState() error {
 
 // initializeHistoryPruning sets bc.historyPrunePoint.
 func (bc *BlockChain) initializeHistoryPruning(latest uint64) error {
-	freezerTail, _ := bc.db.Tail()
+	freezerTail, _ := bc.db.Tail(rawdb.ChainFreezerBlockDataGroup)
 	policy := bc.cfg.HistoryPolicy
 
 	switch policy.Mode {
@@ -2179,7 +2185,7 @@ func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash,
 			// Disable tracing for prefetcher executions.
 			vmCfg := bc.cfg.VmConfig
 			vmCfg.Tracer = nil
-			bc.prefetcher.Prefetch(block, throwaway, vmCfg, &interrupt)
+			bc.prefetcher.Prefetch(block, throwaway, bc.jumpDestCache, vmCfg, &interrupt)
 
 			blockPrefetchExecuteTimer.Update(time.Since(start))
 			if interrupt.Load() {
@@ -2225,7 +2231,7 @@ func (bc *BlockChain) ProcessBlock(ctx context.Context, parentRoot common.Hash,
 	// Process block using the parent state as reference point
 	pstart := time.Now()
 	pctx, _, spanEnd := telemetry.StartSpan(ctx, "bc.processor.Process")
-	res, err := bc.processor.Process(pctx, block, statedb, bc.cfg.VmConfig)
+	res, err := bc.processor.Process(pctx, block, statedb, bc.jumpDestCache, bc.cfg.VmConfig)
 	spanEnd(&err)
 	if err != nil {
 		bc.reportBadBlock(block, res, err)
@@ -2961,7 +2967,7 @@ func (bc *BlockChain) InsertHeadersBeforeCutoff(headers []*types.Header) (int, e
 	}
 	// Truncate the useless chain segment (zero bodies and receipts) in the
 	// ancient store.
-	if _, err := bc.db.TruncateTail(last.Number.Uint64() + 1); err != nil {
+	if _, err := bc.db.TruncateTail(rawdb.ChainFreezerBlockDataGroup, last.Number.Uint64()+1); err != nil {
 		return 0, err
 	}
 	// Last step update all in-memory markers

core/blockchain_reader.go

@@ -296,6 +296,7 @@ func (bc *BlockChain) GetReceiptsRLP(hash common.Hash) rlp.RawValue {
 	return rawdb.ReadReceiptsRLP(bc.db, hash, number)
 }
 
+// GetAccessListRLP retrieves the block access list of a block in RLP encoding.
 func (bc *BlockChain) GetAccessListRLP(hash common.Hash) rlp.RawValue {
 	number, ok := rawdb.ReadHeaderNumber(bc.db, hash)
 	if !ok {

core/blockchain_stats.go

@@ -96,11 +96,15 @@ func (s *ExecuteStats) reportMetrics() {
 	accountCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.AccountCacheMiss)
 	storageCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.StorageCacheHit)
 	storageCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.StateStats.StorageCacheMiss)
+	codeCacheHitPrefetchMeter.Mark(s.StatePrefetchCacheStats.CodeStats.CacheHit)
+	codeCacheMissPrefetchMeter.Mark(s.StatePrefetchCacheStats.CodeStats.CacheMiss)
 
 	accountCacheHitMeter.Mark(s.StateReadCacheStats.StateStats.AccountCacheHit)
 	accountCacheMissMeter.Mark(s.StateReadCacheStats.StateStats.AccountCacheMiss)
 	storageCacheHitMeter.Mark(s.StateReadCacheStats.StateStats.StorageCacheHit)
 	storageCacheMissMeter.Mark(s.StateReadCacheStats.StateStats.StorageCacheMiss)
+	codeCacheHitMeter.Mark(s.StateReadCacheStats.CodeStats.CacheHit)
+	codeCacheMissMeter.Mark(s.StateReadCacheStats.CodeStats.CacheMiss)
 }
 
 // slowBlockLog represents the JSON structure for slow block logging.

core/blockchain_test.go

@@ -160,7 +160,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
 		if err != nil {
 			return err
 		}
-		res, err := blockchain.processor.Process(context.Background(), block, statedb, vm.Config{})
+		res, err := blockchain.processor.Process(context.Background(), block, statedb, nil, vm.Config{})
 		if err != nil {
 			blockchain.reportBadBlock(block, res, err)
 			return err
@@ -4386,7 +4386,7 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64,
 		if header.Hash() != hash {
 			t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
 		}
-		tail, err := db.Tail()
+		tail, err := db.Tail(rawdb.ChainFreezerBlockDataGroup)
 		if err != nil {
 			t.Fatalf("Failed to get chain tail, %v", err)
 		}

core/chain_makers.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/types/bal"
 	"github.com/ethereum/go-ethereum/core/vm"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
@@ -50,6 +51,7 @@ type BlockGen struct {
 	receipts    []*types.Receipt
 	uncles      []*types.Header
 	withdrawals []*types.Withdrawal
+	bal         *bal.ConstructionBlockAccessList
 
 	engine consensus.Engine
 }
@@ -99,7 +101,7 @@ func (b *BlockGen) Difficulty() *big.Int {
 func (b *BlockGen) SetParentBeaconRoot(root common.Hash) {
 	b.header.ParentBeaconRoot = &root
 	blockContext := NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase)
-	ProcessBeaconBlockRoot(root, vm.NewEVM(blockContext, b.statedb, b.cm.config, vm.Config{}))
+	ProcessBeaconBlockRoot(root, vm.NewEVM(blockContext, b.statedb, b.cm.config, vm.Config{}), b.bal)
 }
 
 // addTx adds a transaction to the generated block. If no coinbase has
@@ -118,7 +120,7 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti
 		evm          = vm.NewEVM(blockContext, b.statedb, b.cm.config, vmConfig)
 	)
 	b.statedb.SetTxContext(tx.Hash(), len(b.txs), uint32(len(b.txs)+1))
-	receipt, err := ApplyTransaction(evm, b.gasPool, b.statedb, b.header, tx)
+	receipt, bal, err := ApplyTransaction(evm, b.gasPool, b.statedb, b.header, tx)
 	if err != nil {
 		panic(err)
 	}
@@ -134,6 +136,7 @@ func (b *BlockGen) addTx(bc *BlockChain, vmConfig vm.Config, tx *types.Transacti
 	if b.header.BlobGasUsed != nil {
 		*b.header.BlobGasUsed += receipt.BlobGasUsed
 	}
+	b.bal.Merge(bal)
 }
 
 // AddTx adds a transaction to the generated block. If no coinbase has
@@ -304,10 +307,11 @@ func (b *BlockGen) OffsetTime(seconds int64) {
 
 // ConsensusLayerRequests returns the EIP-7685 requests which have accumulated so far.
 func (b *BlockGen) ConsensusLayerRequests() [][]byte {
-	return b.collectRequests(true)
+	requests, _ := b.collectRequests(true)
+	return requests
 }
 
-func (b *BlockGen) collectRequests(readonly bool) (requests [][]byte) {
+func (b *BlockGen) collectRequests(readonly bool) (requests [][]byte, bal *bal.ConstructionBlockAccessList) {
 	statedb := b.statedb
 	if readonly {
 		// The system contracts clear themselves on a system-initiated read.
@@ -323,11 +327,11 @@ func (b *BlockGen) collectRequests(readonly bool) (requests [][]byte) {
 	blockContext := NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase)
 	evm := vm.NewEVM(blockContext, statedb, b.cm.config, vm.Config{})
 
-	requests, err := PostExecution(context.Background(), b.cm.config, b.header.Number, b.header.Time, blockLogs, evm, uint32(len(b.txs)+1))
+	requests, bal, err := PostExecution(context.Background(), b.cm.config, b.header.Number, b.header.Time, blockLogs, evm, uint32(len(b.txs)+1))
 	if err != nil {
 		panic(fmt.Sprintf("failed to run post-execution: %v", err))
 	}
-	return requests
+	return requests, bal
 }
 
 // GenerateChain creates a chain of n blocks. The first block's
@@ -354,6 +358,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 	genblock := func(i int, parent *types.Block, triedb *triedb.Database, statedb *state.StateDB) (*types.Block, types.Receipts) {
 		b := &BlockGen{i: i, cm: cm, parent: parent, statedb: statedb, engine: engine}
 		b.header = cm.makeHeader(parent, statedb, b.engine)
+		b.bal = bal.NewConstructionBlockAccessList()
 
 		// Set the difficulty for clique block. The chain maker doesn't have access
 		// to a chain, so the difficulty will be left unset (nil). Set it here to the
@@ -386,7 +391,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 			blockContext := NewEVMBlockContext(b.header, cm, &b.header.Coinbase)
 			blockContext.Random = &common.Hash{} // enable post-merge instruction set
 			evm := vm.NewEVM(blockContext, statedb, cm.config, vm.Config{})
-			ProcessParentBlockHash(b.header.ParentHash, evm)
+			ProcessParentBlockHash(b.header.ParentHash, evm, b.bal)
 		}
 
 		// Execute any user modifications to the block
@@ -394,11 +399,12 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 			gen(i, b)
 		}
 
-		requests := b.collectRequests(false)
+		requests, bal := b.collectRequests(false)
 		if requests != nil {
 			reqHash := types.CalcRequestsHash(requests)
 			b.header.RequestsHash = &reqHash
 		}
+		b.bal.Merge(bal)
 
 		body := types.Body{
 			Transactions: b.txs,
@@ -414,8 +420,11 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 				body.Withdrawals = make([]*types.Withdrawal, 0)
 			}
 		}
+		// Apply the consensus-specific post-transaction changes
+		b.engine.Finalize(cm, b.header, statedb, &body, uint32(len(b.txs)+1), b.bal)
+
 		// Assemble the block for delivery.
-		block := AssembleBlock(b.engine, cm, b.header, statedb, &body, b.receipts)
+		block := AssembleBlock(cm, b.header, statedb, &body, b.receipts, b.bal)
 
 		// Write state changes to db
 		root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number), config.IsCancun(b.header.Number, b.header.Time))
@@ -519,6 +528,13 @@ func (cm *chainMaker) makeHeader(parent *types.Block, state *state.StateDB, engi
 		header.BlobGasUsed = new(uint64)
 		header.ParentBeaconRoot = new(common.Hash)
 	}
+	if cm.config.IsAmsterdam(header.Number, header.Time) {
+		var slot uint64
+		if parentHeader.SlotNumber != nil {
+			slot = *parentHeader.SlotNumber + 1
+		}
+		header.SlotNumber = &slot
+	}
 	return header
 }
 

core/genesis.go

@@ -555,6 +555,7 @@ func (g *Genesis) toBlockWithRoot(root common.Hash) *types.Block {
 			if head.SlotNumber == nil {
 				head.SlotNumber = new(uint64)
 			}
+			head.BlockAccessListHash = &types.EmptyBlockAccessListHash
 		}
 	}
 	return types.NewBlock(head, &types.Body{Withdrawals: withdrawals}, nil, trie.NewStackTrie(nil))

core/jumpdest.go

@@ -0,0 +1,76 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package core
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/lru"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/metrics"
+)
+
+var (
+	jumpDestHitMeter  = metrics.NewRegisteredMeter("chain/cache/jumpdest/hit", nil)
+	jumpDestMissMeter = metrics.NewRegisteredMeter("chain/cache/jumpdest/miss", nil)
+)
+
+const (
+	// jumpDestBuckets is the number of independent LRU shards. Code hashes
+	// are dispatched by the low bits of the first byte to spread load across
+	// shards and reduce mutex contention from the parallel prefetcher.
+	jumpDestBuckets = 8
+
+	// jumpDestBucketSize is the per-shard byte budget.
+	jumpDestBucketSize = 8 * 1024 * 1024
+)
+
+// shardedJumpDestCache is a thread-safe, byte-bounded LRU of JUMPDEST analysis
+// bitmaps, sharded into independent buckets to reduce lock contention. It is
+// owned by BlockChain and shared across block processing and prefetching,
+// keyed by the immutable contract code hash.
+type shardedJumpDestCache struct {
+	buckets [jumpDestBuckets]struct {
+		dest *lru.SizeConstrainedCache[common.Hash, vm.BitVec]
+	}
+}
+
+// NewJumpDestCache constructs the analysis cache.
+func NewJumpDestCache() vm.JumpDestCache {
+	c := new(shardedJumpDestCache)
+	for i := range c.buckets {
+		c.buckets[i].dest = lru.NewSizeConstrainedCache[common.Hash, vm.BitVec](jumpDestBucketSize)
+	}
+	return c
+}
+
+// Load retrieves the cached jumpdest analysis for the given code hash.
+func (c *shardedJumpDestCache) Load(hash common.Hash) (vm.BitVec, bool) {
+	bucket := &c.buckets[hash[0]&(jumpDestBuckets-1)]
+	v, ok := bucket.dest.Get(hash)
+	if ok {
+		jumpDestHitMeter.Mark(1)
+	} else {
+		jumpDestMissMeter.Mark(1)
+	}
+	return v, ok
+}
+
+// Store saves the jumpdest analysis for the given code hash.
+func (c *shardedJumpDestCache) Store(hash common.Hash, b vm.BitVec) {
+	bucket := &c.buckets[hash[0]&(jumpDestBuckets-1)]
+	bucket.dest.Add(hash, b)
+}

core/rawdb/accessors_chain.go

@@ -614,9 +614,18 @@ func HasAccessList(db ethdb.Reader, hash common.Hash, number uint64) bool {
 	return has
 }
 
-// ReadAccessListRLP retrieves the RLP-encoded block access list for a block from KV.
+// ReadAccessListRLP retrieves the RLP-encoded block access list for a block.
 func ReadAccessListRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
-	data, _ := db.Get(accessListKey(number, hash))
+	var data []byte
+	db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
+		data, _ = reader.Ancient(ChainFreezerBALTable, number)
+		if len(data) > 0 {
+			return nil
+		}
+		// Block is not in ancients, read from key-value store by hash and number.
+		data, _ = db.Get(accessListKey(number, hash))
+		return nil
+	})
 	return data
 }
 
@@ -759,6 +768,13 @@ func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *type
 	if err := op.Append(ChainFreezerReceiptTable, num, receipts); err != nil {
 		return fmt.Errorf("can't append block %d receipts: %v", num, err)
 	}
+	// The assumption is held that BAL of ancient block is no longer available
+	// (it may still reachable, but it's not worthwhile to even retrieve it
+	// from the network). A nil entry is stored in the BAL table as the absence
+	// placeholder.
+	if err := op.AppendRaw(ChainFreezerBALTable, num, nil); err != nil {
+		return fmt.Errorf("can't append block %d bals: %v", num, err)
+	}
 	return nil
 }
 
@@ -781,6 +797,13 @@ func WriteAncientHeaderChain(db ethdb.AncientWriter, headers []*types.Header) (i
 			if err := op.AppendRaw(ChainFreezerReceiptTable, num, nil); err != nil {
 				return fmt.Errorf("can't append block %d receipts: %v", num, err)
 			}
+			// The assumption is held that BAL of ancient block is no longer available
+			// (it may still reachable, but it's not worthwhile to even retrieve it
+			// from the network). A nil entry is stored in the BAL table as the absence
+			// placeholder.
+			if err := op.AppendRaw(ChainFreezerBALTable, num, nil); err != nil {
+				return fmt.Errorf("can't append block %d bals: %v", num, err)
+			}
 		}
 		return nil
 	})
@@ -791,6 +814,7 @@ func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
 	DeleteReceipts(db, hash, number)
 	DeleteHeader(db, hash, number)
 	DeleteBody(db, hash, number)
+	DeleteAccessList(db, hash, number)
 }
 
 // DeleteBlockWithoutNumber removes all block data associated with a hash, except
@@ -799,6 +823,7 @@ func DeleteBlockWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number
 	DeleteReceipts(db, hash, number)
 	deleteHeaderWithoutNumber(db, hash, number)
 	DeleteBody(db, hash, number)
+	DeleteAccessList(db, hash, number)
 }
 
 const badBlockToKeep = 10

core/rawdb/accessors_chain_test.go

@@ -926,6 +926,47 @@ func makeTestBAL(t *testing.T) (rlp.RawValue, *bal.BlockAccessList) {
 	return encoded, &decoded
 }
 
+// TestWriteAncientBlocksNilBAL ensures that freezing a block with no block
+// access list produces an empty entry in the BAL ancient table and that
+// ReadAccessList returns nil afterwards (i.e. the empty entry is not surfaced
+// as a malformed BAL).
+func TestWriteAncientBlocksNilBAL(t *testing.T) {
+	db, err := Open(NewMemoryDatabase(), OpenOptions{Ancient: t.TempDir()})
+	if err != nil {
+		t.Fatalf("failed to create database with ancient backend: %v", err)
+	}
+	defer db.Close()
+
+	block := types.NewBlockWithHeader(&types.Header{
+		Number:      big.NewInt(0),
+		Extra:       []byte("nil-bal block"),
+		UncleHash:   types.EmptyUncleHash,
+		TxHash:      types.EmptyTxsHash,
+		ReceiptHash: types.EmptyReceiptsHash,
+	})
+	if block.AccessList() != nil {
+		t.Fatalf("test precondition: block must have nil access list")
+	}
+	if _, err := WriteAncientBlocks(db, []*types.Block{block}, types.EncodeBlockReceiptLists([]types.Receipts{nil})); err != nil {
+		t.Fatalf("WriteAncientBlocks failed: %v", err)
+	}
+	hash, number := block.Hash(), block.NumberU64()
+
+	// The BAL ancient entry should exist as an empty blob.
+	if blob := ReadAccessListRLP(db, hash, number); len(blob) != 0 {
+		t.Fatalf("ReadAccessListRLP: got %x, want empty", blob)
+	}
+	// ReadAccessList must surface nil rather than attempting to RLP-decode
+	// the empty payload.
+	if b := ReadAccessList(db, hash, number); b != nil {
+		t.Fatalf("ReadAccessList: got %v, want nil", b)
+	}
+	// HasAccessList only consults the KV store and there's nothing there.
+	if HasAccessList(db, hash, number) {
+		t.Fatal("HasAccessList returned true for absent BAL")
+	}
+}
+
 // TestBALStorage tests write/read/delete of BALs in the KV store.
 func TestBALStorage(t *testing.T) {
 	db := NewMemoryDatabase()

core/rawdb/accessors_snapshot.go

@@ -208,3 +208,45 @@ func WriteSnapshotSyncStatus(db ethdb.KeyValueWriter, status []byte) {
 		log.Crit("Failed to store snapshot sync status", "err", err)
 	}
 }
+
+// ReadGenerateTriePartitionDone returns the raw subtree root blob for a
+// partition that has previously completed.
+func ReadGenerateTriePartitionDone(db ethdb.KeyValueReader, partition byte) ([]byte, bool) {
+	data, err := db.Get(generateTriePartitionDoneKey(partition))
+	if err != nil {
+		return nil, false
+	}
+	if len(data) == 0 {
+		return nil, false
+	}
+	switch data[0] {
+	case 0x00:
+		// Partition is done and it is empty.
+		return nil, true
+	case 0x01:
+		// Partition is done and the blob follows.
+		return data[1:], true
+	default:
+		return nil, false
+	}
+}
+
+// WriteGenerateTriePartitionDone records a completed partition.
+func WriteGenerateTriePartitionDone(db ethdb.KeyValueWriter, partition byte, blob []byte) {
+	var value []byte
+	if blob == nil {
+		value = []byte{0x00}
+	} else {
+		value = append([]byte{0x01}, blob...)
+	}
+	if err := db.Put(generateTriePartitionDoneKey(partition), value); err != nil {
+		log.Crit("Failed to store generate-trie done marker", "err", err)
+	}
+}
+
+// DeleteGenerateTriePartitionDone removes a partition's done marker.
+func DeleteGenerateTriePartitionDone(db ethdb.KeyValueWriter, partition byte) {
+	if err := db.Delete(generateTriePartitionDoneKey(partition)); err != nil {
+		log.Crit("Failed to remove generate-trie done marker", "err", err)
+	}
+}

core/rawdb/ancient_scheme.go

@@ -35,6 +35,23 @@ const (
 
 	// ChainFreezerReceiptTable indicates the name of the freezer receipts table.
 	ChainFreezerReceiptTable = "receipts"
+
+	// ChainFreezerBALTable indicates the name of the freezer block access list
+	// table introduced by EIP-7928.
+	ChainFreezerBALTable = "bals"
+)
+
+// Identifiers of tail groups used by the chain freezer.
+const (
+	// ChainFreezerBlockDataGroup is the tail group shared by the body and
+	// receipt tables. The two tables are pruned together and therefore have
+	// the same tail position.
+	ChainFreezerBlockDataGroup = "blockdata"
+
+	// ChainFreezerBALGroup is the tail group for the block access list table.
+	// BAL is only populated after EIP-7928 activates, so it generally has a
+	// higher tail than the block-data group and is pruned independently.
+	ChainFreezerBALGroup = "bal"
 )
 
 // chainFreezerTableConfigs configures the settings for tables in the chain freezer.
@@ -42,16 +59,23 @@ const (
 // tail truncation is disabled for the header and hash tables, as these are intended
 // to be retained long-term.
 var chainFreezerTableConfigs = map[string]freezerTableConfig{
-	ChainFreezerHeaderTable:  {noSnappy: false, prunable: false},
-	ChainFreezerHashTable:    {noSnappy: true, prunable: false},
-	ChainFreezerBodiesTable:  {noSnappy: false, prunable: true},
-	ChainFreezerReceiptTable: {noSnappy: false, prunable: true},
+	ChainFreezerHeaderTable:  {noSnappy: false},
+	ChainFreezerHashTable:    {noSnappy: true},
+	ChainFreezerBodiesTable:  {noSnappy: false, tailGroup: ChainFreezerBlockDataGroup},
+	ChainFreezerReceiptTable: {noSnappy: false, tailGroup: ChainFreezerBlockDataGroup},
+	ChainFreezerBALTable:     {noSnappy: false, tailGroup: ChainFreezerBALGroup},
 }
 
 // freezerTableConfig contains the settings for a freezer table.
 type freezerTableConfig struct {
-	noSnappy bool // disables item compression
-	prunable bool // true for tables that can be pruned by TruncateTail
+	// noSnappy disables item compression when true.
+	noSnappy bool
+
+	// tailGroup names a logical group of tables that share the same tail
+	// position. Tables in the same group are pruned together and must agree
+	// on their tail. An empty value means the table is not prunable; its
+	// tail is always 0.
+	tailGroup string
 }
 
 const (
@@ -66,13 +90,17 @@ const (
 	stateHistoryStorageData  = "storage.data"
 )
 
+// DefaultHistoryGroup is the tail group shared by all state/trienode history
+// tables with tail pruning enabled.
+const DefaultHistoryGroup = "history"
+
 // stateFreezerTableConfigs configures the settings for tables in the state freezer.
 var stateFreezerTableConfigs = map[string]freezerTableConfig{
-	stateHistoryMeta:         {noSnappy: true, prunable: true},
-	stateHistoryAccountIndex: {noSnappy: false, prunable: true},
-	stateHistoryStorageIndex: {noSnappy: false, prunable: true},
-	stateHistoryAccountData:  {noSnappy: false, prunable: true},
-	stateHistoryStorageData:  {noSnappy: false, prunable: true},
+	stateHistoryMeta:         {noSnappy: true, tailGroup: DefaultHistoryGroup},
+	stateHistoryAccountIndex: {noSnappy: false, tailGroup: DefaultHistoryGroup},
+	stateHistoryStorageIndex: {noSnappy: false, tailGroup: DefaultHistoryGroup},
+	stateHistoryAccountData:  {noSnappy: false, tailGroup: DefaultHistoryGroup},
+	stateHistoryStorageData:  {noSnappy: false, tailGroup: DefaultHistoryGroup},
 }
 
 const (
@@ -83,13 +111,13 @@ const (
 
 // trienodeFreezerTableConfigs configures the settings for tables in the trienode freezer.
 var trienodeFreezerTableConfigs = map[string]freezerTableConfig{
-	trienodeHistoryHeaderTable: {noSnappy: false, prunable: true},
+	trienodeHistoryHeaderTable: {noSnappy: false, tailGroup: DefaultHistoryGroup},
 
 	// Disable snappy compression to allow efficient partial read.
-	trienodeHistoryKeySectionTable: {noSnappy: true, prunable: true},
+	trienodeHistoryKeySectionTable: {noSnappy: true, tailGroup: DefaultHistoryGroup},
 
 	// Disable snappy compression to allow efficient partial read.
-	trienodeHistoryValueSectionTable: {noSnappy: true, prunable: true},
+	trienodeHistoryValueSectionTable: {noSnappy: true, tailGroup: DefaultHistoryGroup},
 }
 
 // The list of identifiers of ancient stores.

core/rawdb/ancient_utils.go

@@ -24,24 +24,23 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 )
 
-type tableSize struct {
-	name string
-	size common.StorageSize
+type tableInfo struct {
+	name  string
+	size  common.StorageSize
+	count uint64
 }
 
 // freezerInfo contains the basic information of the freezer.
 type freezerInfo struct {
-	name  string      // The identifier of freezer
-	head  uint64      // The number of last stored item in the freezer
-	tail  uint64      // The number of first stored item in the freezer
-	count uint64      // The number of stored items in the freezer
-	sizes []tableSize // The storage size per table
+	name   string      // The identifier of freezer
+	head   uint64      // The number of last stored item in the freezer
+	tables []tableInfo // Per-table storage size and item count
 }
 
 // size returns the storage size of the entire freezer.
 func (info *freezerInfo) size() common.StorageSize {
 	var total common.StorageSize
-	for _, table := range info.sizes {
+	for _, table := range info.tables {
 		total += table.size
 	}
 	return total
@@ -49,35 +48,41 @@ func (info *freezerInfo) size() common.StorageSize {
 
 func inspect(name string, order map[string]freezerTableConfig, reader ethdb.AncientReader) (freezerInfo, error) {
 	info := freezerInfo{name: name}
-	for t := range order {
-		size, err := reader.AncientSize(t)
-		if err != nil {
-			return freezerInfo{}, err
-		}
-		info.sizes = append(info.sizes, tableSize{name: t, size: common.StorageSize(size)})
-	}
-	// Retrieve the number of last stored item
+
+	// Retrieve the number of last stored item.
 	ancients, err := reader.Ancients()
 	if err != nil {
 		return freezerInfo{}, err
 	}
 	if ancients > 0 {
 		info.head = ancients - 1
-	} else {
-		info.head = 0
 	}
-
-	// Retrieve the number of first stored item
-	tail, err := reader.Tail()
-	if err != nil {
-		return freezerInfo{}, err
+	// Resolve per-group tails so each table can report its own item count.
+	groupTails := make(map[string]uint64)
+	for _, cfg := range order {
+		if cfg.tailGroup == "" {
+			continue
+		}
+		if _, ok := groupTails[cfg.tailGroup]; ok {
+			continue
+		}
+		t, err := reader.Tail(cfg.tailGroup)
+		if err != nil {
+			return freezerInfo{}, err
+		}
+		groupTails[cfg.tailGroup] = t
 	}
-	info.tail = tail
-
-	if ancients == 0 {
-		info.count = 0
-	} else {
-		info.count = info.head - info.tail + 1
+	for t, cfg := range order {
+		size, err := reader.AncientSize(t)
+		if err != nil {
+			return freezerInfo{}, err
+		}
+		var count uint64
+		if ancients > 0 {
+			tail := groupTails[cfg.tailGroup] // 0 for non-prunable tables
+			count = ancients - tail
+		}
+		info.tables = append(info.tables, tableInfo{name: t, size: common.StorageSize(size), count: count})
 	}
 	return info, nil
 }

core/rawdb/ancienttest/testsuite.go

@@ -25,6 +25,11 @@ import (
 	"github.com/ethereum/go-ethereum/internal/testrand"
 )
 
+// TailGroup is the tail group used by tables created in this test suite. The
+// store factory passed to TestAncientSuite must wire its tables to this group
+// so that the suite can query the freezer's tail consistently.
+const TailGroup = "test"
+
 // TestAncientSuite runs a suite of tests against an ancient database
 // implementation.
 func TestAncientSuite(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
@@ -58,11 +63,11 @@ func basicRead(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
 	}); err != nil {
 		t.Fatalf("Failed to write ancient data %v", err)
 	}
-	db.TruncateTail(10)
+	db.TruncateTail(TailGroup, 10)
 	db.TruncateHead(90)
 
 	// Test basic tail and head retrievals
-	tail, err := db.Tail()
+	tail, err := db.Tail(TailGroup)
 	if err != nil || tail != 10 {
 		t.Fatal("Failed to retrieve tail")
 	}
@@ -123,7 +128,7 @@ func batchRead(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
 	}); err != nil {
 		t.Fatalf("Failed to write ancient data %v", err)
 	}
-	db.TruncateTail(10)
+	db.TruncateTail(TailGroup, 10)
 	db.TruncateHead(90)
 
 	// Test the items in range should be reachable
@@ -262,12 +267,12 @@ func basicWrite(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
 	}
 
 	// Write should work after truncating from tail but over the head
-	db.TruncateTail(200)
+	db.TruncateTail(TailGroup, 200)
 	head, err := db.Ancients()
 	if err != nil {
 		t.Fatalf("Failed to retrieve head ancients %v", err)
 	}
-	tail, err := db.Tail()
+	tail, err := db.Tail(TailGroup)
 	if err != nil {
 		t.Fatalf("Failed to retrieve tail ancients %v", err)
 	}
@@ -293,7 +298,7 @@ func basicWrite(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
 	if err != nil {
 		t.Fatalf("Failed to retrieve head ancients %v", err)
 	}
-	tail, err = db.Tail()
+	tail, err = db.Tail(TailGroup)
 	if err != nil {
 		t.Fatalf("Failed to retrieve tail ancients %v", err)
 	}
@@ -351,7 +356,7 @@ func TestResettableAncientSuite(t *testing.T, newFn func(kinds []string) ethdb.R
 		}); err != nil {
 			t.Fatalf("Failed to write ancient data %v", err)
 		}
-		db.TruncateTail(10)
+		db.TruncateTail(TailGroup, 10)
 		db.TruncateHead(90)
 
 		// Ancient write should work after resetting

core/rawdb/chain_freezer.go

@@ -45,9 +45,7 @@ const (
 // key-value database to flat files for saving space on live database.
 type chainFreezer struct {
 	ancients ethdb.AncientStore // Ancient store for storing cold chain segment
-
-	// Optional Era database used as a backup for the pruned chain.
-	eradb *eradb.Store
+	eradb    *eradb.Store       // Optional Era database used as a backup for the pruned chain
 
 	quit    chan struct{}
 	wg      sync.WaitGroup
@@ -327,6 +325,16 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
 			if len(receipts) == 0 {
 				return fmt.Errorf("block receipts missing, can't freeze block %d", number)
 			}
+			// An empty block access list is allowed and may occur in multiple
+			// scenarios, such as:
+			//   - pre-Amsterdam blocks
+			//   - post-Amsterdam blocks with the BAL absent (e.g. pruned by network)
+			//   - post-Amsterdam blocks with an explicitly empty BAL
+			//
+			// In these cases, a nil entry will be stored in the BAL table as the
+			// absence placeholder.
+			bals := ReadAccessListRLP(nfdb, hash, number)
+
 			// Write to the batch.
 			if err := op.AppendRaw(ChainFreezerHashTable, number, hash[:]); err != nil {
 				return fmt.Errorf("can't write hash to Freezer: %v", err)
@@ -340,6 +348,9 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
 			if err := op.AppendRaw(ChainFreezerReceiptTable, number, receipts); err != nil {
 				return fmt.Errorf("can't write receipts to Freezer: %v", err)
 			}
+			if err := op.AppendRaw(ChainFreezerBALTable, number, bals); err != nil {
+				return fmt.Errorf("can't write bals to Freezer: %v", err)
+			}
 			hashes = append(hashes, hash)
 		}
 		return nil
@@ -354,7 +365,11 @@ func (f *chainFreezer) Ancient(kind string, number uint64) ([]byte, error) {
 	if kind == ChainFreezerHeaderTable || kind == ChainFreezerHashTable {
 		return f.ancients.Ancient(kind, number)
 	}
-	tail, err := f.ancients.Tail()
+	group, err := tableTailGroup(kind)
+	if err != nil {
+		return nil, err
+	}
+	tail, err := f.ancients.Tail(group)
 	if err != nil {
 		return nil, err
 	}
@@ -371,10 +386,20 @@ func (f *chainFreezer) Ancient(kind string, number uint64) ([]byte, error) {
 		return f.eradb.GetRawBody(number)
 	case ChainFreezerReceiptTable:
 		return f.eradb.GetRawReceipts(number)
+	case ChainFreezerBALTable:
+		return nil, errOutOfBounds
 	}
 	return nil, errUnknownTable
 }
 
+// tableTailGroup returns the tail group identifier for a chain freezer table.
+func tableTailGroup(kind string) (string, error) {
+	if cfg, ok := chainFreezerTableConfigs[kind]; ok {
+		return cfg.tailGroup, nil
+	}
+	return "", errUnknownTable
+}
+
 // ReadAncients executes an operation while preventing mutations to the freezer,
 // i.e. if fn performs multiple reads, they will be consistent with each other.
 func (f *chainFreezer) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
@@ -391,8 +416,8 @@ func (f *chainFreezer) Ancients() (uint64, error) {
 	return f.ancients.Ancients()
 }
 
-func (f *chainFreezer) Tail() (uint64, error) {
-	return f.ancients.Tail()
+func (f *chainFreezer) Tail(group string) (uint64, error) {
+	return f.ancients.Tail(group)
 }
 
 func (f *chainFreezer) AncientSize(kind string) (uint64, error) {
@@ -415,8 +440,8 @@ func (f *chainFreezer) TruncateHead(items uint64) (uint64, error) {
 	return f.ancients.TruncateHead(items)
 }
 
-func (f *chainFreezer) TruncateTail(items uint64) (uint64, error) {
-	return f.ancients.TruncateTail(items)
+func (f *chainFreezer) TruncateTail(group string, items uint64) (uint64, error) {
+	return f.ancients.TruncateTail(group, items)
 }
 
 func (f *chainFreezer) SyncAncient() error {

core/rawdb/database.go

@@ -135,7 +135,7 @@ func (db *nofreezedb) Ancients() (uint64, error) {
 }
 
 // Tail returns an error as we don't have a backing chain freezer.
-func (db *nofreezedb) Tail() (uint64, error) {
+func (db *nofreezedb) Tail(group string) (uint64, error) {
 	return 0, errNotSupported
 }
 
@@ -155,7 +155,7 @@ func (db *nofreezedb) TruncateHead(items uint64) (uint64, error) {
 }
 
 // TruncateTail returns an error as we don't have a backing chain freezer.
-func (db *nofreezedb) TruncateTail(items uint64) (uint64, error) {
+func (db *nofreezedb) TruncateTail(group string, items uint64) (uint64, error) {
 	return 0, errNotSupported
 }
 
@@ -585,6 +585,8 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 				}
 
 			// Metadata keys
+			case bytes.HasPrefix(key, generateTriePartitionDonePrefix) && len(key) == len(generateTriePartitionDonePrefix)+1:
+				metadata.add(size)
 			case slices.ContainsFunc(knownMetadataKeys, func(x []byte) bool { return bytes.Equal(x, key) }):
 				metadata.add(size)
 
@@ -680,12 +682,12 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		return err
 	}
 	for _, ancient := range ancients {
-		for _, table := range ancient.sizes {
+		for _, table := range ancient.tables {
 			stats = append(stats, []string{
 				fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)),
 				strings.Title(table.name),
 				table.size.String(),
-				fmt.Sprintf("%d", ancient.count),
+				fmt.Sprintf("%d", table.count),
 			})
 		}
 		total.Add(uint64(ancient.size()))

core/rawdb/eradb/eradb.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Lesser General Public License
 // along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
 
-// Package eradb implements a history backend using era1 and erae files.
+// Package eradb implements a history backend using era1 and ere files.
 package eradb
 
 import (
@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"io/fs"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common/lru"
@@ -38,7 +39,7 @@ const openFileLimit = 64
 
 var errClosed = errors.New("era store is closed")
 
-// Store manages read access to a directory of era1 and erae files.
+// Store manages read access to a directory of era1 and ere files.
 // The getter methods are thread-safe.
 type Store struct {
 	datadir string
@@ -54,7 +55,7 @@ type Store struct {
 type fileCacheEntry struct {
 	refcount int           // reference count. This is protected by Store.mu!
 	opened   chan struct{} // signals opening of file has completed
-	file     era.Era       // the file (era1 or erae)
+	file     era.Era       // the file (era1 or ere)
 	err      error         // error from opening the file
 }
 
@@ -302,8 +303,8 @@ func (db *Store) fileFailedToOpen(epoch uint64, entry *fileCacheEntry, err error
 
 func (db *Store) openEraFile(epoch uint64) (era.Era, error) {
 	// File name scheme is <network>-<epoch>-<root>.<ext>
-	// Try era1 first, then erae.
-	for _, ext := range []string{"era1", "erae"} {
+	// Try era1 first, then ere.
+	for _, ext := range []string{"era1", "ere"} {
 		glob := fmt.Sprintf("*-%05d-*.%s", epoch, ext)
 		matches, err := filepath.Glob(filepath.Join(db.datadir, glob))
 		if err != nil {
@@ -320,7 +321,12 @@ func (db *Store) openEraFile(epoch uint64) (era.Era, error) {
 		switch ext {
 		case "era1":
 			e, err = onedb.Open(filename)
-		case "erae":
+		case "ere":
+			// The era store serves receipts via RPC. Reject noreceipts
+			// profiles to avoid silently returning empty receipt data.
+			if strings.Contains(filepath.Base(filename), "-noreceipts") {
+				return nil, fmt.Errorf("era store does not support noreceipts profile: %s", filepath.Base(filename))
+			}
 			e, err = execdb.Open(filename)
 		}
 		if err != nil {
@@ -365,9 +371,9 @@ func (entry *fileCacheEntry) derefAndClose(epoch uint64) (closed bool) {
 
 	closeErr := entry.file.Close()
 	if closeErr == nil {
-		log.Debug("Closed era1 file", "epoch", epoch)
+		log.Debug("Closed era file", "epoch", epoch)
 	} else {
-		log.Warn("Error closing era1 file", "epoch", epoch, "err", closeErr)
+		log.Warn("Error closing era file", "epoch", epoch, "err", closeErr)
 	}
 	return true
 }

core/rawdb/eradb/eradb_test.go

@@ -17,6 +17,8 @@
 package eradb
 
 import (
+	"os"
+	"path/filepath"
 	"sync"
 	"testing"
 
@@ -48,6 +50,32 @@ func TestEraDatabase(t *testing.T) {
 	assert.Equal(t, 3, len(receipts), "receipts length mismatch")
 }
 
+func TestEraStoreRejectsNoReceiptsProfile(t *testing.T) {
+	dir := t.TempDir()
+	stubName := "mainnet-00000-deadbeef-noreceipts.ere"
+	stubPath := filepath.Join(dir, stubName)
+
+	// Write a non-empty stub so the glob finds the file. Contents don't matter
+	// because the noreceipts check fires before execdb.Open is called.
+	err := os.WriteFile(stubPath, []byte("stub"), 0644)
+	require.NoError(t, err)
+
+	db, err := New(dir)
+	require.NoError(t, err)
+	defer db.Close()
+
+	// Any block in epoch 0 should trigger the same rejection.
+	const block = uint64(0)
+
+	_, err = db.GetRawBody(block)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "era store does not support noreceipts profile")
+
+	_, err = db.GetRawReceipts(block)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "era store does not support noreceipts profile")
+}
+
 func TestEraDatabaseConcurrentOpen(t *testing.T) {
 	db, err := New("testdata")
 	require.NoError(t, err)

core/rawdb/freezer.go

@@ -59,8 +59,8 @@ const freezerTableSize = 2 * 1000 * 1000 * 1000
 // - The in-order data ensures that disk reads are always optimized.
 type Freezer struct {
 	datadir string
-	head    atomic.Uint64 // Number of items stored (including items removed from tail)
-	tail    atomic.Uint64 // Number of the first stored item in the freezer
+	head    atomic.Uint64             // Number of items stored (including items removed from tail)
+	tails   map[string]*atomic.Uint64 // Per-group tail cache, keyed by tail group name
 
 	// This lock synchronizes writers and the truncate operation, as well as
 	// the "atomic" (batched) read operations.
@@ -77,8 +77,8 @@ type Freezer struct {
 // data according to the given parameters.
 //
 // The 'tables' argument defines the freezer tables and their configuration.
-// Each value is a freezerTableConfig specifying whether snappy compression is
-// disabled (noSnappy) and whether the table is prunable (prunable).
+// Each value is a freezerTableConfig describing whether Snappy compression
+// is disabled (noSnappy) and which tail group the table belongs to.
 func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]freezerTableConfig) (*Freezer, error) {
 	// Create the initial freezer object
 	var (
@@ -118,6 +118,7 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
 		datadir:      datadir,
 		readonly:     readonly,
 		tables:       make(map[string]*freezerTable),
+		tails:        make(map[string]*atomic.Uint64),
 		instanceLock: lock,
 	}
 
@@ -216,9 +217,19 @@ func (f *Freezer) Ancients() (uint64, error) {
 	return f.head.Load(), nil
 }
 
-// Tail returns the number of first stored item in the freezer.
-func (f *Freezer) Tail() (uint64, error) {
-	return f.tail.Load(), nil
+// Tail returns the lowest accessible item index for the given tail group.
+// All tables sharing this group agree on the tail; an empty group name
+// refers to non-prunable tables and always returns 0. Unknown groups return
+// an error.
+func (f *Freezer) Tail(group string) (uint64, error) {
+	if group == "" {
+		return 0, nil
+	}
+	tail, ok := f.tails[group]
+	if !ok {
+		return 0, fmt.Errorf("unknown tail group: %q", group)
+	}
+	return tail.Load(), nil
 }
 
 // AncientSize returns the ancient size of the specified category.
@@ -299,33 +310,43 @@ func (f *Freezer) TruncateHead(items uint64) (uint64, error) {
 	return oitems, nil
 }
 
-// TruncateTail discards all data below the specified threshold. Note that only
-// 'prunable' tables will be truncated.
-func (f *Freezer) TruncateTail(tail uint64) (uint64, error) {
+// TruncateTail discards all data below the specified threshold across every
+// table that belongs to the named tail group. Tables that are already past
+// the threshold are left untouched. The previous tail of the group is
+// returned. An empty group name or an unknown group name returns an error.
+func (f *Freezer) TruncateTail(group string, tail uint64) (uint64, error) {
 	if f.readonly {
 		return 0, errReadOnly
 	}
+	if group == "" {
+		return 0, errors.New("empty tail group")
+	}
+	cached, ok := f.tails[group]
+	if !ok {
+		return 0, fmt.Errorf("unknown tail group: %q", group)
+	}
 	f.writeLock.Lock()
 	defer f.writeLock.Unlock()
 
-	old := f.tail.Load()
-	if old >= tail {
-		return old, nil
+	prev := cached.Load()
+	if prev >= tail {
+		return prev, nil
 	}
 	for _, table := range f.tables {
-		if table.config.prunable {
-			if err := table.truncateTail(tail); err != nil {
-				return 0, err
-			}
+		if table.config.tailGroup != group {
+			continue
+		}
+		if err := table.truncateTail(tail); err != nil {
+			return 0, err
 		}
 	}
-	f.tail.Store(tail)
+	cached.Store(tail)
 
-	// Update the head if the requested tail exceeds the current head
+	// Update the head if the requested tail exceeds the current head.
 	if f.head.Load() < tail {
 		f.head.Store(tail)
 	}
-	return old, nil
+	return prev, nil
 }
 
 // SyncAncient flushes all data tables to disk.
@@ -342,84 +363,133 @@ func (f *Freezer) SyncAncient() error {
 	return nil
 }
 
-// validate checks that every table has the same boundary.
-// Used instead of `repair` in readonly mode.
+// validate checks that every table has the same head and that tables sharing
+// a tail group also share a tail. Used instead of `repair` in readonly mode.
 func (f *Freezer) validate() error {
 	if len(f.tables) == 0 {
 		return nil
 	}
 	var (
-		head       uint64
-		prunedTail *uint64
+		head    uint64
+		headSet bool
+		tails   = make(map[string]uint64)
 	)
-	// get any head value
-	for _, table := range f.tables {
-		head = table.items.Load()
-		break
-	}
 	for kind, table := range f.tables {
-		// all tables have to have the same head
-		if head != table.items.Load() {
-			return fmt.Errorf("freezer table %s has a differing head: %d != %d", kind, table.items.Load(), head)
+		// A freshly added table is empty and has not yet been aligned to the
+		// common head, skip the error here.
+		//
+		// Tradeoff:
+		// It loosens corruption detection slightly: a table that lost its data
+		// and now reports items == 0 would be treated as "freshly added" rather
+		// than flagged. It's the tradeoff we accept.
+		items := table.items.Load()
+		if items == 0 {
+			continue
 		}
-		if !table.config.prunable {
-			// non-prunable tables have to start at 0
+		// Validate the table head
+		if !headSet {
+			head = items
+			headSet = true
+		} else if items != head {
+			return fmt.Errorf("freezer table %s has a differing head: %d != %d", kind, items, head)
+		}
+		// Validate the table tail
+		if table.config.tailGroup == "" {
 			if table.itemHidden.Load() != 0 {
 				return fmt.Errorf("non-prunable freezer table '%s' has a non-zero tail: %d", kind, table.itemHidden.Load())
 			}
+			continue
+		}
+		hidden := table.itemHidden.Load()
+		if t, ok := tails[table.config.tailGroup]; ok {
+			if t != hidden {
+				return fmt.Errorf("freezer table %s has differing tail in group %q: %d != %d", kind, table.config.tailGroup, hidden, t)
+			}
 		} else {
-			// prunable tables have to have the same length
-			if prunedTail == nil {
-				tmp := table.itemHidden.Load()
-				prunedTail = &tmp
-			}
-			if *prunedTail != table.itemHidden.Load() {
-				return fmt.Errorf("freezer table %s has differing tail: %d != %d", kind, table.itemHidden.Load(), *prunedTail)
-			}
+			tails[table.config.tailGroup] = hidden
 		}
 	}
-
-	if prunedTail == nil {
-		tmp := uint64(0)
-		prunedTail = &tmp
-	}
-
 	f.head.Store(head)
-	f.tail.Store(*prunedTail)
+
+	for group, tail := range tails {
+		counter := new(atomic.Uint64)
+		counter.Store(tail)
+		f.tails[group] = counter
+	}
 	return nil
 }
 
-// repair truncates all data tables to the same length.
+// repair brings every table into a consistent state. The common head is taken
+// as the minimum item count among non-empty tables; freshly added empty tables
+// are fast-forwarded to that head via tail truncation. Within each tail group
+// the maximum tail wins, and prunable tables are truncated to it.
 func (f *Freezer) repair() error {
+	// Determine the common head from non-empty tables. Empty tables are
+	// excluded so that a freshly added table cannot drag the existing head
+	// down to zero on first cold-start.
 	var (
-		head       = uint64(math.MaxUint64)
-		prunedTail = uint64(0)
+		hasNonEmpty bool
+		head        uint64 = math.MaxUint64
 	)
-	// get the minimal head and the maximum tail
 	for _, table := range f.tables {
-		head = min(head, table.items.Load())
-		prunedTail = max(prunedTail, table.itemHidden.Load())
+		if table.items.Load() == 0 {
+			continue
+		}
+		if items := table.items.Load(); items < head {
+			head = items
+		}
+		hasNonEmpty = true
 	}
-	// apply the pruning
-	for kind, table := range f.tables {
-		// all tables need to have the same head
+	if !hasNonEmpty {
+		head = 0
+	}
+	// Align newly added empty tables to the common head. truncateTail
+	// internally calls resetTo when the requested tail exceeds the current
+	// head, which is exactly what we need here.
+	if head > 0 {
+		for _, table := range f.tables {
+			if table.items.Load() == 0 {
+				if err := table.truncateTail(head); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	// Truncate every table to the common head.
+	for _, table := range f.tables {
 		if err := table.truncateHead(head); err != nil {
 			return err
 		}
-		if !table.config.prunable {
-			// non-prunable tables have to start at 0
+	}
+	// Per-group tail alignment: take the maximum tail in each group and apply
+	// it to all members. Non-prunable tables must remain at tail 0.
+	tails := make(map[string]uint64)
+	for kind, table := range f.tables {
+		if table.config.tailGroup == "" {
 			if table.itemHidden.Load() != 0 {
 				panic(fmt.Sprintf("non-prunable freezer table %s has non-zero tail: %v", kind, table.itemHidden.Load()))
 			}
-		} else {
-			// prunable tables have to have the same length
-			if err := table.truncateTail(prunedTail); err != nil {
-				return err
-			}
+			continue
+		}
+		hidden := table.itemHidden.Load()
+		if t, ok := tails[table.config.tailGroup]; !ok || hidden > t {
+			tails[table.config.tailGroup] = hidden
+		}
+	}
+	for _, table := range f.tables {
+		if table.config.tailGroup == "" {
+			continue
+		}
+		if err := table.truncateTail(tails[table.config.tailGroup]); err != nil {
+			return err
 		}
 	}
-
 	f.head.Store(head)
-	f.tail.Store(prunedTail)
+
+	for group, tail := range tails {
+		counter := new(atomic.Uint64)
+		counter.Store(tail)
+		f.tails[group] = counter
+	}
 	return nil
 }

core/rawdb/freezer_memory.go

@@ -228,7 +228,7 @@ func (b *memoryBatch) commit(freezer *MemoryFreezer) (items uint64, writeSize in
 // interface and can be used along with ephemeral key-value store.
 type MemoryFreezer struct {
 	items      uint64                  // Number of items stored
-	tail       uint64                  // Number of the first stored item in the freezer
+	tails      map[string]uint64       // Per-group tail cache; access serialized by lock
 	readonly   bool                    // Flag if the freezer is only for reading
 	lock       sync.RWMutex            // Lock to protect fields
 	tables     map[string]*memoryTable // Tables for storing everything
@@ -237,14 +237,21 @@ type MemoryFreezer struct {
 
 // NewMemoryFreezer initializes an in-memory freezer instance.
 func NewMemoryFreezer(readonly bool, tableName map[string]freezerTableConfig) *MemoryFreezer {
-	tables := make(map[string]*memoryTable)
+	var (
+		tables = make(map[string]*memoryTable)
+		tails  = make(map[string]uint64)
+	)
 	for name, cfg := range tableName {
 		tables[name] = newMemoryTable(name, cfg)
+		if cfg.tailGroup != "" {
+			tails[cfg.tailGroup] = 0
+		}
 	}
 	return &MemoryFreezer{
 		writeBatch: newMemoryBatch(),
 		readonly:   readonly,
 		tables:     tables,
+		tails:      tails,
 	}
 }
 
@@ -289,13 +296,21 @@ func (f *MemoryFreezer) Ancients() (uint64, error) {
 	return f.items, nil
 }
 
-// Tail returns the number of first stored item in the freezer.
-// This number can also be interpreted as the total deleted item numbers.
-func (f *MemoryFreezer) Tail() (uint64, error) {
+// Tail returns the lowest accessible item index for the given tail group.
+// All tables sharing the group agree on the tail; an empty group name
+// refers to non-prunable tables and always returns 0.
+func (f *MemoryFreezer) Tail(group string) (uint64, error) {
 	f.lock.RLock()
 	defer f.lock.RUnlock()
 
-	return f.tail, nil
+	if group == "" {
+		return 0, nil
+	}
+	tail, ok := f.tails[group]
+	if !ok {
+		return 0, fmt.Errorf("unknown tail group: %q", group)
+	}
+	return tail, nil
 }
 
 // AncientSize returns the ancient size of the specified category.
@@ -375,32 +390,39 @@ func (f *MemoryFreezer) TruncateHead(items uint64) (uint64, error) {
 	return old, nil
 }
 
-// TruncateTail discards all data below the provided threshold number.
-// Note this will only truncate 'prunable' tables. Block headers and canonical
-// hashes cannot be truncated at this time.
-func (f *MemoryFreezer) TruncateTail(tail uint64) (uint64, error) {
+// TruncateTail discards all data below the provided threshold across every
+// table that belongs to the named tail group. Tables already past the
+// threshold are left untouched. The previous tail of the group is returned.
+func (f *MemoryFreezer) TruncateTail(group string, tail uint64) (uint64, error) {
 	f.lock.Lock()
 	defer f.lock.Unlock()
 
 	if f.readonly {
 		return 0, errReadOnly
 	}
-	old := f.tail
-	if old >= tail {
-		return old, nil
+	if group == "" {
+		return 0, errors.New("empty tail group")
+	}
+	prev, ok := f.tails[group]
+	if !ok {
+		return 0, fmt.Errorf("unknown tail group: %q", group)
+	}
+	if prev >= tail {
+		return prev, nil
 	}
 	for _, table := range f.tables {
-		if table.config.prunable {
-			if err := table.truncateTail(tail); err != nil {
-				return 0, err
-			}
+		if table.config.tailGroup != group {
+			continue
+		}
+		if err := table.truncateTail(tail); err != nil {
+			return 0, err
 		}
 	}
-	f.tail = tail
+	f.tails[group] = tail
 	if f.items < tail {
 		f.items = tail
 	}
-	return old, nil
+	return prev, nil
 }
 
 // SyncAncient flushes all data tables to disk.
@@ -426,11 +448,16 @@ func (f *MemoryFreezer) Reset() error {
 	defer f.lock.Unlock()
 
 	tables := make(map[string]*memoryTable)
+	tails := make(map[string]uint64)
 	for name, table := range f.tables {
 		tables[name] = newMemoryTable(name, table.config)
+		if table.config.tailGroup != "" {
+			tails[table.config.tailGroup] = 0
+		}
 	}
 	f.tables = tables
-	f.items, f.tail = 0, 0
+	f.tails = tails
+	f.items = 0
 	return nil
 }
 

core/rawdb/freezer_memory_test.go

@@ -28,8 +28,8 @@ func TestMemoryFreezer(t *testing.T) {
 		tables := make(map[string]freezerTableConfig)
 		for _, kind := range kinds {
 			tables[kind] = freezerTableConfig{
-				noSnappy: true,
-				prunable: true,
+				noSnappy:  true,
+				tailGroup: ancienttest.TailGroup,
 			}
 		}
 		return NewMemoryFreezer(false, tables)
@@ -38,8 +38,8 @@ func TestMemoryFreezer(t *testing.T) {
 		tables := make(map[string]freezerTableConfig)
 		for _, kind := range kinds {
 			tables[kind] = freezerTableConfig{
-				noSnappy: true,
-				prunable: true,
+				noSnappy:  true,
+				tailGroup: ancienttest.TailGroup,
 			}
 		}
 		return NewMemoryFreezer(false, tables)

core/rawdb/freezer_resettable.go

@@ -143,12 +143,12 @@ func (f *resettableFreezer) Ancients() (uint64, error) {
 	return f.freezer.Ancients()
 }
 
-// Tail returns the number of first stored item in the freezer.
-func (f *resettableFreezer) Tail() (uint64, error) {
+// Tail returns the lowest accessible item index for the given tail group.
+func (f *resettableFreezer) Tail(group string) (uint64, error) {
 	f.lock.RLock()
 	defer f.lock.RUnlock()
 
-	return f.freezer.Tail()
+	return f.freezer.Tail(group)
 }
 
 // AncientSize returns the ancient size of the specified category.
@@ -185,13 +185,13 @@ func (f *resettableFreezer) TruncateHead(items uint64) (uint64, error) {
 	return f.freezer.TruncateHead(items)
 }
 
-// TruncateTail discards any recent data below the provided threshold number.
-// It returns the previous value
-func (f *resettableFreezer) TruncateTail(tail uint64) (uint64, error) {
+// TruncateTail discards data below the provided threshold for the named tail
+// group. It returns the previous tail of the group.
+func (f *resettableFreezer) TruncateTail(group string, tail uint64) (uint64, error) {
 	f.lock.RLock()
 	defer f.lock.RUnlock()
 
-	return f.freezer.TruncateTail(tail)
+	return f.freezer.TruncateTail(group, tail)
 }
 
 // SyncAncient flushes all data tables to disk.

core/rawdb/freezer_test.go

@@ -371,6 +371,133 @@ func checkAncientCount(t *testing.T, f *Freezer, kind string, n uint64) {
 	}
 }
 
+// TestChainFreezerBALAlignment exercises the new-table alignment path: a chain
+// freezer is first opened with the legacy table set (no BAL), populated with a
+// few blocks and closed. It is then re-opened with the full chain freezer
+// table set (which includes the BAL column). The expectation is that the BAL
+// table is fast-forwarded to the existing head without disturbing the body /
+// receipt tables, and that subsequent writes append cleanly across all tables.
+func TestChainFreezerBALAlignment(t *testing.T) {
+	dir := t.TempDir()
+
+	// Build a "legacy" subset of the chain freezer table set, omitting BAL.
+	legacyTables := make(map[string]freezerTableConfig)
+	for name, cfg := range chainFreezerTableConfigs {
+		if name == ChainFreezerBALTable {
+			continue
+		}
+		legacyTables[name] = cfg
+	}
+
+	// First open: legacy config. Fill in `items` blocks of dummy data.
+	const items = uint64(10)
+	payload := bytes.Repeat([]byte{0xab}, 64)
+
+	f, err := NewFreezer(dir, "", false, 2049, legacyTables)
+	if err != nil {
+		t.Fatalf("can't open legacy freezer: %v", err)
+	}
+	if _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
+		for i := uint64(0); i < items; i++ {
+			if err := op.AppendRaw(ChainFreezerHashTable, i, payload); err != nil {
+				return err
+			}
+			if err := op.AppendRaw(ChainFreezerHeaderTable, i, payload); err != nil {
+				return err
+			}
+			if err := op.AppendRaw(ChainFreezerBodiesTable, i, payload); err != nil {
+				return err
+			}
+			if err := op.AppendRaw(ChainFreezerReceiptTable, i, payload); err != nil {
+				return err
+			}
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("legacy write failed: %v", err)
+	}
+	if got, _ := f.Ancients(); got != items {
+		t.Fatalf("legacy head: got %d, want %d", got, items)
+	}
+	require.NoError(t, f.Close())
+
+	// Re-open with the full chain freezer table set, which now includes BAL.
+	// repair() should detect the empty BAL table and fast-forward it to the
+	// existing head rather than truncating everyone down to zero.
+	f, err = NewFreezer(dir, "", false, 2049, chainFreezerTableConfigs)
+	if err != nil {
+		t.Fatalf("can't re-open freezer with BAL added: %v", err)
+	}
+	defer f.Close()
+
+	// The head must be preserved.
+	if got, _ := f.Ancients(); got != items {
+		t.Fatalf("head after re-open: got %d, want %d", got, items)
+	}
+	// Existing data must still be readable in full.
+	for i := uint64(0); i < items; i++ {
+		for _, kind := range []string{
+			ChainFreezerHashTable, ChainFreezerHeaderTable,
+			ChainFreezerBodiesTable, ChainFreezerReceiptTable,
+		} {
+			got, err := f.Ancient(kind, i)
+			if err != nil {
+				t.Fatalf("read %s[%d]: %v", kind, i, err)
+			}
+			if !bytes.Equal(got, payload) {
+				t.Fatalf("read %s[%d]: payload mismatch", kind, i)
+			}
+		}
+	}
+	// The block-data tail must be unchanged (no spurious tail bump).
+	if tail, err := f.Tail(ChainFreezerBlockDataGroup); err != nil || tail != 0 {
+		t.Fatalf("blockdata tail: got %d (err %v), want 0", tail, err)
+	}
+	// The BAL tail should equal the head — the table is empty but aligned.
+	if tail, err := f.Tail(ChainFreezerBALGroup); err != nil || tail != items {
+		t.Fatalf("BAL tail: got %d (err %v), want %d", tail, err, items)
+	}
+	// Reads to BAL for any pre-alignment block must report out-of-bounds.
+	for i := uint64(0); i < items; i++ {
+		if _, err := f.Ancient(ChainFreezerBALTable, i); err == nil {
+			t.Fatalf("reading BAL[%d] succeeded; want error (out of bounds)", i)
+		}
+	}
+	// A subsequent batch must append uniformly to every table, BAL included.
+	balPayload := []byte("real-bal")
+	if _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
+		i := items
+		if err := op.AppendRaw(ChainFreezerHashTable, i, payload); err != nil {
+			return err
+		}
+		if err := op.AppendRaw(ChainFreezerHeaderTable, i, payload); err != nil {
+			return err
+		}
+		if err := op.AppendRaw(ChainFreezerBodiesTable, i, payload); err != nil {
+			return err
+		}
+		if err := op.AppendRaw(ChainFreezerReceiptTable, i, payload); err != nil {
+			return err
+		}
+		if err := op.AppendRaw(ChainFreezerBALTable, i, balPayload); err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("post-alignment write failed: %v", err)
+	}
+	if got, _ := f.Ancients(); got != items+1 {
+		t.Fatalf("head after post-alignment write: got %d, want %d", got, items+1)
+	}
+	got, err := f.Ancient(ChainFreezerBALTable, items)
+	if err != nil {
+		t.Fatalf("BAL[%d]: %v", items, err)
+	}
+	if !bytes.Equal(got, balPayload) {
+		t.Fatalf("BAL[%d]: got %x, want %x", items, got, balPayload)
+	}
+}
+
 func TestFreezerCloseSync(t *testing.T) {
 	t.Parallel()
 	f, _ := newFreezerForTesting(t, map[string]freezerTableConfig{"a": {noSnappy: true}, "b": {noSnappy: true}})
@@ -398,8 +525,8 @@ func TestFreezerSuite(t *testing.T) {
 		tables := make(map[string]freezerTableConfig)
 		for _, kind := range kinds {
 			tables[kind] = freezerTableConfig{
-				noSnappy: true,
-				prunable: true,
+				noSnappy:  true,
+				tailGroup: ancienttest.TailGroup,
 			}
 		}
 		f, _ := newFreezerForTesting(t, tables)
@@ -409,8 +536,8 @@ func TestFreezerSuite(t *testing.T) {
 		tables := make(map[string]freezerTableConfig)
 		for _, kind := range kinds {
 			tables[kind] = freezerTableConfig{
-				noSnappy: true,
-				prunable: true,
+				noSnappy:  true,
+				tailGroup: ancienttest.TailGroup,
 			}
 		}
 		f, _ := newResettableFreezer(t.TempDir(), "", false, 2048, tables)

core/rawdb/schema.go

@@ -104,6 +104,10 @@ var (
 	// snapSyncStatusFlagKey flags that status of snap sync.
 	snapSyncStatusFlagKey = []byte("SnapSyncStatus")
 
+	// generateTriePartitionDonePrefix stores the subtree root hash of each
+	// triedb.GenerateTrie partition once it finishes.
+	generateTriePartitionDonePrefix = []byte("gtd") // generateTriePartitionDonePrefix + partition byte -> subtree root hash
+
 	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
 	headerPrefix       = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
 	headerTDSuffix     = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td (deprecated)
@@ -465,3 +469,8 @@ func trienodeHistoryIndexBlockKey(addressHash common.Hash, path []byte, blockID
 func transitionStateKey(hash common.Hash) []byte {
 	return append(VerkleTransitionStatePrefix, hash.Bytes()...)
 }
+
+// generateTriePartitionDoneKey = generateTriePartitionDonePrefix + partition (single byte).
+func generateTriePartitionDoneKey(partition byte) []byte {
+	return append(generateTriePartitionDonePrefix, partition)
+}

core/rawdb/table.go

@@ -76,8 +76,8 @@ func (t *table) Ancients() (uint64, error) {
 
 // Tail is a noop passthrough that just forwards the request to the underlying
 // database.
-func (t *table) Tail() (uint64, error) {
-	return t.db.Tail()
+func (t *table) Tail(group string) (uint64, error) {
+	return t.db.Tail(group)
 }
 
 // AncientSize is a noop passthrough that just forwards the request to the underlying
@@ -103,8 +103,8 @@ func (t *table) TruncateHead(items uint64) (uint64, error) {
 
 // TruncateTail is a noop passthrough that just forwards the request to the underlying
 // database.
-func (t *table) TruncateTail(items uint64) (uint64, error) {
-	return t.db.TruncateTail(items)
+func (t *table) TruncateTail(group string, items uint64) (uint64, error) {
+	return t.db.TruncateTail(group, items)
 }
 
 // SyncAncient is a noop passthrough that just forwards the request to the underlying

core/state/database_ubt.go

@@ -27,10 +27,26 @@ import (
 // It provides the same functionality as MPTDatabase but uses unified binary
 // trie for state hashing instead of Merkle Patricia Tries.
 type UBTDatabase struct {
-	triedb *triedb.Database
-	codedb *CodeDB
+	triedb   *triedb.Database
+	codedb   *CodeDB
+	recorder *bintrie.Recorder
 }
 
+// EnableAllocRecording installs an alloc recorder shared across every binary
+// trie opened from this database. The recorder captures account, storage, and
+// code writes keyed by their original (unhashed) addresses, which is required
+// for tooling like evm t8n to render the post-state as a types.GenesisAlloc.
+func (db *UBTDatabase) EnableAllocRecording() *bintrie.Recorder {
+	if db.recorder == nil {
+		db.recorder = bintrie.NewRecorder()
+	}
+	return db.recorder
+}
+
+// AllocRecorder returns the attached recorder, or nil if recording was never
+// enabled on this database.
+func (db *UBTDatabase) AllocRecorder() *bintrie.Recorder { return db.recorder }
+
 // Type returns Binary, indicating this database is backed by a Universal Binary Trie.
 func (db *UBTDatabase) Type() DatabaseType { return TypeUBT }
 
@@ -96,7 +112,14 @@ func (db *UBTDatabase) ReadersWithCacheStats(stateRoot common.Hash) (Reader, Rea
 
 // OpenTrie opens the main account trie at a specific root hash.
 func (db *UBTDatabase) OpenTrie(root common.Hash) (Trie, error) {
-	return bintrie.NewBinaryTrie(root, db.triedb, db.triedb.BinTrieGroupDepth())
+	tr, err := bintrie.NewBinaryTrie(root, db.triedb, db.triedb.BinTrieGroupDepth())
+	if err != nil {
+		return nil, err
+	}
+	if db.recorder != nil {
+		tr.SetRecorder(db.recorder)
+	}
+	return tr, nil
 }
 
 // OpenStorageTrie opens the storage trie of an account. In binary trie mode,

core/state/snapshot/disklayer.go

@@ -38,9 +38,17 @@ type diskLayer struct {
 	root  common.Hash // Root hash of the base snapshot
 	stale bool        // Signals that the layer became stale (state progressed)
 
-	genMarker  []byte                    // Marker for the state that's indexed during initial layer generation
-	genPending chan struct{}             // Notification channel when generation is done (test synchronicity)
-	genAbort   chan chan *generatorStats // Notification channel to abort generating the snapshot in this layer
+	genMarker  []byte        // Marker for the state that's indexed during initial layer generation
+	genPending chan struct{} // Notification channel when generation is done (test synchronicity)
+
+	// Generator lifecycle management:
+	// - [cancel] is closed to request termination (broadcast).
+	// - [done] is closed by the generator goroutine on exit.
+	cancel     chan struct{}
+	done       chan struct{}
+	cancelOnce sync.Once
+
+	genStats *generatorStats // Stats for snapshot generation (generation aborted/finished if non-nil)
 
 	lock sync.RWMutex
 }
@@ -49,6 +57,10 @@ type diskLayer struct {
 // Reset() in order to not leak memory.
 // OBS: It does not invoke Close on the diskdb
 func (dl *diskLayer) Release() error {
+	// Stop any ongoing snapshot generation to prevent it from accessing
+	// the database after it's closed during shutdown
+	dl.stopGeneration()
+
 	if dl.cache != nil {
 		dl.cache.Reset()
 	}
@@ -184,17 +196,27 @@ func (dl *diskLayer) Update(blockHash common.Hash, accounts map[common.Hash][]by
 	return newDiffLayer(dl, blockHash, accounts, storage)
 }
 
-// stopGeneration aborts the state snapshot generation if it is currently running.
+// stopGeneration requests cancellation of any running snapshot generation and
+// blocks until the generator goroutine (if running) has fully terminated.
+//
+// Concurrency guarantees:
+//   - Thread-safe: May be called concurrently from multiple goroutines
+//   - Idempotent: Safe to call multiple times; subsequent calls have no effect
+//   - Blocking: Returns only after the generator goroutine (if any) has exited
+//   - Safe to call at any time, including when no generation is running
+//
+// After return, it is **guaranteed** that:
+//   - The generator goroutine has terminated
+//   - It is safe to proceed with cleanup operations (e.g. closing databases)
 func (dl *diskLayer) stopGeneration() {
-	dl.lock.RLock()
-	generating := dl.genMarker != nil
-	dl.lock.RUnlock()
-	if !generating {
+	cancel := dl.cancel
+	done := dl.done
+	if cancel == nil || done == nil {
 		return
 	}
-	if dl.genAbort != nil {
-		abort := make(chan *generatorStats)
-		dl.genAbort <- abort
-		<-abort
-	}
+
+	dl.cancelOnce.Do(func() {
+		close(cancel)
+	})
+	<-done
 }

core/state/snapshot/generate.go

@@ -50,6 +50,9 @@ var (
 	// errMissingTrie is returned if the target trie is missing while the generation
 	// is running. In this case the generation is aborted and wait the new signal.
 	errMissingTrie = errors.New("missing trie")
+
+	// errAborted is returned when snapshot generation was interrupted/aborted
+	errAborted = errors.New("aborted")
 )
 
 // generateSnapshot regenerates a brand new snapshot based on an existing state
@@ -74,7 +77,8 @@ func generateSnapshot(diskdb ethdb.KeyValueStore, triedb *triedb.Database, cache
 		cache:      fastcache.New(cache * 1024 * 1024),
 		genMarker:  genMarker,
 		genPending: make(chan struct{}),
-		genAbort:   make(chan chan *generatorStats),
+		cancel:     make(chan struct{}),
+		done:       make(chan struct{}),
 	}
 	go base.generate(stats)
 	log.Debug("Start snapshot generation", "root", root)
@@ -467,12 +471,14 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi
 // checkAndFlush checks if an interruption signal is received or the
 // batch size has exceeded the allowance.
 func (dl *diskLayer) checkAndFlush(ctx *generatorContext, current []byte) error {
-	var abort chan *generatorStats
+	aborting := false
 	select {
-	case abort = <-dl.genAbort:
+	case <-dl.cancel:
+		aborting = true
 	default:
 	}
-	if ctx.batch.ValueSize() > ethdb.IdealBatchSize || abort != nil {
+
+	if ctx.batch.ValueSize() > ethdb.IdealBatchSize || aborting {
 		if bytes.Compare(current, dl.genMarker) < 0 {
 			log.Error("Snapshot generator went backwards", "current", fmt.Sprintf("%x", current), "genMarker", fmt.Sprintf("%x", dl.genMarker))
 		}
@@ -490,9 +496,9 @@ func (dl *diskLayer) checkAndFlush(ctx *generatorContext, current []byte) error
 		dl.genMarker = current
 		dl.lock.Unlock()
 
-		if abort != nil {
+		if aborting {
 			ctx.stats.Log("Aborting state snapshot generation", dl.root, current)
-			return newAbortErr(abort) // bubble up an error for interruption
+			return errAborted
 		}
 		// Don't hold the iterators too long, release them to let compactor works
 		ctx.reopenIterator(snapAccount)
@@ -648,10 +654,11 @@ func generateAccounts(ctx *generatorContext, dl *diskLayer, accMarker []byte) er
 // gathering and logging, since the method surfs the blocks as they arrive, often
 // being restarted.
 func (dl *diskLayer) generate(stats *generatorStats) {
-	var (
-		accMarker []byte
-		abort     chan *generatorStats
-	)
+	if dl.done != nil {
+		defer close(dl.done)
+	}
+
+	var accMarker []byte
 	if len(dl.genMarker) > 0 { // []byte{} is the start, use nil for that
 		accMarker = dl.genMarker[:common.HashLength]
 	}
@@ -669,15 +676,11 @@ func (dl *diskLayer) generate(stats *generatorStats) {
 	defer ctx.close()
 
 	if err := generateAccounts(ctx, dl, accMarker); err != nil {
-		// Extract the received interruption signal if exists
-		if aerr, ok := err.(*abortErr); ok {
-			abort = aerr.abort
+		// Check if error was due to abort
+		if err == errAborted {
+			stats.Log("Aborting state snapshot generation", dl.root, dl.genMarker)
 		}
-		// Aborted by internal error, wait the signal
-		if abort == nil {
-			abort = <-dl.genAbort
-		}
-		abort <- stats
+		dl.genStats = stats
 		return
 	}
 	// Snapshot fully generated, set the marker to nil.
@@ -686,9 +689,7 @@ func (dl *diskLayer) generate(stats *generatorStats) {
 	journalProgress(ctx.batch, nil, stats)
 	if err := ctx.batch.Write(); err != nil {
 		log.Error("Failed to flush batch", "err", err)
-
-		abort = <-dl.genAbort
-		abort <- stats
+		dl.genStats = stats
 		return
 	}
 	ctx.batch.Reset()
@@ -698,12 +699,9 @@ func (dl *diskLayer) generate(stats *generatorStats) {
 
 	dl.lock.Lock()
 	dl.genMarker = nil
+	dl.genStats = stats
 	close(dl.genPending)
 	dl.lock.Unlock()
-
-	// Someone will be looking for us, wait it out
-	abort = <-dl.genAbort
-	abort <- nil
 }
 
 // increaseKey increase the input key by one bit. Return nil if the entire
@@ -717,17 +715,3 @@ func increaseKey(key []byte) []byte {
 	}
 	return nil
 }
-
-// abortErr wraps an interruption signal received to represent the
-// generation is aborted by external processes.
-type abortErr struct {
-	abort chan *generatorStats
-}
-
-func newAbortErr(abort chan *generatorStats) error {
-	return &abortErr{abort: abort}
-}
-
-func (err *abortErr) Error() string {
-	return "aborted"
-}

core/state/snapshot/generate_test.go

@@ -35,6 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/triedb/hashdb"
 	"github.com/ethereum/go-ethereum/triedb/pathdb"
 	"github.com/holiman/uint256"
+	"go.uber.org/goleak"
 )
 
 func hashData(input []byte) common.Hash {
@@ -74,10 +75,10 @@ func testGeneration(t *testing.T, scheme string) {
 	}
 	checkSnapRoot(t, snap, root)
 
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // Tests that snapshot generation with existent flat state.
@@ -115,10 +116,10 @@ func testGenerateExistentState(t *testing.T, scheme string) {
 	}
 	checkSnapRoot(t, snap, root)
 
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) {
@@ -351,10 +352,10 @@ func testGenerateExistentStateWithWrongStorage(t *testing.T, scheme string) {
 		t.Errorf("Snapshot generation failed")
 	}
 	checkSnapRoot(t, snap, root)
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // Tests that snapshot generation with existent flat state, where the flat state
@@ -414,10 +415,10 @@ func testGenerateExistentStateWithWrongAccounts(t *testing.T, scheme string) {
 	}
 	checkSnapRoot(t, snap, root)
 
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // Tests that snapshot generation errors out correctly in case of a missing trie
@@ -454,10 +455,10 @@ func testGenerateCorruptAccountTrie(t *testing.T, scheme string) {
 	case <-time.After(time.Second):
 		// Not generated fast enough, hopefully blocked inside on missing trie node fail
 	}
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // Tests that snapshot generation errors out correctly in case of a missing root
@@ -498,10 +499,10 @@ func testGenerateMissingStorageTrie(t *testing.T, scheme string) {
 	case <-time.After(time.Second):
 		// Not generated fast enough, hopefully blocked inside on missing trie node fail
 	}
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // Tests that snapshot generation errors out correctly in case of a missing trie
@@ -540,10 +541,10 @@ func testGenerateCorruptStorageTrie(t *testing.T, scheme string) {
 	case <-time.After(time.Second):
 		// Not generated fast enough, hopefully blocked inside on missing trie node fail
 	}
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // Tests that snapshot generation when an extra account with storage exists in the snap state.
@@ -605,10 +606,10 @@ func testGenerateWithExtraAccounts(t *testing.T, scheme string) {
 	}
 	checkSnapRoot(t, snap, root)
 
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 	// If we now inspect the snap db, there should exist no extraneous storage items
 	if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
 		t.Fatalf("expected slot to be removed, got %v", string(data))
@@ -666,10 +667,10 @@ func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) {
 		t.Errorf("Snapshot generation failed")
 	}
 	checkSnapRoot(t, snap, root)
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // Tests this case
@@ -715,10 +716,10 @@ func testGenerateWithExtraBeforeAndAfter(t *testing.T, scheme string) {
 		t.Errorf("Snapshot generation failed")
 	}
 	checkSnapRoot(t, snap, root)
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // TestGenerateWithMalformedSnapdata tests what happes if we have some junk
@@ -755,10 +756,10 @@ func testGenerateWithMalformedSnapdata(t *testing.T, scheme string) {
 		t.Errorf("Snapshot generation failed")
 	}
 	checkSnapRoot(t, snap, root)
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 	// If we now inspect the snap db, there should exist no extraneous storage items
 	if data := rawdb.ReadStorageSnapshot(helper.diskdb, hashData([]byte("acc-2")), hashData([]byte("b-key-1"))); data != nil {
 		t.Fatalf("expected slot to be removed, got %v", string(data))
@@ -792,10 +793,10 @@ func testGenerateFromEmptySnap(t *testing.T, scheme string) {
 		t.Errorf("Snapshot generation failed")
 	}
 	checkSnapRoot(t, snap, root)
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // Tests that snapshot generation with existent flat state, where the flat state
@@ -843,10 +844,10 @@ func testGenerateWithIncompleteStorage(t *testing.T, scheme string) {
 		t.Errorf("Snapshot generation failed")
 	}
 	checkSnapRoot(t, snap, root)
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 func incKey(key []byte) []byte {
@@ -939,10 +940,10 @@ func testGenerateCompleteSnapshotWithDanglingStorage(t *testing.T, scheme string
 	}
 	checkSnapRoot(t, snap, root)
 
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
 }
 
 // Tests that snapshot generation with dangling storages. Dangling storage means
@@ -976,8 +977,49 @@ func testGenerateBrokenSnapshotWithDanglingStorage(t *testing.T, scheme string)
 	}
 	checkSnapRoot(t, snap, root)
 
-	// Signal abortion to the generator and wait for it to tear down
-	stop := make(chan *generatorStats)
-	snap.genAbort <- stop
-	<-stop
+	// Stop the generator (if still running) and wait for it to exit.
+	if err := snap.Release(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+// TestGenerateGoroutineLeak verifies that Release() tears down the generator
+// goroutine. Even after generation completes, the goroutine parks waiting for
+// an abort signal. If Release() does not stop it, it lingers and can touch the
+// database after it has been closed during shutdown.
+func TestGenerateGoroutineLeak(t *testing.T) {
+	testGenerateGoroutineLeak(t, rawdb.HashScheme)
+	testGenerateGoroutineLeak(t, rawdb.PathScheme)
+}
+
+// generateAndRelease builds a minimal state, runs snapshot generation to
+// completion, and releases the resulting disk layer.
+func generateAndRelease(t *testing.T, scheme string) {
+	t.Helper()
+
+	helper := newHelper(scheme)
+	helper.addTrieAccount("acc-1", &types.StateAccount{Balance: uint256.NewInt(1), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()})
+
+	_, snap := helper.CommitAndGenerate()
+
+	// Wait for generation to run to completion.
+	select {
+	case <-snap.genPending:
+	case <-time.After(3 * time.Second):
+		t.Fatal("snapshot generation did not complete in time")
+	}
+
+	if err := snap.Release(); err != nil {
+		t.Fatalf("Release returned error: %v", err)
+	}
+}
+
+func testGenerateGoroutineLeak(t *testing.T, scheme string) {
+	generateAndRelease(t, scheme)
+
+	// Snapshot the current goroutines now berfore verifying the run
+	// below leaks none of its own.
+	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
+
+	generateAndRelease(t, scheme)
 }

core/state/snapshot/journal.go

@@ -179,7 +179,8 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *triedb.Database, root comm
 	// if the background generation is allowed
 	if !generator.Done && !noBuild {
 		base.genPending = make(chan struct{})
-		base.genAbort = make(chan chan *generatorStats)
+		base.cancel = make(chan struct{})
+		base.done = make(chan struct{})
 
 		var origin uint64
 		if len(generator.Marker) >= 8 {
@@ -199,16 +200,9 @@ func loadSnapshot(diskdb ethdb.KeyValueStore, triedb *triedb.Database, root comm
 // Journal terminates any in-progress snapshot generation, also implicitly pushing
 // the progress into the database.
 func (dl *diskLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) {
-	// If the snapshot is currently being generated, abort it
-	var stats *generatorStats
-	if dl.genAbort != nil {
-		abort := make(chan *generatorStats)
-		dl.genAbort <- abort
+	// If the snapshot is currently being generated, stop it
+	dl.stopGeneration()
 
-		if stats = <-abort; stats != nil {
-			stats.Log("Journalling in-progress snapshot", dl.root, dl.genMarker)
-		}
-	}
 	// Ensure the layer didn't get stale
 	dl.lock.RLock()
 	defer dl.lock.RUnlock()
@@ -216,8 +210,8 @@ func (dl *diskLayer) Journal(buffer *bytes.Buffer) (common.Hash, error) {
 	if dl.stale {
 		return common.Hash{}, ErrSnapshotStale
 	}
-	// Ensure the generator stats is written even if none was ran this cycle
-	journalProgress(dl.diskdb, dl.genMarker, stats)
+	// Ensure the generator marker is written even if none was ran this cycle
+	journalProgress(dl.diskdb, dl.genMarker, dl.genStats)
 
 	log.Debug("Journalled disk layer", "root", dl.root)
 	return dl.root, nil