core/rawdb: retain BAL in bad blocks (#35423)

This PR introduces the functionalities to persist the local-built bad
blocks alongside additional execution details.

cmd/geth/dbcmd.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"encoding/json"
 	"fmt"
 	"math"
 	"os"
@@ -104,6 +105,7 @@ Remove blockchain and state databases`,
 			dbCheckStateContentCmd,
 			dbInspectHistoryCmd,
 			dbPebbleUpgradeCmd,
+			dbImportBadBlocksCmd,
 		},
 	}
 	dbInspectCmd = &cli.Command{
@@ -208,6 +210,17 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "The import command imports the specific chain data from an RLP encoded stream.",
 	}
+	dbImportBadBlocksCmd = &cli.Command{
+		Action:    importBadBlocks,
+		Name:      "import-badblocks",
+		Usage:     "Imports bad blocks into the local bad-block store from a file",
+		ArgsUsage: "<file>",
+		Flags:     slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
+		Description: `The import-badblocks command loads bad blocks into the database's bad-block
+store so they can be inspected locally.
+
+The input is what debug_getBadBlocks writes when given a file argument.`,
+	}
 	dbExportCmd = &cli.Command{
 		Action:      exportChaindata,
 		Name:        "export",
@@ -870,6 +883,85 @@ var chainExporters = map[string]func(db ethdb.Database) utils.ChainDataIterator{
 	},
 }
 
+type badBlockEntry struct {
+	Hash   common.Hash `json:"hash"`
+	RLP    string      `json:"rlp"`
+	Detail string      `json:"detail,omitempty"`
+}
+
+func parseBadBlockDump(blob []byte) ([]badBlockEntry, error) {
+	var entries []badBlockEntry
+	if err := json.Unmarshal(blob, &entries); err != nil {
+		return nil, err
+	}
+	return entries, nil
+}
+
+// importBadBlocks loads bad blocks from a file into the local bad-block store so
+// they can later be inspected.
+func importBadBlocks(ctx *cli.Context) error {
+	if ctx.NArg() != 1 {
+		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, false)
+	defer db.Close()
+
+	path := ctx.Args().Get(0)
+	blob, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("failed to read %q: %v", path, err)
+	}
+	entries, err := parseBadBlockDump(blob)
+	if err != nil {
+		return fmt.Errorf("failed to parse %q as bad-block json: %v", path, err)
+	}
+	if len(entries) == 0 {
+		return fmt.Errorf("no bad blocks found in %q", path)
+	}
+
+	var imported int
+	for i, entry := range entries {
+		if entry.RLP == "" {
+			log.Warn("Skipping bad block without rlp", "index", i, "hash", entry.Hash)
+			continue
+		}
+		raw, err := hexutil.Decode(entry.RLP)
+		if err != nil {
+			log.Warn("Skipping bad block with invalid rlp", "index", i, "hash", entry.Hash, "err", err)
+			continue
+		}
+		var block types.Block
+		if err := rlp.DecodeBytes(raw, &block); err != nil {
+			log.Warn("Skipping bad block that failed to decode", "index", i, "hash", entry.Hash, "err", err)
+			continue
+		}
+		if entry.Hash != (common.Hash{}) && block.Hash() != entry.Hash {
+			log.Warn("Bad block hash mismatch, importing decoded block anyway", "index", i, "want", entry.Hash, "have", block.Hash())
+		}
+		var detail *rawdb.ExecutionDetail
+		if entry.Detail != "" {
+			blob, err := hexutil.Decode(entry.Detail)
+			if err != nil {
+				log.Warn("Skipping bad block with invalid detail", "index", i, "hash", entry.Hash, "err", err)
+				continue
+			}
+			detail = new(rawdb.ExecutionDetail)
+			if err := rlp.DecodeBytes(blob, detail); err != nil {
+				log.Warn("Skipping bad block whose detail failed to decode", "index", i, "hash", entry.Hash, "err", err)
+				continue
+			}
+		}
+		rawdb.WriteBadBlockWithDetails(db, &block, detail)
+		imported++
+		log.Info("Imported bad block", "number", block.NumberU64(), "hash", block.Hash(), "detail", detail != nil)
+	}
+	log.Info("Imported bad blocks", "file", path, "count", imported)
+	return nil
+}
+
 func exportChaindata(ctx *cli.Context) error {
 	if ctx.NArg() < 2 {
 		return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)

core/blockchain.go

@@ -2930,9 +2930,13 @@ func (bc *BlockChain) logForkReadiness(block *types.Block) {
 func summarizeBadBlock(block *types.Block, receipts []*types.Receipt, config *params.ChainConfig, err error) string {
 	var receiptString string
 	for i, receipt := range receipts {
-		receiptString += fmt.Sprintf("\n  %d: cumulative: %v gas: %v contract: %v status: %v tx: %v logs: %v bloom: %x state: %x",
+		logStrings := make([]string, 0, len(receipt.Logs))
+		for _, l := range receipt.Logs {
+			logStrings = append(logStrings, fmt.Sprintf("{address: %v, topics: %v, data: %#x}", l.Address, l.Topics, l.Data))
+		}
+		receiptString += fmt.Sprintf("\n  %d: cumulative: %v gas: %v contract: %v status: %v tx: %v logs: [%s] bloom: %x state: %x",
 			i, receipt.CumulativeGasUsed, receipt.GasUsed, receipt.ContractAddress.Hex(),
-			receipt.Status, receipt.TxHash.Hex(), receipt.Logs, receipt.Bloom, receipt.PostState)
+			receipt.Status, receipt.TxHash.Hex(), strings.Join(logStrings, ", "), receipt.Bloom, receipt.PostState)
 	}
 	version, vcs := version.Info()
 	platform := fmt.Sprintf("%s %s %s %s", version, runtime.Version(), runtime.GOARCH, runtime.GOOS)
@@ -2944,7 +2948,7 @@ func summarizeBadBlock(block *types.Block, receipts []*types.Receipt, config *pa
 Block: %v (%#x)
 Error: %v
 Platform: %v%v
-Chain config: %#v
+Chain config: %v
 Receipts: %v
 ##############################
 `, block.Number(), block.Hash(), err, platform, vcs, config, receiptString)

core/rawdb/accessors_chain.go

@@ -839,10 +839,54 @@ const badBlockToKeep = 10
 type badBlock struct {
 	Header *types.Header
 	Body   *types.Body
+
+	// Detail is optional and trailing, so that bad blocks reported by other
+	// clients, which carry none of it, still decode cleanly. It is populated for
+	// blocks that were built locally but then failed to re-import (a
+	// build-vs-import inconsistency), to aid debugging.
+	Detail *ExecutionDetail `rlp:"optional"`
 }
 
-// ReadBadBlock retrieves the bad block with the corresponding block hash.
-func ReadBadBlock(db ethdb.Reader, hash common.Hash) *types.Block {
+// ExecutionDetail records the execution details when a local block is produced.
+type ExecutionDetail struct {
+	// AccessList is the EIP-7928 block-level access list attached to the block.
+	AccessList *bal.BlockAccessList
+
+	// Receipts are the receipts produced during local block building.
+	Receipts []*types.ReceiptForStorage
+
+	// Reason is the error that made the block fail to re-import.
+	Reason string
+
+	// Reverted holds the transactions that were executed during local block
+	// building but then reverted (excluded from the block), with the index
+	// each was assigned. They allow the build process to be fully replayed.
+	Reverted []*RevertedTx
+}
+
+// RevertedTx records a transaction that was executed during local block building
+// but then reverted (and excluded from the block), along with the index it was
+// assigned when tried.
+type RevertedTx struct {
+	Index uint32
+	Tx    *types.Transaction
+}
+
+// toBlock reconstructs the stored block, re-attaching the access list if present.
+func (b *badBlock) toBlock() *types.Block {
+	block := types.NewBlockWithHeader(b.Header)
+	if b.Body != nil {
+		block = block.WithBody(*b.Body)
+	}
+	if b.Detail != nil && b.Detail.AccessList != nil && len(*b.Detail.AccessList) > 0 {
+		block = block.WithAccessListUnsafe(b.Detail.AccessList)
+	}
+	return block
+}
+
+// readBadBlocks decodes the stored list of bad blocks, returning nil if the list
+// is absent or cannot be decoded.
+func readBadBlocks(db ethdb.Reader) []*badBlock {
 	blob, err := db.Get(badBlockKey)
 	if err != nil {
 		return nil
@@ -851,13 +895,14 @@ func ReadBadBlock(db ethdb.Reader, hash common.Hash) *types.Block {
 	if err := rlp.DecodeBytes(blob, &badBlocks); err != nil {
 		return nil
 	}
-	for _, bad := range badBlocks {
+	return badBlocks
+}
+
+// ReadBadBlock retrieves the bad block with the corresponding block hash.
+func ReadBadBlock(db ethdb.Reader, hash common.Hash) *types.Block {
+	for _, bad := range readBadBlocks(db) {
 		if bad.Header.Hash() == hash {
-			block := types.NewBlockWithHeader(bad.Header)
-			if bad.Body != nil {
-				block = block.WithBody(*bad.Body)
-			}
-			return block
+			return bad.toBlock()
 		}
 	}
 	return nil
@@ -866,28 +911,75 @@ func ReadBadBlock(db ethdb.Reader, hash common.Hash) *types.Block {
 // ReadAllBadBlocks retrieves all the bad blocks in the database.
 // All returned blocks are sorted in reverse order by number.
 func ReadAllBadBlocks(db ethdb.Reader) []*types.Block {
-	blob, err := db.Get(badBlockKey)
-	if err != nil {
-		return nil
-	}
-	var badBlocks []*badBlock
-	if err := rlp.DecodeBytes(blob, &badBlocks); err != nil {
-		return nil
-	}
 	var blocks []*types.Block
-	for _, bad := range badBlocks {
-		block := types.NewBlockWithHeader(bad.Header)
-		if bad.Body != nil {
-			block = block.WithBody(*bad.Body)
-		}
-		blocks = append(blocks, block)
+	for _, bad := range readBadBlocks(db) {
+		blocks = append(blocks, bad.toBlock())
 	}
 	return blocks
 }
 
+// ReadAllBadBlocksWithDetails retrieves all bad blocks together with the raw
+// execution detail recorded for each, in the same order as ReadAllBadBlocks. The
+// detail is nil for blocks that carry none. It is returned unpacked so callers
+// can transport it verbatim, which BadBlockDetails cannot express.
+func ReadAllBadBlocksWithDetails(db ethdb.Reader) ([]*types.Block, []*ExecutionDetail) {
+	var (
+		blocks  []*types.Block
+		details []*ExecutionDetail
+	)
+	for _, bad := range readBadBlocks(db) {
+		blocks = append(blocks, bad.toBlock())
+		details = append(details, bad.Detail)
+	}
+	return blocks, details
+}
+
+// BadBlockDetails carries a bad block together with any locally-built receipts,
+// access list and failure reason recorded alongside it.
+type BadBlockDetails struct {
+	Block    *types.Block   // block as stored (carries the access list if present)
+	Receipts types.Receipts // receipts from local block building, nil if not recorded
+	Reason   string         // reason the block failed to re-import, empty if not recorded
+	Reverted []*RevertedTx  // txs tried-and-reverted during local build, nil if not recorded
+}
+
+// ReadBadBlockWithDetails retrieves a bad block along with the extra debugging
+// information recorded for build-vs-import mismatches. Returns nil if not found.
+func ReadBadBlockWithDetails(db ethdb.Reader, hash common.Hash) *BadBlockDetails {
+	for _, bad := range readBadBlocks(db) {
+		if bad.Header.Hash() == hash {
+			detail := bad.Detail
+			if detail == nil {
+				detail = new(ExecutionDetail)
+			}
+			var receipts types.Receipts
+			if len(detail.Receipts) > 0 {
+				receipts = make(types.Receipts, len(detail.Receipts))
+				for i, r := range detail.Receipts {
+					receipts[i] = (*types.Receipt)(r)
+				}
+			}
+			return &BadBlockDetails{
+				Block:    bad.toBlock(),
+				Receipts: receipts,
+				Reason:   detail.Reason,
+				Reverted: detail.Reverted,
+			}
+		}
+	}
+	return nil
+}
+
 // WriteBadBlock serializes the bad block into the database. If the cumulated
 // bad blocks exceeds the limitation, the oldest will be dropped.
 func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) {
+	WriteBadBlockWithDetails(db, block, nil)
+}
+
+// WriteBadBlockWithDetails serializes a bad block into the database together with
+// the execution detail recorded for it, which may be nil for blocks that carry
+// none.
+func WriteBadBlockWithDetails(db ethdb.KeyValueStore, block *types.Block, detail *ExecutionDetail) {
 	blob, err := db.Get(badBlockKey)
 	if err != nil {
 		log.Warn("Failed to load old bad blocks", "error", err)
@@ -895,7 +987,8 @@ func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) {
 	var badBlocks []*badBlock
 	if len(blob) > 0 {
 		if err := rlp.DecodeBytes(blob, &badBlocks); err != nil {
-			log.Crit("Failed to decode old bad blocks", "error", err)
+			log.Warn("Discarding undecodable bad block history", "error", err)
+			badBlocks = nil
 		}
 	}
 	for _, b := range badBlocks {
@@ -904,9 +997,17 @@ func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) {
 			return
 		}
 	}
+	// A block's access list is not part of its RLP encoding, so the
+	// detail is the only place it survives.
+	if detail == nil && block.AccessList() != nil {
+		detail = &ExecutionDetail{
+			AccessList: block.AccessList(),
+		}
+	}
 	badBlocks = append(badBlocks, &badBlock{
 		Header: block.Header(),
 		Body:   block.Body(),
+		Detail: detail,
 	})
 	slices.SortFunc(badBlocks, func(a, b *badBlock) int {
 		// Note: sorting in descending number order.

eth/api_debug.go

@@ -18,8 +18,10 @@ package eth
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -102,20 +104,40 @@ type BadBlockArgs struct {
 	Hash  common.Hash            `json:"hash"`
 	Block map[string]interface{} `json:"block"`
 	RLP   string                 `json:"rlp"`
+
+	// Detail is the RLP-encoded execution detail recorded for blocks that were
+	// built locally and then failed to re-import: the build-time receipts,
+	// tried-and-reverted transactions and the failure reason. It is empty for
+	// ordinary bad blocks reported by peers.
+	//
+	// It is carried as opaque RLP rather than expanded into JSON so that it
+	// round-trips exactly, which is what lets a bad block be moved to another
+	// node and replayed there with debug_replayBadBlock.
+	Detail string `json:"detail,omitempty"`
 }
 
-// GetBadBlocks returns a list of the last 'bad blocks' that the client has seen on the network
-// and returns them as a JSON list of block hashes.
-func (api *DebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error) {
+// GetBadBlocks returns a list of the last 'bad blocks' that the client has
+// seen on the network and returns them as a JSON list of block hashes.
+//
+// If file is given, the bad block list will be written into the file instead.
+func (api *DebugAPI) GetBadBlocks(ctx context.Context, file *string) ([]*BadBlockArgs, error) {
 	var (
-		blocks  = rawdb.ReadAllBadBlocks(api.eth.chainDb)
-		results = make([]*BadBlockArgs, 0, len(blocks))
+		blocks, details = rawdb.ReadAllBadBlocksWithDetails(api.eth.chainDb)
+		results         = make([]*BadBlockArgs, 0, len(blocks))
 	)
-	for _, block := range blocks {
+	for i, block := range blocks {
 		var (
 			blockRlp  string
 			blockJSON map[string]interface{}
+			detailRlp string
 		)
+		if details[i] != nil {
+			if b, err := rlp.EncodeToBytes(details[i]); err != nil {
+				log.Warn("Failed to encode bad block detail", "hash", block.Hash(), "err", err)
+			} else {
+				detailRlp = hexutil.Encode(b)
+			}
+		}
 		if rlpBytes, err := rlp.EncodeToBytes(block); err != nil {
 			blockRlp = err.Error() // Hacky, but hey, it works
 		} else {
@@ -123,14 +145,38 @@ func (api *DebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, error)
 		}
 		blockJSON = ethapi.RPCMarshalBlock(block, true, true, api.eth.APIBackend.ChainConfig())
 		results = append(results, &BadBlockArgs{
-			Hash:  block.Hash(),
-			RLP:   blockRlp,
-			Block: blockJSON,
+			Hash:   block.Hash(),
+			RLP:    blockRlp,
+			Block:  blockJSON,
+			Detail: detailRlp,
 		})
 	}
+	if file != nil {
+		err := dumpBadBlocks(*file, results)
+		return nil, err
+	}
 	return results, nil
 }
 
+// dumpBadBlocks writes the bad blocks to path as JSON.
+//
+// It refuses to touch an existing file: the path comes from the caller, so
+// clobbering whatever happens to be there would turn this into an
+// arbitrary-write primitive. The file is created exclusively rather than stat-ed
+// first, so that two concurrent calls cannot both decide the path is free.
+func dumpBadBlocks(path string, results []*BadBlockArgs) error {
+	out, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644)
+	if err != nil {
+		if os.IsExist(err) {
+			return errors.New("location would overwrite an existing file")
+		}
+		return err
+	}
+	defer out.Close()
+
+	return json.NewEncoder(out).Encode(results)
+}
+
 // AccountRangeMaxResults is the maximum number of results to be returned per call
 const AccountRangeMaxResults = 256
 

eth/api_debug_replay.go

@@ -0,0 +1,200 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/types/bal"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// ReplayBadBlock re-executes a stored bad block using all the information
+// captured during the block construction, and returns execution details
+// side by side for diffing.
+func (api *DebugAPI) ReplayBadBlock(ctx context.Context, hash common.Hash) (map[string]interface{}, error) {
+	details := rawdb.ReadBadBlockWithDetails(api.eth.chainDb, hash)
+	if details == nil {
+		return nil, fmt.Errorf("bad block %#x not found", hash)
+	}
+	block := details.Block
+
+	parent := api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)
+	if parent == nil {
+		return nil, fmt.Errorf("parent block %#x not found", block.ParentHash())
+	}
+	statedb, release, err := api.eth.stateAtBlock(ctx, parent, nil, false, false)
+	if err != nil {
+		return nil, fmt.Errorf("parent state unavailable: %w", err)
+	}
+	defer release()
+
+	importState := statedb.Copy() // snapshot before the build replay mutates it
+
+	// Build side: same serial execution, with the reverted transactions injected.
+	buildBAL, buildReceipts, buildGas, buildRoot, err := api.replayBuild(ctx, block, statedb, details.Reverted)
+	if err != nil {
+		return nil, fmt.Errorf("build replay: %w", err)
+	}
+	// Import side: identical execution without the reverted transactions.
+	importBAL, importReceipts, importGas, importRoot, err := api.replayBuild(ctx, block, importState, nil)
+	if err != nil {
+		return nil, fmt.Errorf("import replay: %w", err)
+	}
+	buildHash, importHash := buildBAL.Hash(), importBAL.Hash()
+
+	out := map[string]interface{}{
+		"reason":        details.Reason,
+		"revertedCount": len(details.Reverted),
+		"reverted":      details.Reverted,
+
+		// Gas accounting.
+		"headerGasUsed": block.GasUsed(),
+		"buildGasUsed":  buildGas,
+		"importGasUsed": importGas,
+
+		// Block-level access list.
+		"headerBALHash": block.Header().BlockAccessListHash,
+		"buildBALHash":  buildHash,
+		"importBALHash": importHash,
+		"buildBAL":      buildBAL,
+		"importBAL":     importBAL,
+
+		// State root.
+		"headerStateRoot": block.Root(),
+		"buildStateRoot":  buildRoot,
+		"importStateRoot": importRoot,
+
+		// Receipts: the ones recorded at build time, plus both replays.
+		"storedBuildReceipts": details.Receipts,
+		"buildReceipts":       buildReceipts,
+		"importReceipts":      importReceipts,
+	}
+	return out, nil
+}
+
+// replayBuild re-executes a block's construction against the provided state.
+func (api *DebugAPI) replayBuild(ctx context.Context, block *types.Block, statedb *state.StateDB, reverted []*rawdb.RevertedTx) (*bal.BlockAccessList, types.Receipts, uint64, common.Hash, error) {
+	bc := api.eth.blockchain
+	config := bc.Config()
+
+	parent := bc.GetHeader(block.ParentHash(), block.NumberU64()-1)
+	if parent == nil {
+		return nil, nil, 0, common.Hash{}, fmt.Errorf("parent header %#x not found", block.ParentHash())
+	}
+	// Reconstruct the sealing header, resetting the fields accumulated during
+	// execution so the replay starts from the same point as building/import.
+	header := types.CopyHeader(block.Header())
+	header.GasUsed = 0
+	if header.BlobGasUsed != nil {
+		header.BlobGasUsed = new(uint64)
+	}
+	header.RequestsHash = nil
+	header.BlockAccessListHash = nil
+
+	coinbase := block.Coinbase()
+	evm := vm.NewEVM(core.NewEVMBlockContext(header, bc, &coinbase), statedb, config, vm.Config{})
+	defer evm.Release()
+
+	var (
+		signer    = types.MakeSigner(config, header.Number, header.Time)
+		gp        = core.NewGasPool(header.GasLimit)
+		blockAL   = bal.NewConstructionBlockAccessList()
+		blockHash = block.Hash()
+		committed = block.Transactions()
+		receipts  types.Receipts
+		tcount    = 0
+	)
+	// Pre-execution system calls.
+	blockAL.Merge(core.PreExecution(ctx, header.ParentBeaconRoot, parent, config, evm, header.Number, header.Time))
+
+	// Group the reverted transactions by build slot.
+	revBySlot := make(map[int][]*types.Transaction)
+	for _, r := range reverted {
+		revBySlot[int(r.Index)] = append(revBySlot[int(r.Index)], r.Tx)
+	}
+
+	// applyReverted executes a tx that was reverted during building.
+	applyReverted := func(tx *types.Transaction) {
+		msg, err := core.TransactionToMessage(tx, signer, header.BaseFee)
+		if err != nil {
+			return
+		}
+		statedb.SetTxContext(tx.Hash(), tcount, uint32(tcount+1))
+		snap, gpSnap := statedb.Snapshot(), gp.Snapshot()
+		_, _, err = core.ApplyTransactionWithEVM(msg, gp, statedb, header.Number, blockHash, header.Time, tx, evm)
+		if err == nil {
+			txRLP, _ := rlp.EncodeToBytes(tx)
+			log.Warn("Expect the transaction to be failed", "index", tcount, "hash", tx.Hash(), "rlp", txRLP)
+		}
+		statedb.RevertToSnapshot(snap)
+		gp.Set(gpSnap)
+	}
+
+	for k := 0; k <= len(committed); k++ {
+		for _, tx := range revBySlot[k] {
+			applyReverted(tx)
+		}
+		if k == len(committed) {
+			break
+		}
+		tx := committed[k]
+		msg, err := core.TransactionToMessage(tx, signer, header.BaseFee)
+		if err != nil {
+			return nil, nil, 0, common.Hash{}, fmt.Errorf("could not convert tx %d [%v]: %w", k, tx.Hash().Hex(), err)
+		}
+		statedb.SetTxContext(tx.Hash(), tcount, uint32(tcount+1))
+		receipt, txBal, err := core.ApplyTransactionWithEVM(msg, gp, statedb, header.Number, blockHash, header.Time, tx, evm)
+		if err != nil {
+			return nil, nil, 0, common.Hash{}, fmt.Errorf("could not apply committed tx %d [%v]: %w", k, tx.Hash().Hex(), err)
+		}
+		receipts = append(receipts, receipt)
+		if tx.Type() == types.BlobTxType && header.BlobGasUsed != nil {
+			*header.BlobGasUsed += receipt.BlobGasUsed
+		}
+		tcount++
+		blockAL.Merge(txBal)
+	}
+
+	// Post-execution system calls and finalize.
+	var allLogs []*types.Log
+	for _, r := range receipts {
+		allLogs = append(allLogs, r.Logs...)
+	}
+	_, postBal, err := core.PostExecution(ctx, config, header.Number, header.Time, allLogs, evm, uint32(tcount+1))
+	if err != nil {
+		return nil, nil, 0, common.Hash{}, err
+	}
+	blockAL.Merge(postBal)
+
+	body := types.Body{
+		Transactions: committed,
+		Withdrawals:  block.Withdrawals(),
+	}
+	bc.Engine().Finalize(bc, header, statedb, &body, uint32(tcount+1), blockAL)
+	root := statedb.IntermediateRoot(config.IsEIP158(header.Number))
+
+	return blockAL.ToEncodingObj(), receipts, gp.Used(), root, nil
+}

eth/catalyst/api.go

@@ -997,6 +997,30 @@ func (api *ConsensusAPI) newPayload(ctx context.Context, params engine.Executabl
 	if err != nil {
 		log.Warn("NewPayload: inserting block failed", "error", err)
 
+		// If this block was also built locally, its local build succeeded while
+		// re-import now fails.
+		localBlock, localReceipts, revertedTxs, revertedIdx := api.localBlocks.getWithDetails(block.Root())
+		if localBlock != nil {
+			log.Warn("NewPayload: locally-built block failed to import", "number", localBlock.NumberU64(), "hash", localBlock.Hash(), "root", localBlock.Root())
+
+			reverted := make([]*rawdb.RevertedTx, len(revertedTxs))
+			for i, tx := range revertedTxs {
+				reverted[i] = &rawdb.RevertedTx{
+					Index: revertedIdx[i],
+					Tx:    tx,
+				}
+			}
+			receipts := make([]*types.ReceiptForStorage, len(localReceipts))
+			for i, r := range localReceipts {
+				receipts[i] = (*types.ReceiptForStorage)(r)
+			}
+			rawdb.WriteBadBlockWithDetails(api.eth.ChainDb(), localBlock, &rawdb.ExecutionDetail{
+				AccessList: localBlock.AccessList(),
+				Receipts:   receipts,
+				Reason:     err.Error(),
+				Reverted:   reverted,
+			})
+		}
 		api.invalidLock.Lock()
 		api.invalidBlocksHits[block.Hash()] = 1
 		api.invalidTipsets[block.Hash()] = block.Header()

eth/catalyst/queue.go

@@ -91,6 +91,28 @@ func (q *payloadQueue) get(id engine.PayloadID, full bool) *engine.ExecutionPayl
 	return nil
 }
 
+// getWithDetails returns the tracked local payload with the given state root,
+// along with the block construction details.
+//
+// The state root is used as the key because the consensus client may mutate
+// the returned payload externally by modifying fields (such as Extra), which
+// would otherwise change the block hash.
+func (q *payloadQueue) getWithDetails(root common.Hash) (*types.Block, []*types.Receipt, []*types.Transaction, []uint32) {
+	q.lock.RLock()
+	defer q.lock.RUnlock()
+
+	for _, item := range q.payloads {
+		if item == nil {
+			return nil, nil, nil, nil // no more items
+		}
+		block, receipts, revertedTxs, revertedIdx := item.payload.FullBlockAndReceipts()
+		if block != nil && block.Root() == root {
+			return block, receipts, revertedTxs, revertedIdx
+		}
+	}
+	return nil, nil, nil, nil
+}
+
 // has checks if a particular payload is already tracked.
 func (q *payloadQueue) has(id engine.PayloadID) bool {
 	q.lock.RLock()

internal/web3ext/web3ext.go

@@ -413,7 +413,14 @@ web3._extend({
 		new web3._extend.Method({
 			name: 'getBadBlocks',
 			call: 'debug_getBadBlocks',
-			params: 0,
+			params: 1,
+			inputFormatter: [null]
+		}),
+		new web3._extend.Method({
+			name: 'replayBadBlock',
+			call: 'debug_replayBadBlock',
+			params: 1,
+			inputFormatter: [null]
 		}),
 		new web3._extend.Method({
 			name: 'storageRangeAt',

miner/payload_building.go

@@ -80,11 +80,16 @@ func (args *BuildPayloadArgs) Id() engine.PayloadID {
 // the revenue. Therefore, the empty-block here is always available and full-block
 // will be set/updated afterwards.
 type Payload struct {
-	id            engine.PayloadID
-	empty         *types.Block
-	emptyWitness  *stateless.Witness
-	full          *types.Block
-	fullWitness   *stateless.Witness
+	id           engine.PayloadID
+	empty        *types.Block
+	emptyWitness *stateless.Witness
+
+	full            *types.Block
+	fullReceipts    []*types.Receipt
+	fullRevertedTxs []*types.Transaction
+	fullRevertedIdx []uint32
+	fullWitness     *stateless.Witness
+
 	sidecars      []*types.BlobTxSidecar
 	emptyRequests [][]byte
 	requests      [][]byte
@@ -124,13 +129,19 @@ func (payload *Payload) update(r *newPayloadResult, elapsed time.Duration) (resu
 	// fee(apart from the mev revenue) is the only indicator for comparison.
 	if payload.full == nil || r.fees.Cmp(payload.fullFees) > 0 {
 		payload.full = r.block
+		payload.fullReceipts = r.receipts
+		payload.fullRevertedTxs = r.revertedTxs
+		payload.fullRevertedIdx = r.revertedIdx
 		payload.fullFees = r.fees
 		payload.sidecars = r.sidecars
 		payload.requests = r.requests
 		payload.fullWitness = r.witness
 
-		feesInEther := new(big.Float).Quo(new(big.Float).SetInt(r.fees), big.NewFloat(params.Ether))
-		log.Info("Updated payload",
+		var (
+			attrs       []any
+			feesInEther = new(big.Float).Quo(new(big.Float).SetInt(r.fees), big.NewFloat(params.Ether))
+		)
+		attrs = append(attrs,
 			"id", payload.id,
 			"number", r.block.NumberU64(),
 			"hash", r.block.Hash(),
@@ -141,12 +152,26 @@ func (payload *Payload) update(r *newPayloadResult, elapsed time.Duration) (resu
 			"root", r.block.Root(),
 			"elapsed", common.PrettyDuration(elapsed),
 		)
+		if r.block.BlockAccessListHash() != nil {
+			attrs = append(attrs, "balhash", r.block.BlockAccessListHash().Hex())
+		}
+		log.Info("Updated payload", attrs...)
 		result = true
 	}
 	payload.cond.Broadcast() // fire signal for notifying full block
 	return
 }
 
+// FullBlockAndReceipts returns the latest built full block together with the
+// receipts produced during its construction and the transactions that were
+// tried-and-reverted during building.
+func (payload *Payload) FullBlockAndReceipts() (*types.Block, []*types.Receipt, []*types.Transaction, []uint32) {
+	payload.lock.Lock()
+	defer payload.lock.Unlock()
+
+	return payload.full, payload.fullReceipts, payload.fullRevertedTxs, payload.fullRevertedIdx
+}
+
 // Resolve returns the latest built payload and also terminates the background
 // thread for updating payload. It's safe to be called multiple times.
 func (payload *Payload) Resolve() *engine.ExecutionPayloadEnvelope {

miner/worker.go

@@ -76,6 +76,12 @@ type environment struct {
 	blobs    int
 	bal      *bal.ConstructionBlockAccessList
 
+	// revertedTxs and revertedIdx record transactions that were executed during
+	// block building but then reverted (excluded from the block), together with
+	// the index each was tried at.
+	revertedTxs []*types.Transaction
+	revertedIdx []uint32
+
 	witness *stateless.Witness
 }
 
@@ -114,6 +120,11 @@ type newPayloadResult struct {
 	receipts []*types.Receipt       // Receipts collected during construction
 	requests [][]byte               // Consensus layer requests collected during block construction
 	witness  *stateless.Witness     // Witness is an optional stateless proof
+
+	// revertedTxs and revertedIdx record the transactions tried-and-reverted
+	// during construction and the index each was assigned.
+	revertedTxs []*types.Transaction
+	revertedIdx []uint32
 }
 
 // generateParams wraps various settings for generating sealing task.
@@ -237,13 +248,15 @@ func (miner *Miner) generateWork(ctx context.Context, genParam *generateParams,
 		return &newPayloadResult{err: fmt.Errorf("%w: %v", errStateReadFailure, dbErr)}
 	}
 	return &newPayloadResult{
-		block:    block,
-		fees:     totalFees(block, work.receipts),
-		sidecars: work.sidecars,
-		stateDB:  work.state,
-		receipts: work.receipts,
-		requests: requests,
-		witness:  work.witness,
+		block:       block,
+		fees:        totalFees(block, work.receipts),
+		sidecars:    work.sidecars,
+		stateDB:     work.state,
+		receipts:    work.receipts,
+		requests:    requests,
+		witness:     work.witness,
+		revertedTxs: work.revertedTxs,
+		revertedIdx: work.revertedIdx,
 	}
 }
 
@@ -433,6 +446,9 @@ func (miner *Miner) applyTransaction(env *environment, tx *types.Transaction) (*
 	if err != nil {
 		env.state.RevertToSnapshot(snap)
 		env.gasPool.Set(gp)
+
+		env.revertedTxs = append(env.revertedTxs, tx.WithoutBlobTxSidecar())
+		env.revertedIdx = append(env.revertedIdx, uint32(env.tcount))
 		return nil, nil, err
 	}
 	env.header.GasUsed = env.gasPool.Used()

params/config.go

@@ -512,6 +512,14 @@ func (c CliqueConfig) String() string {
 
 // String implements the fmt.Stringer interface, returning a string representation
 // of ChainConfig.
+// GoString implements fmt.GoStringer, so that %#v prints the same readable
+// summary as %v. Without it the default Go-syntax formatting renders every
+// timestamp-based fork as a bare pointer address, which is exactly the part of
+// the config a reader needs when diagnosing a fork-activation problem.
+func (c *ChainConfig) GoString() string {
+	return c.String()
+}
+
 func (c *ChainConfig) String() string {
 	result := fmt.Sprintf("ChainConfig{ChainID: %v", c.ChainID)