Merge branch 'ethereum:master' into gethintegration

.gitignore

@@ -43,3 +43,16 @@ profile.cov
 .vscode
 
 tests/spec-tests/
+
+# binaries
+cmd/abidump/abidump
+cmd/abigen/abigen
+cmd/blsync/blsync
+cmd/clef/clef
+cmd/devp2p/devp2p
+cmd/era/era
+cmd/ethkey/ethkey
+cmd/evm/evm
+cmd/geth/geth
+cmd/rlpdump/rlpdump
+cmd/workload/workload

.travis.yml

@@ -2,12 +2,6 @@ language: go
 go_import_path: github.com/ethereum/go-ethereum
 sudo: false
 jobs:
-  allow_failures:
-    - stage: build
-      os: osx
-      env:
-        - azure-osx
-
   include:
     # This builder create and push the Docker images for all architectures
     - stage: build
@@ -62,23 +56,6 @@ jobs:
         - go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc
         - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
 
-    # This builder does the OSX Azure uploads
-    - stage: build
-      if: type = push
-      os: osx
-      osx_image: xcode14.2
-      go: 1.23.1 # See https://github.com/ethereum/go-ethereum/pull/30478
-      env:
-        - azure-osx
-      git:
-        submodules: false # avoid cloning ethereum/tests
-      script:
-        - ln -sf /Users/travis/gopath/bin/go1.23.1 /usr/local/bin/go # Work around travis go-setup bug
-        - go run build/ci.go install -dlgo
-        - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
-        - go run build/ci.go install -dlgo -arch arm64
-        - go run build/ci.go archive -arch arm64 -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds
-
     # These builders run the tests
     - stage: build
       if: type = push

accounts/abi/abigen/bind_test.go

@@ -1806,7 +1806,9 @@ var bindTests = []struct {
 		[]string{"608060405234801561001057600080fd5b50610113806100206000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806324ec1d3f14602d575b600080fd5b60336035565b005b7fb4b2ff75e30cb4317eaae16dd8a187dd89978df17565104caa6c2797caae27d460405180604001604052806001815260200160028152506040516078919060ba565b60405180910390a1565b6040820160008201516096600085018260ad565b50602082015160a7602085018260ad565b50505050565b60b48160d3565b82525050565b600060408201905060cd60008301846082565b92915050565b600081905091905056fea26469706673582212208823628796125bf9941ce4eda18da1be3cf2931b231708ab848e1bd7151c0c9a64736f6c63430008070033"},
 		[]string{`[{"anonymous":false,"inputs":[{"components":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"uint256","name":"b","type":"uint256"}],"indexed":false,"internalType":"struct Test.MyStruct","name":"s","type":"tuple"}],"name":"StructEvent","type":"event"},{"inputs":[],"name":"TestEvent","outputs":[],"stateMutability":"nonpayable","type":"function"}]`},
 		`
+			"context"
 			"math/big"
+			"time"
 
 			"github.com/ethereum/go-ethereum/accounts/abi/bind"
 			"github.com/ethereum/go-ethereum/accounts/abi/bind/backends"
@@ -1828,12 +1830,23 @@ var bindTests = []struct {
 			}
 			sim.Commit()
 
-			_, err = d.TestEvent(user)
+			tx, err := d.TestEvent(user)
 			if err != nil {
 				t.Fatalf("Failed to call contract %v", err)
 			}
 			sim.Commit()
 
+			// Wait for the transaction to be mined
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+			receipt, err := bind.WaitMined(ctx, sim, tx)
+			if err != nil {
+				t.Fatalf("Failed to wait for tx to be mined: %v", err)
+			}
+			if receipt.Status != types.ReceiptStatusSuccessful {
+				t.Fatal("Transaction failed")
+			}
+
 			it, err := d.FilterStructEvent(nil)
 			if err != nil {
 				t.Fatalf("Failed to filter contract event %v", err)

accounts/abi/bind/v2/base.go

@@ -390,6 +390,7 @@ func (c *BoundContract) estimateGasLimit(opts *TransactOpts, contract *common.Ad
 		GasFeeCap:  gasFeeCap,
 		Value:      value,
 		Data:       input,
+		AccessList: opts.AccessList,
 	}
 	return c.transactor.EstimateGas(ensureContext(opts.Context), msg)
 }

beacon/params/checkpoint_holesky.hex

@@ -0,0 +1 @@
+0xf5606a019f0f1006e9ec2070695045f4334450362a48da4c5965314510e0b4c3

beacon/params/checkpoint_mainnet.hex

@@ -0,0 +1 @@
+0x3c0cb4aa83beded1803d262664ba4392b1023f334d9cca02dc3a6925987ebe91

beacon/params/checkpoint_sepolia.hex

@@ -0,0 +1 @@
+0xa8d56457aa414523d93659aef1f7409bbfb72ad75e94d917c8c0b1baa38153ef

beacon/params/networks.go

@@ -17,14 +17,25 @@
 package params
 
 import (
+	_ "embed"
+
 	"github.com/ethereum/go-ethereum/common"
 )
 
+//go:embed checkpoint_mainnet.hex
+var checkpointMainnet string
+
+//go:embed checkpoint_sepolia.hex
+var checkpointSepolia string
+
+//go:embed checkpoint_holesky.hex
+var checkpointHolesky string
+
 var (
 	MainnetLightConfig = (&ChainConfig{
 		GenesisValidatorsRoot: common.HexToHash("0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95"),
 		GenesisTime:           1606824023,
-		Checkpoint:            common.HexToHash("0x6509b691f4de4f7b083f2784938fd52f0e131675432b3fd85ea549af9aebd3d0"),
+		Checkpoint:            common.HexToHash(checkpointMainnet),
 	}).
 		AddFork("GENESIS", 0, []byte{0, 0, 0, 0}).
 		AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}).
@@ -35,7 +46,7 @@ var (
 	SepoliaLightConfig = (&ChainConfig{
 		GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"),
 		GenesisTime:           1655733600,
-		Checkpoint:            common.HexToHash("0x456e85f5608afab3465a0580bff8572255f6d97af0c5f939e3f7536b5edb2d3f"),
+		Checkpoint:            common.HexToHash(checkpointSepolia),
 	}).
 		AddFork("GENESIS", 0, []byte{144, 0, 0, 105}).
 		AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}).
@@ -47,7 +58,7 @@ var (
 	HoleskyLightConfig = (&ChainConfig{
 		GenesisValidatorsRoot: common.HexToHash("0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1"),
 		GenesisTime:           1695902400,
-		Checkpoint:            common.HexToHash("0x6456a1317f54d4b4f2cb5bc9d153b5af0988fe767ef0609f0236cf29030bcff7"),
+		Checkpoint:            common.HexToHash(checkpointHolesky),
 	}).
 		AddFork("GENESIS", 0, []byte{1, 1, 112, 0}).
 		AddFork("ALTAIR", 0, []byte{2, 1, 112, 0}).

cmd/geth/chaincmd.go

@@ -35,8 +35,11 @@ import (
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/internal/era"
+	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/urfave/cli/v2"
@@ -65,7 +68,7 @@ It expects the genesis file as argument.`,
 		Name:      "dumpgenesis",
 		Usage:     "Dumps genesis block JSON configuration to stdout",
 		ArgsUsage: "",
-		Flags:     append([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags...),
+		Flags:     slices.Concat([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags),
 		Description: `
 The dumpgenesis command prints the genesis configuration of the network preset
 if one is set.  Otherwise it prints the genesis from the datadir.`,
@@ -77,11 +80,11 @@ if one is set.  Otherwise it prints the genesis from the datadir.`,
 		ArgsUsage: "<filename> (<filename 2> ... <filename N>) ",
 		Flags: slices.Concat([]cli.Flag{
 			utils.CacheFlag,
-			utils.SyncModeFlag,
 			utils.GCModeFlag,
 			utils.SnapshotFlag,
 			utils.CacheDatabaseFlag,
 			utils.CacheGCFlag,
+			utils.NoCompactionFlag,
 			utils.MetricsEnabledFlag,
 			utils.MetricsEnabledExpensiveFlag,
 			utils.MetricsHTTPFlag,
@@ -104,7 +107,11 @@ if one is set.  Otherwise it prints the genesis from the datadir.`,
 			utils.LogNoHistoryFlag,
 			utils.LogExportCheckpointsFlag,
 			utils.StateHistoryFlag,
-		}, utils.DatabaseFlags),
+		}, utils.DatabaseFlags, debug.Flags),
+		Before: func(ctx *cli.Context) error {
+			flags.MigrateGlobalFlags(ctx)
+			return debug.Setup(ctx)
+		},
 		Description: `
 The import command allows the import of blocks from an RLP-encoded format. This format can be a single file
 containing multiple RLP-encoded blocks, or multiple files can be given.
@@ -118,10 +125,7 @@ to import successfully.`,
 		Name:      "export",
 		Usage:     "Export blockchain into file",
 		ArgsUsage: "<filename> [<blockNumFirst> <blockNumLast>]",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:     slices.Concat([]cli.Flag{utils.CacheFlag}, utils.DatabaseFlags),
-			utils.CacheFlag,
-			utils.SyncModeFlag,
-		}, utils.DatabaseFlags),
 		Description: `
 Requires a first argument of the file to write to.
 Optional second and third arguments control the first and
@@ -134,12 +138,7 @@ be gzipped.`,
 		Name:      "import-history",
 		Usage:     "Import an Era archive",
 		ArgsUsage: "<dir>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:     slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag}, utils.DatabaseFlags, utils.NetworkFlags),
-			utils.TxLookupLimitFlag,
-		},
-			utils.DatabaseFlags,
-			utils.NetworkFlags,
-		),
 		Description: `
 The import-history command will import blocks and their corresponding receipts
 from Era archives.
@@ -150,7 +149,7 @@ from Era archives.
 		Name:      "export-history",
 		Usage:     "Export blockchain history to Era archives",
 		ArgsUsage: "<dir> <first> <last>",
-		Flags:     slices.Concat(utils.DatabaseFlags),
+		Flags:     utils.DatabaseFlags,
 		Description: `
 The export-history command will export blocks and their corresponding receipts
 into Era archives. Eras are typically packaged in steps of 8192 blocks.
@@ -161,10 +160,7 @@ into Era archives. Eras are typically packaged in steps of 8192 blocks.
 		Name:      "import-preimages",
 		Usage:     "Import the preimage database from an RLP stream",
 		ArgsUsage: "<datafile>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:     slices.Concat([]cli.Flag{utils.CacheFlag}, utils.DatabaseFlags),
-			utils.CacheFlag,
-			utils.SyncModeFlag,
-		}, utils.DatabaseFlags),
 		Description: `
 The import-preimages command imports hash preimages from an RLP encoded stream.
 It's deprecated, please use "geth db import" instead.
@@ -189,6 +185,18 @@ It's deprecated, please use "geth db import" instead.
 This command dumps out the state for a given block (or latest, if none provided).
 `,
 	}
+
+	pruneCommand = &cli.Command{
+		Action:    pruneHistory,
+		Name:      "prune-history",
+		Usage:     "Prune blockchain history (block bodies and receipts) up to the merge block",
+		ArgsUsage: "",
+		Flags:     utils.DatabaseFlags,
+		Description: `
+The prune-history command removes historical block bodies and receipts from the
+blockchain database up to the merge block, while preserving block headers. This
+helps reduce storage requirements for nodes that don't need full historical data.`,
+	}
 )
 
 // initGenesis will initialise the given JSON format genesis file and writes it as
@@ -422,6 +430,10 @@ func importHistory(ctx *cli.Context) error {
 			network = "mainnet"
 		case ctx.Bool(utils.SepoliaFlag.Name):
 			network = "sepolia"
+		case ctx.Bool(utils.HoleskyFlag.Name):
+			network = "holesky"
+		case ctx.Bool(utils.HoodiFlag.Name):
+			network = "hoodi"
 		}
 	} else {
 		// No network flag set, try to determine network based on files
@@ -598,3 +610,51 @@ func hashish(x string) bool {
 	_, err := strconv.Atoi(x)
 	return err != nil
 }
+
+func pruneHistory(ctx *cli.Context) error {
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	// Open the chain database
+	chain, chaindb := utils.MakeChain(ctx, stack, false)
+	defer chaindb.Close()
+	defer chain.Stop()
+
+	// Determine the prune point. This will be the first PoS block.
+	prunePoint, ok := ethconfig.HistoryPrunePoints[chain.Genesis().Hash()]
+	if !ok || prunePoint == nil {
+		return errors.New("prune point not found")
+	}
+	var (
+		mergeBlock     = prunePoint.BlockNumber
+		mergeBlockHash = prunePoint.BlockHash.Hex()
+	)
+
+	// Check we're far enough past merge to ensure all data is in freezer
+	currentHeader := chain.CurrentHeader()
+	if currentHeader == nil {
+		return errors.New("current header not found")
+	}
+	if currentHeader.Number.Uint64() < mergeBlock+params.FullImmutabilityThreshold {
+		return fmt.Errorf("chain not far enough past merge block, need %d more blocks",
+			mergeBlock+params.FullImmutabilityThreshold-currentHeader.Number.Uint64())
+	}
+
+	// Double-check the prune block in db has the expected hash.
+	hash := rawdb.ReadCanonicalHash(chaindb, mergeBlock)
+	if hash != common.HexToHash(mergeBlockHash) {
+		return fmt.Errorf("merge block hash mismatch: got %s, want %s", hash.Hex(), mergeBlockHash)
+	}
+
+	log.Info("Starting history pruning", "head", currentHeader.Number, "tail", mergeBlock, "tailHash", mergeBlockHash)
+	start := time.Now()
+	rawdb.PruneTransactionIndex(chaindb, mergeBlock)
+	if _, err := chaindb.TruncateTail(mergeBlock); err != nil {
+		return fmt.Errorf("failed to truncate ancient data: %v", err)
+	}
+	log.Info("History pruning completed", "tail", mergeBlock, "elapsed", common.PrettyDuration(time.Since(start)))
+
+	// TODO(s1na): what if there is a crash between the two prune operations?
+
+	return nil
+}

cmd/geth/dbcmd.go

@@ -89,9 +89,7 @@ Remove blockchain and state databases`,
 		Action:      inspect,
 		Name:        "inspect",
 		ArgsUsage:   "<prefix> <start>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Usage:       "Inspect the storage size for each type of data in the database",
 		Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
 	}
@@ -109,16 +107,13 @@ a data corruption.`,
 		Action: dbStats,
 		Name:   "stats",
 		Usage:  "Print leveldb statistics",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:  slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 	}
 	dbCompactCmd = &cli.Command{
 		Action: dbCompact,
 		Name:   "compact",
 		Usage:  "Compact leveldb database. WARNING: May take a very long time",
 		Flags: slices.Concat([]cli.Flag{
-			utils.SyncModeFlag,
 			utils.CacheFlag,
 			utils.CacheDatabaseFlag,
 		}, utils.NetworkFlags, utils.DatabaseFlags),
@@ -131,9 +126,7 @@ corruption if it is aborted during execution'!`,
 		Name:        "get",
 		Usage:       "Show the value of a database key",
 		ArgsUsage:   "<hex-encoded key>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDeleteCmd = &cli.Command{
@@ -141,9 +134,7 @@ corruption if it is aborted during execution'!`,
 		Name:      "delete",
 		Usage:     "Delete a database key (WARNING: may corrupt your database)",
 		ArgsUsage: "<hex-encoded key>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:     slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: `This command deletes the specified database key from the database.
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -152,9 +143,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:      "put",
 		Usage:     "Set the value of a database key (WARNING: may corrupt your database)",
 		ArgsUsage: "<hex-encoded key> <hex-encoded value>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:     slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: `This command sets a given database key to the given value.
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -163,9 +152,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:        "dumptrie",
 		Usage:       "Show the storage key/values of a given storage trie",
 		ArgsUsage:   "<hex-encoded state root> <hex-encoded account hash> <hex-encoded storage trie root> <hex-encoded start (optional)> <int max elements (optional)>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDumpFreezerIndex = &cli.Command{
@@ -173,9 +160,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:        "freezer-index",
 		Usage:       "Dump out the index of a specific freezer table",
 		ArgsUsage:   "<freezer-type> <table-type> <start (int)> <end (int)>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "This command displays information about the freezer index.",
 	}
 	dbImportCmd = &cli.Command{
@@ -183,9 +168,7 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:        "import",
 		Usage:       "Imports leveldb-data from an exported RLP dump.",
 		ArgsUsage:   "<dumpfile> <start (optional)",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "The import command imports the specific chain data from an RLP encoded stream.",
 	}
 	dbExportCmd = &cli.Command{
@@ -193,18 +176,14 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:        "export",
 		Usage:       "Exports the chain data into an RLP dump. If the <dumpfile> has .gz suffix, gzip compression will be used.",
 		ArgsUsage:   "<type> <dumpfile>",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.",
 	}
 	dbMetadataCmd = &cli.Command{
 		Action:      showMetaData,
 		Name:        "metadata",
 		Usage:       "Shows metadata about the chain status.",
-		Flags: slices.Concat([]cli.Flag{
+		Flags:       slices.Concat(utils.NetworkFlags, utils.DatabaseFlags),
-			utils.SyncModeFlag,
-		}, utils.NetworkFlags, utils.DatabaseFlags),
 		Description: "Shows metadata about the chain status.",
 	}
 	dbInspectHistoryCmd = &cli.Command{
@@ -213,7 +192,6 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Usage:     "Inspect the state history within block range",
 		ArgsUsage: "<address> [OPTIONAL <storage-slot>]",
 		Flags: slices.Concat([]cli.Flag{
-			utils.SyncModeFlag,
 			&cli.Uint64Flag{
 				Name:  "start",
 				Usage: "block number of the range start, zero means earliest history",

cmd/geth/main.go

@@ -142,7 +142,6 @@ var (
 		utils.VMTraceJsonConfigFlag,
 		utils.NetworkIdFlag,
 		utils.EthStatsURLFlag,
-		utils.NoCompactionFlag,
 		utils.GpoBlocksFlag,
 		utils.GpoPercentileFlag,
 		utils.GpoMaxGasPriceFlag,
@@ -226,6 +225,7 @@ func init() {
 		removedbCommand,
 		dumpCommand,
 		dumpGenesisCommand,
+		pruneCommand,
 		// See accountcmd.go:
 		accountCommand,
 		walletCommand,

cmd/utils/export_test.go

@@ -54,8 +54,8 @@ func (iter *testIterator) Next() (byte, []byte, []byte, bool) {
 	if iter.index == 42 {
 		iter.index += 1
 	}
-	return OpBatchAdd, []byte(fmt.Sprintf("key-%04d", iter.index)),
+	return OpBatchAdd, fmt.Appendf(nil, "key-%04d", iter.index),
-		[]byte(fmt.Sprintf("value %d", iter.index)), true
+		fmt.Appendf(nil, "value %d", iter.index), true
 }
 
 func (iter *testIterator) Release() {}
@@ -72,7 +72,7 @@ func testExport(t *testing.T, f string) {
 	}
 	// verify
 	for i := 0; i < 1000; i++ {
-		v, err := db.Get([]byte(fmt.Sprintf("key-%04d", i)))
+		v, err := db.Get(fmt.Appendf(nil, "key-%04d", i))
 		if (i < 5 || i == 42) && err == nil {
 			t.Fatalf("expected no element at idx %d, got '%v'", i, string(v))
 		}
@@ -85,7 +85,7 @@ func testExport(t *testing.T, f string) {
 			}
 		}
 	}
-	v, err := db.Get([]byte(fmt.Sprintf("key-%04d", 1000)))
+	v, err := db.Get(fmt.Appendf(nil, "key-%04d", 1000))
 	if err == nil {
 		t.Fatalf("expected no element at idx %d, got '%v'", 1000, string(v))
 	}
@@ -120,7 +120,7 @@ func (iter *deletionIterator) Next() (byte, []byte, []byte, bool) {
 	if iter.index == 42 {
 		iter.index += 1
 	}
-	return OpBatchDel, []byte(fmt.Sprintf("key-%04d", iter.index)), nil, true
+	return OpBatchDel, fmt.Appendf(nil, "key-%04d", iter.index), nil, true
 }
 
 func (iter *deletionIterator) Release() {}
@@ -132,14 +132,14 @@ func testDeletion(t *testing.T, f string) {
 	}
 	db := rawdb.NewMemoryDatabase()
 	for i := 0; i < 1000; i++ {
-		db.Put([]byte(fmt.Sprintf("key-%04d", i)), []byte(fmt.Sprintf("value %d", i)))
+		db.Put(fmt.Appendf(nil, "key-%04d", i), fmt.Appendf(nil, "value %d", i))
 	}
 	err = ImportLDBData(db, f, 5, make(chan struct{}))
 	if err != nil {
 		t.Fatal(err)
 	}
 	for i := 0; i < 1000; i++ {
-		v, err := db.Get([]byte(fmt.Sprintf("key-%04d", i)))
+		v, err := db.Get(fmt.Appendf(nil, "key-%04d", i))
 		if i < 5 || i == 42 {
 			if err != nil {
 				t.Fatalf("expected element at idx %d, got '%v'", i, err)

cmd/utils/flags.go

@@ -2189,6 +2189,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
 	}
 	if !ctx.Bool(SnapshotFlag.Name) {
 		cache.SnapshotLimit = 0 // Disabled
+	} else if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheSnapshotFlag.Name) {
+		cache.SnapshotLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheSnapshotFlag.Name) / 100
 	}
 	// If we're in readonly, do not bother generating snapshot data.
 	if readonly {

common/math/big.go

@@ -72,7 +72,7 @@ func (i *HexOrDecimal256) MarshalText() ([]byte, error) {
 	if i == nil {
 		return []byte("0x0"), nil
 	}
-	return []byte(fmt.Sprintf("%#x", (*big.Int)(i))), nil
+	return fmt.Appendf(nil, "%#x", (*big.Int)(i)), nil
 }
 
 // Decimal256 unmarshals big.Int as a decimal string. When unmarshalling,

common/math/integer.go

@@ -48,7 +48,7 @@ func (i *HexOrDecimal64) UnmarshalText(input []byte) error {
 
 // MarshalText implements encoding.TextMarshaler.
 func (i HexOrDecimal64) MarshalText() ([]byte, error) {
-	return []byte(fmt.Sprintf("%#x", uint64(i))), nil
+	return fmt.Appendf(nil, "%#x", uint64(i)), nil
 }
 
 // ParseUint64 parses s as an integer in decimal or hexadecimal syntax.

core/blockchain.go

@@ -332,7 +332,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 	bc.prefetcher = newStatePrefetcher(chainConfig, bc.hc)
 	bc.processor = NewStateProcessor(chainConfig, bc.hc)
 
-	bc.genesisBlock = bc.GetBlockByNumber(0)
+	genesisHeader := bc.GetHeaderByNumber(0)
+	bc.genesisBlock = types.NewBlockWithHeader(genesisHeader)
 	if bc.genesisBlock == nil {
 		return nil, ErrNoGenesis
 	}
@@ -906,7 +907,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 			rawdb.DeleteBody(db, hash, num)
 			rawdb.DeleteReceipts(db, hash, num)
 		}
-		// Todo(rjl493456442) txlookup, bloombits, etc
+		// Todo(rjl493456442) txlookup, log index, etc
 	}
 	// If SetHead was only called as a chain reparation method, try to skip
 	// touching the header chain altogether, unless the freezer is broken

core/blockchain_reader.go

@@ -86,6 +86,11 @@ func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header {
 	return bc.hc.GetHeaderByNumber(number)
 }
 
+// GetBlockNumber retrieves the block number associated with a block hash.
+func (bc *BlockChain) GetBlockNumber(hash common.Hash) *uint64 {
+	return bc.hc.GetBlockNumber(hash)
+}
+
 // GetHeadersFrom returns a contiguous segment of headers, in rlp-form, going
 // backwards from the given number.
 func (bc *BlockChain) GetHeadersFrom(number, count uint64) []rlp.RawValue {

core/bloom_indexer.go

@@ -1,92 +0,0 @@
-// Copyright 2021 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package core
-
-import (
-	"context"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/bitutil"
-	"github.com/ethereum/go-ethereum/core/bloombits"
-	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethdb"
-)
-
-const (
-	// bloomThrottling is the time to wait between processing two consecutive index
-	// sections. It's useful during chain upgrades to prevent disk overload.
-	bloomThrottling = 100 * time.Millisecond
-)
-
-// BloomIndexer implements a core.ChainIndexer, building up a rotated bloom bits index
-// for the Ethereum header bloom filters, permitting blazing fast filtering.
-type BloomIndexer struct {
-	size    uint64               // section size to generate bloombits for
-	db      ethdb.Database       // database instance to write index data and metadata into
-	gen     *bloombits.Generator // generator to rotate the bloom bits crating the bloom index
-	section uint64               // Section is the section number being processed currently
-	head    common.Hash          // Head is the hash of the last header processed
-}
-
-// NewBloomIndexer returns a chain indexer that generates bloom bits data for the
-// canonical chain for fast logs filtering.
-func NewBloomIndexer(db ethdb.Database, size, confirms uint64) *ChainIndexer {
-	backend := &BloomIndexer{
-		db:   db,
-		size: size,
-	}
-	table := rawdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix))
-
-	return NewChainIndexer(db, table, backend, size, confirms, bloomThrottling, "bloombits")
-}
-
-// Reset implements core.ChainIndexerBackend, starting a new bloombits index
-// section.
-func (b *BloomIndexer) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error {
-	gen, err := bloombits.NewGenerator(uint(b.size))
-	b.gen, b.section, b.head = gen, section, common.Hash{}
-	return err
-}
-
-// Process implements core.ChainIndexerBackend, adding a new header's bloom into
-// the index.
-func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error {
-	b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom)
-	b.head = header.Hash()
-	return nil
-}
-
-// Commit implements core.ChainIndexerBackend, finalizing the bloom section and
-// writing it out into the database.
-func (b *BloomIndexer) Commit() error {
-	batch := b.db.NewBatchWithSize((int(b.size) / 8) * types.BloomBitLength)
-	for i := 0; i < types.BloomBitLength; i++ {
-		bits, err := b.gen.Bitset(uint(i))
-		if err != nil {
-			return err
-		}
-		rawdb.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits))
-	}
-	return batch.Write()
-}
-
-// Prune returns an empty error since we don't support pruning here.
-func (b *BloomIndexer) Prune(threshold uint64) error {
-	return nil
-}

core/bloombits/doc.go

@@ -1,18 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-// Package bloombits implements bloom filtering on batches of data.
-package bloombits

core/bloombits/generator.go

@@ -1,98 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"errors"
-
-	"github.com/ethereum/go-ethereum/core/types"
-)
-
-var (
-	// errSectionOutOfBounds is returned if the user tried to add more bloom filters
-	// to the batch than available space, or if tries to retrieve above the capacity.
-	errSectionOutOfBounds = errors.New("section out of bounds")
-
-	// errBloomBitOutOfBounds is returned if the user tried to retrieve specified
-	// bit bloom above the capacity.
-	errBloomBitOutOfBounds = errors.New("bloom bit out of bounds")
-)
-
-// Generator takes a number of bloom filters and generates the rotated bloom bits
-// to be used for batched filtering.
-type Generator struct {
-	blooms   [types.BloomBitLength][]byte // Rotated blooms for per-bit matching
-	sections uint                         // Number of sections to batch together
-	nextSec  uint                         // Next section to set when adding a bloom
-}
-
-// NewGenerator creates a rotated bloom generator that can iteratively fill a
-// batched bloom filter's bits.
-func NewGenerator(sections uint) (*Generator, error) {
-	if sections%8 != 0 {
-		return nil, errors.New("section count not multiple of 8")
-	}
-	b := &Generator{sections: sections}
-	for i := 0; i < types.BloomBitLength; i++ {
-		b.blooms[i] = make([]byte, sections/8)
-	}
-	return b, nil
-}
-
-// AddBloom takes a single bloom filter and sets the corresponding bit column
-// in memory accordingly.
-func (b *Generator) AddBloom(index uint, bloom types.Bloom) error {
-	// Make sure we're not adding more bloom filters than our capacity
-	if b.nextSec >= b.sections {
-		return errSectionOutOfBounds
-	}
-	if b.nextSec != index {
-		return errors.New("bloom filter with unexpected index")
-	}
-	// Rotate the bloom and insert into our collection
-	byteIndex := b.nextSec / 8
-	bitIndex := byte(7 - b.nextSec%8)
-	for byt := 0; byt < types.BloomByteLength; byt++ {
-		bloomByte := bloom[types.BloomByteLength-1-byt]
-		if bloomByte == 0 {
-			continue
-		}
-		base := 8 * byt
-		b.blooms[base+7][byteIndex] |= ((bloomByte >> 7) & 1) << bitIndex
-		b.blooms[base+6][byteIndex] |= ((bloomByte >> 6) & 1) << bitIndex
-		b.blooms[base+5][byteIndex] |= ((bloomByte >> 5) & 1) << bitIndex
-		b.blooms[base+4][byteIndex] |= ((bloomByte >> 4) & 1) << bitIndex
-		b.blooms[base+3][byteIndex] |= ((bloomByte >> 3) & 1) << bitIndex
-		b.blooms[base+2][byteIndex] |= ((bloomByte >> 2) & 1) << bitIndex
-		b.blooms[base+1][byteIndex] |= ((bloomByte >> 1) & 1) << bitIndex
-		b.blooms[base][byteIndex] |= (bloomByte & 1) << bitIndex
-	}
-	b.nextSec++
-	return nil
-}
-
-// Bitset returns the bit vector belonging to the given bit index after all
-// blooms have been added.
-func (b *Generator) Bitset(idx uint) ([]byte, error) {
-	if b.nextSec != b.sections {
-		return nil, errors.New("bloom not fully generated yet")
-	}
-	if idx >= types.BloomBitLength {
-		return nil, errBloomBitOutOfBounds
-	}
-	return b.blooms[idx], nil
-}

core/bloombits/generator_test.go

@@ -1,100 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"bytes"
-	crand "crypto/rand"
-	"math/rand"
-	"testing"
-
-	"github.com/ethereum/go-ethereum/core/types"
-)
-
-// Tests that batched bloom bits are correctly rotated from the input bloom
-// filters.
-func TestGenerator(t *testing.T) {
-	// Generate the input and the rotated output
-	var input, output [types.BloomBitLength][types.BloomByteLength]byte
-
-	for i := 0; i < types.BloomBitLength; i++ {
-		for j := 0; j < types.BloomBitLength; j++ {
-			bit := byte(rand.Int() % 2)
-
-			input[i][j/8] |= bit << byte(7-j%8)
-			output[types.BloomBitLength-1-j][i/8] |= bit << byte(7-i%8)
-		}
-	}
-	// Crunch the input through the generator and verify the result
-	gen, err := NewGenerator(types.BloomBitLength)
-	if err != nil {
-		t.Fatalf("failed to create bloombit generator: %v", err)
-	}
-	for i, bloom := range input {
-		if err := gen.AddBloom(uint(i), bloom); err != nil {
-			t.Fatalf("bloom %d: failed to add: %v", i, err)
-		}
-	}
-	for i, want := range output {
-		have, err := gen.Bitset(uint(i))
-		if err != nil {
-			t.Fatalf("output %d: failed to retrieve bits: %v", i, err)
-		}
-		if !bytes.Equal(have, want[:]) {
-			t.Errorf("output %d: bit vector mismatch have %x, want %x", i, have, want)
-		}
-	}
-}
-
-func BenchmarkGenerator(b *testing.B) {
-	var input [types.BloomBitLength][types.BloomByteLength]byte
-	b.Run("empty", func(b *testing.B) {
-		b.ReportAllocs()
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			// Crunch the input through the generator and verify the result
-			gen, err := NewGenerator(types.BloomBitLength)
-			if err != nil {
-				b.Fatalf("failed to create bloombit generator: %v", err)
-			}
-			for j, bloom := range &input {
-				if err := gen.AddBloom(uint(j), bloom); err != nil {
-					b.Fatalf("bloom %d: failed to add: %v", i, err)
-				}
-			}
-		}
-	})
-	for i := 0; i < types.BloomBitLength; i++ {
-		crand.Read(input[i][:])
-	}
-	b.Run("random", func(b *testing.B) {
-		b.ReportAllocs()
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			// Crunch the input through the generator and verify the result
-			gen, err := NewGenerator(types.BloomBitLength)
-			if err != nil {
-				b.Fatalf("failed to create bloombit generator: %v", err)
-			}
-			for j, bloom := range &input {
-				if err := gen.AddBloom(uint(j), bloom); err != nil {
-					b.Fatalf("bloom %d: failed to add: %v", i, err)
-				}
-			}
-		}
-	})
-}

core/bloombits/matcher.go

@@ -1,649 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"math"
-	"sort"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common/bitutil"
-	"github.com/ethereum/go-ethereum/crypto"
-)
-
-// bloomIndexes represents the bit indexes inside the bloom filter that belong
-// to some key.
-type bloomIndexes [3]uint
-
-// calcBloomIndexes returns the bloom filter bit indexes belonging to the given key.
-func calcBloomIndexes(b []byte) bloomIndexes {
-	b = crypto.Keccak256(b)
-
-	var idxs bloomIndexes
-	for i := 0; i < len(idxs); i++ {
-		idxs[i] = (uint(b[2*i])<<8)&2047 + uint(b[2*i+1])
-	}
-	return idxs
-}
-
-// partialMatches with a non-nil vector represents a section in which some sub-
-// matchers have already found potential matches. Subsequent sub-matchers will
-// binary AND their matches with this vector. If vector is nil, it represents a
-// section to be processed by the first sub-matcher.
-type partialMatches struct {
-	section uint64
-	bitset  []byte
-}
-
-// Retrieval represents a request for retrieval task assignments for a given
-// bit with the given number of fetch elements, or a response for such a request.
-// It can also have the actual results set to be used as a delivery data struct.
-//
-// The context and error fields are used by the light client to terminate matching
-// early if an error is encountered on some path of the pipeline.
-type Retrieval struct {
-	Bit      uint
-	Sections []uint64
-	Bitsets  [][]byte
-
-	Context context.Context
-	Error   error
-}
-
-// Matcher is a pipelined system of schedulers and logic matchers which perform
-// binary AND/OR operations on the bit-streams, creating a stream of potential
-// blocks to inspect for data content.
-type Matcher struct {
-	sectionSize uint64 // Size of the data batches to filter on
-
-	filters    [][]bloomIndexes    // Filter the system is matching for
-	schedulers map[uint]*scheduler // Retrieval schedulers for loading bloom bits
-
-	retrievers chan chan uint       // Retriever processes waiting for bit allocations
-	counters   chan chan uint       // Retriever processes waiting for task count reports
-	retrievals chan chan *Retrieval // Retriever processes waiting for task allocations
-	deliveries chan *Retrieval      // Retriever processes waiting for task response deliveries
-
-	running atomic.Bool // Atomic flag whether a session is live or not
-}
-
-// NewMatcher creates a new pipeline for retrieving bloom bit streams and doing
-// address and topic filtering on them. Setting a filter component to `nil` is
-// allowed and will result in that filter rule being skipped (OR 0x11...1).
-func NewMatcher(sectionSize uint64, filters [][][]byte) *Matcher {
-	// Create the matcher instance
-	m := &Matcher{
-		sectionSize: sectionSize,
-		schedulers:  make(map[uint]*scheduler),
-		retrievers:  make(chan chan uint),
-		counters:    make(chan chan uint),
-		retrievals:  make(chan chan *Retrieval),
-		deliveries:  make(chan *Retrieval),
-	}
-	// Calculate the bloom bit indexes for the groups we're interested in
-	m.filters = nil
-
-	for _, filter := range filters {
-		// Gather the bit indexes of the filter rule, special casing the nil filter
-		if len(filter) == 0 {
-			continue
-		}
-		bloomBits := make([]bloomIndexes, len(filter))
-		for i, clause := range filter {
-			if clause == nil {
-				bloomBits = nil
-				break
-			}
-			bloomBits[i] = calcBloomIndexes(clause)
-		}
-		// Accumulate the filter rules if no nil rule was within
-		if bloomBits != nil {
-			m.filters = append(m.filters, bloomBits)
-		}
-	}
-	// For every bit, create a scheduler to load/download the bit vectors
-	for _, bloomIndexLists := range m.filters {
-		for _, bloomIndexList := range bloomIndexLists {
-			for _, bloomIndex := range bloomIndexList {
-				m.addScheduler(bloomIndex)
-			}
-		}
-	}
-	return m
-}
-
-// addScheduler adds a bit stream retrieval scheduler for the given bit index if
-// it has not existed before. If the bit is already selected for filtering, the
-// existing scheduler can be used.
-func (m *Matcher) addScheduler(idx uint) {
-	if _, ok := m.schedulers[idx]; ok {
-		return
-	}
-	m.schedulers[idx] = newScheduler(idx)
-}
-
-// Start starts the matching process and returns a stream of bloom matches in
-// a given range of blocks. If there are no more matches in the range, the result
-// channel is closed.
-func (m *Matcher) Start(ctx context.Context, begin, end uint64, results chan uint64) (*MatcherSession, error) {
-	// Make sure we're not creating concurrent sessions
-	if m.running.Swap(true) {
-		return nil, errors.New("matcher already running")
-	}
-	defer m.running.Store(false)
-
-	// Initiate a new matching round
-	session := &MatcherSession{
-		matcher: m,
-		quit:    make(chan struct{}),
-		ctx:     ctx,
-	}
-	for _, scheduler := range m.schedulers {
-		scheduler.reset()
-	}
-	sink := m.run(begin, end, cap(results), session)
-
-	// Read the output from the result sink and deliver to the user
-	session.pend.Add(1)
-	go func() {
-		defer session.pend.Done()
-		defer close(results)
-
-		for {
-			select {
-			case <-session.quit:
-				return
-
-			case res, ok := <-sink:
-				// New match result found
-				if !ok {
-					return
-				}
-				// Calculate the first and last blocks of the section
-				sectionStart := res.section * m.sectionSize
-
-				first := sectionStart
-				if begin > first {
-					first = begin
-				}
-				last := sectionStart + m.sectionSize - 1
-				if end < last {
-					last = end
-				}
-				// Iterate over all the blocks in the section and return the matching ones
-				for i := first; i <= last; i++ {
-					// Skip the entire byte if no matches are found inside (and we're processing an entire byte!)
-					next := res.bitset[(i-sectionStart)/8]
-					if next == 0 {
-						if i%8 == 0 {
-							i += 7
-						}
-						continue
-					}
-					// Some bit it set, do the actual submatching
-					if bit := 7 - i%8; next&(1<<bit) != 0 {
-						select {
-						case <-session.quit:
-							return
-						case results <- i:
-						}
-					}
-				}
-			}
-		}
-	}()
-	return session, nil
-}
-
-// run creates a daisy-chain of sub-matchers, one for the address set and one
-// for each topic set, each sub-matcher receiving a section only if the previous
-// ones have all found a potential match in one of the blocks of the section,
-// then binary AND-ing its own matches and forwarding the result to the next one.
-//
-// The method starts feeding the section indexes into the first sub-matcher on a
-// new goroutine and returns a sink channel receiving the results.
-func (m *Matcher) run(begin, end uint64, buffer int, session *MatcherSession) chan *partialMatches {
-	// Create the source channel and feed section indexes into
-	source := make(chan *partialMatches, buffer)
-
-	session.pend.Add(1)
-	go func() {
-		defer session.pend.Done()
-		defer close(source)
-
-		for i := begin / m.sectionSize; i <= end/m.sectionSize; i++ {
-			select {
-			case <-session.quit:
-				return
-			case source <- &partialMatches{i, bytes.Repeat([]byte{0xff}, int(m.sectionSize/8))}:
-			}
-		}
-	}()
-	// Assemble the daisy-chained filtering pipeline
-	next := source
-	dist := make(chan *request, buffer)
-
-	for _, bloom := range m.filters {
-		next = m.subMatch(next, dist, bloom, session)
-	}
-	// Start the request distribution
-	session.pend.Add(1)
-	go m.distributor(dist, session)
-
-	return next
-}
-
-// subMatch creates a sub-matcher that filters for a set of addresses or topics, binary OR-s those matches, then
-// binary AND-s the result to the daisy-chain input (source) and forwards it to the daisy-chain output.
-// The matches of each address/topic are calculated by fetching the given sections of the three bloom bit indexes belonging to
-// that address/topic, and binary AND-ing those vectors together.
-func (m *Matcher) subMatch(source chan *partialMatches, dist chan *request, bloom []bloomIndexes, session *MatcherSession) chan *partialMatches {
-	// Start the concurrent schedulers for each bit required by the bloom filter
-	sectionSources := make([][3]chan uint64, len(bloom))
-	sectionSinks := make([][3]chan []byte, len(bloom))
-	for i, bits := range bloom {
-		for j, bit := range bits {
-			sectionSources[i][j] = make(chan uint64, cap(source))
-			sectionSinks[i][j] = make(chan []byte, cap(source))
-
-			m.schedulers[bit].run(sectionSources[i][j], dist, sectionSinks[i][j], session.quit, &session.pend)
-		}
-	}
-
-	process := make(chan *partialMatches, cap(source)) // entries from source are forwarded here after fetches have been initiated
-	results := make(chan *partialMatches, cap(source))
-
-	session.pend.Add(2)
-	go func() {
-		// Tear down the goroutine and terminate all source channels
-		defer session.pend.Done()
-		defer close(process)
-
-		defer func() {
-			for _, bloomSources := range sectionSources {
-				for _, bitSource := range bloomSources {
-					close(bitSource)
-				}
-			}
-		}()
-		// Read sections from the source channel and multiplex into all bit-schedulers
-		for {
-			select {
-			case <-session.quit:
-				return
-
-			case subres, ok := <-source:
-				// New subresult from previous link
-				if !ok {
-					return
-				}
-				// Multiplex the section index to all bit-schedulers
-				for _, bloomSources := range sectionSources {
-					for _, bitSource := range bloomSources {
-						select {
-						case <-session.quit:
-							return
-						case bitSource <- subres.section:
-						}
-					}
-				}
-				// Notify the processor that this section will become available
-				select {
-				case <-session.quit:
-					return
-				case process <- subres:
-				}
-			}
-		}
-	}()
-
-	go func() {
-		// Tear down the goroutine and terminate the final sink channel
-		defer session.pend.Done()
-		defer close(results)
-
-		// Read the source notifications and collect the delivered results
-		for {
-			select {
-			case <-session.quit:
-				return
-
-			case subres, ok := <-process:
-				// Notified of a section being retrieved
-				if !ok {
-					return
-				}
-				// Gather all the sub-results and merge them together
-				var orVector []byte
-				for _, bloomSinks := range sectionSinks {
-					var andVector []byte
-					for _, bitSink := range bloomSinks {
-						var data []byte
-						select {
-						case <-session.quit:
-							return
-						case data = <-bitSink:
-						}
-						if andVector == nil {
-							andVector = make([]byte, int(m.sectionSize/8))
-							copy(andVector, data)
-						} else {
-							bitutil.ANDBytes(andVector, andVector, data)
-						}
-					}
-					if orVector == nil {
-						orVector = andVector
-					} else {
-						bitutil.ORBytes(orVector, orVector, andVector)
-					}
-				}
-
-				if orVector == nil {
-					orVector = make([]byte, int(m.sectionSize/8))
-				}
-				if subres.bitset != nil {
-					bitutil.ANDBytes(orVector, orVector, subres.bitset)
-				}
-				if bitutil.TestBytes(orVector) {
-					select {
-					case <-session.quit:
-						return
-					case results <- &partialMatches{subres.section, orVector}:
-					}
-				}
-			}
-		}
-	}()
-	return results
-}
-
-// distributor receives requests from the schedulers and queues them into a set
-// of pending requests, which are assigned to retrievers wanting to fulfil them.
-func (m *Matcher) distributor(dist chan *request, session *MatcherSession) {
-	defer session.pend.Done()
-
-	var (
-		requests   = make(map[uint][]uint64) // Per-bit list of section requests, ordered by section number
-		unallocs   = make(map[uint]struct{}) // Bits with pending requests but not allocated to any retriever
-		retrievers chan chan uint            // Waiting retrievers (toggled to nil if unallocs is empty)
-		allocs     int                       // Number of active allocations to handle graceful shutdown requests
-		shutdown   = session.quit            // Shutdown request channel, will gracefully wait for pending requests
-	)
-
-	// assign is a helper method to try to assign a pending bit an actively
-	// listening servicer, or schedule it up for later when one arrives.
-	assign := func(bit uint) {
-		select {
-		case fetcher := <-m.retrievers:
-			allocs++
-			fetcher <- bit
-		default:
-			// No retrievers active, start listening for new ones
-			retrievers = m.retrievers
-			unallocs[bit] = struct{}{}
-		}
-	}
-
-	for {
-		select {
-		case <-shutdown:
-			// Shutdown requested. No more retrievers can be allocated,
-			// but we still need to wait until all pending requests have returned.
-			shutdown = nil
-			if allocs == 0 {
-				return
-			}
-
-		case req := <-dist:
-			// New retrieval request arrived to be distributed to some fetcher process
-			queue := requests[req.bit]
-			index := sort.Search(len(queue), func(i int) bool { return queue[i] >= req.section })
-			requests[req.bit] = append(queue[:index], append([]uint64{req.section}, queue[index:]...)...)
-
-			// If it's a new bit and we have waiting fetchers, allocate to them
-			if len(queue) == 0 {
-				assign(req.bit)
-			}
-
-		case fetcher := <-retrievers:
-			// New retriever arrived, find the lowest section-ed bit to assign
-			bit, best := uint(0), uint64(math.MaxUint64)
-			for idx := range unallocs {
-				if requests[idx][0] < best {
-					bit, best = idx, requests[idx][0]
-				}
-			}
-			// Stop tracking this bit (and alloc notifications if no more work is available)
-			delete(unallocs, bit)
-			if len(unallocs) == 0 {
-				retrievers = nil
-			}
-			allocs++
-			fetcher <- bit
-
-		case fetcher := <-m.counters:
-			// New task count request arrives, return number of items
-			fetcher <- uint(len(requests[<-fetcher]))
-
-		case fetcher := <-m.retrievals:
-			// New fetcher waiting for tasks to retrieve, assign
-			task := <-fetcher
-			if want := len(task.Sections); want >= len(requests[task.Bit]) {
-				task.Sections = requests[task.Bit]
-				delete(requests, task.Bit)
-			} else {
-				task.Sections = append(task.Sections[:0], requests[task.Bit][:want]...)
-				requests[task.Bit] = append(requests[task.Bit][:0], requests[task.Bit][want:]...)
-			}
-			fetcher <- task
-
-			// If anything was left unallocated, try to assign to someone else
-			if len(requests[task.Bit]) > 0 {
-				assign(task.Bit)
-			}
-
-		case result := <-m.deliveries:
-			// New retrieval task response from fetcher, split out missing sections and
-			// deliver complete ones
-			var (
-				sections = make([]uint64, 0, len(result.Sections))
-				bitsets  = make([][]byte, 0, len(result.Bitsets))
-				missing  = make([]uint64, 0, len(result.Sections))
-			)
-			for i, bitset := range result.Bitsets {
-				if len(bitset) == 0 {
-					missing = append(missing, result.Sections[i])
-					continue
-				}
-				sections = append(sections, result.Sections[i])
-				bitsets = append(bitsets, bitset)
-			}
-			m.schedulers[result.Bit].deliver(sections, bitsets)
-			allocs--
-
-			// Reschedule missing sections and allocate bit if newly available
-			if len(missing) > 0 {
-				queue := requests[result.Bit]
-				for _, section := range missing {
-					index := sort.Search(len(queue), func(i int) bool { return queue[i] >= section })
-					queue = append(queue[:index], append([]uint64{section}, queue[index:]...)...)
-				}
-				requests[result.Bit] = queue
-
-				if len(queue) == len(missing) {
-					assign(result.Bit)
-				}
-			}
-
-			// End the session when all pending deliveries have arrived.
-			if shutdown == nil && allocs == 0 {
-				return
-			}
-		}
-	}
-}
-
-// MatcherSession is returned by a started matcher to be used as a terminator
-// for the actively running matching operation.
-type MatcherSession struct {
-	matcher *Matcher
-
-	closer sync.Once     // Sync object to ensure we only ever close once
-	quit   chan struct{} // Quit channel to request pipeline termination
-
-	ctx     context.Context // Context used by the light client to abort filtering
-	err     error           // Global error to track retrieval failures deep in the chain
-	errLock sync.Mutex
-
-	pend sync.WaitGroup
-}
-
-// Close stops the matching process and waits for all subprocesses to terminate
-// before returning. The timeout may be used for graceful shutdown, allowing the
-// currently running retrievals to complete before this time.
-func (s *MatcherSession) Close() {
-	s.closer.Do(func() {
-		// Signal termination and wait for all goroutines to tear down
-		close(s.quit)
-		s.pend.Wait()
-	})
-}
-
-// Error returns any failure encountered during the matching session.
-func (s *MatcherSession) Error() error {
-	s.errLock.Lock()
-	defer s.errLock.Unlock()
-
-	return s.err
-}
-
-// allocateRetrieval assigns a bloom bit index to a client process that can either
-// immediately request and fetch the section contents assigned to this bit or wait
-// a little while for more sections to be requested.
-func (s *MatcherSession) allocateRetrieval() (uint, bool) {
-	fetcher := make(chan uint)
-
-	select {
-	case <-s.quit:
-		return 0, false
-	case s.matcher.retrievers <- fetcher:
-		bit, ok := <-fetcher
-		return bit, ok
-	}
-}
-
-// pendingSections returns the number of pending section retrievals belonging to
-// the given bloom bit index.
-func (s *MatcherSession) pendingSections(bit uint) int {
-	fetcher := make(chan uint)
-
-	select {
-	case <-s.quit:
-		return 0
-	case s.matcher.counters <- fetcher:
-		fetcher <- bit
-		return int(<-fetcher)
-	}
-}
-
-// allocateSections assigns all or part of an already allocated bit-task queue
-// to the requesting process.
-func (s *MatcherSession) allocateSections(bit uint, count int) []uint64 {
-	fetcher := make(chan *Retrieval)
-
-	select {
-	case <-s.quit:
-		return nil
-	case s.matcher.retrievals <- fetcher:
-		task := &Retrieval{
-			Bit:      bit,
-			Sections: make([]uint64, count),
-		}
-		fetcher <- task
-		return (<-fetcher).Sections
-	}
-}
-
-// deliverSections delivers a batch of section bit-vectors for a specific bloom
-// bit index to be injected into the processing pipeline.
-func (s *MatcherSession) deliverSections(bit uint, sections []uint64, bitsets [][]byte) {
-	s.matcher.deliveries <- &Retrieval{Bit: bit, Sections: sections, Bitsets: bitsets}
-}
-
-// Multiplex polls the matcher session for retrieval tasks and multiplexes it into
-// the requested retrieval queue to be serviced together with other sessions.
-//
-// This method will block for the lifetime of the session. Even after termination
-// of the session, any request in-flight need to be responded to! Empty responses
-// are fine though in that case.
-func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan *Retrieval) {
-	waitTimer := time.NewTimer(wait)
-	defer waitTimer.Stop()
-
-	for {
-		// Allocate a new bloom bit index to retrieve data for, stopping when done
-		bit, ok := s.allocateRetrieval()
-		if !ok {
-			return
-		}
-		// Bit allocated, throttle a bit if we're below our batch limit
-		if s.pendingSections(bit) < batch {
-			waitTimer.Reset(wait)
-			select {
-			case <-s.quit:
-				// Session terminating, we can't meaningfully service, abort
-				s.allocateSections(bit, 0)
-				s.deliverSections(bit, []uint64{}, [][]byte{})
-				return
-
-			case <-waitTimer.C:
-				// Throttling up, fetch whatever is available
-			}
-		}
-		// Allocate as much as we can handle and request servicing
-		sections := s.allocateSections(bit, batch)
-		request := make(chan *Retrieval)
-
-		select {
-		case <-s.quit:
-			// Session terminating, we can't meaningfully service, abort
-			s.deliverSections(bit, sections, make([][]byte, len(sections)))
-			return
-
-		case mux <- request:
-			// Retrieval accepted, something must arrive before we're aborting
-			request <- &Retrieval{Bit: bit, Sections: sections, Context: s.ctx}
-
-			result := <-request
-
-			// Deliver a result before s.Close() to avoid a deadlock
-			s.deliverSections(result.Bit, result.Sections, result.Bitsets)
-
-			if result.Error != nil {
-				s.errLock.Lock()
-				s.err = result.Error
-				s.errLock.Unlock()
-				s.Close()
-			}
-		}
-	}
-}

core/bloombits/matcher_test.go

@@ -1,292 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"context"
-	"math/rand"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-)
-
-const testSectionSize = 4096
-
-// Tests that wildcard filter rules (nil) can be specified and are handled well.
-func TestMatcherWildcards(t *testing.T) {
-	t.Parallel()
-	matcher := NewMatcher(testSectionSize, [][][]byte{
-		{common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard
-		{common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()},       // Default hash is not a wildcard
-		{common.Hash{0x01}.Bytes()},                              // Plain rule, sanity check
-		{common.Hash{0x01}.Bytes(), nil},                         // Wildcard suffix, drop rule
-		{nil, common.Hash{0x01}.Bytes()},                         // Wildcard prefix, drop rule
-		{nil, nil},                                               // Wildcard combo, drop rule
-		{},                                                       // Inited wildcard rule, drop rule
-		nil,                                                      // Proper wildcard rule, drop rule
-	})
-	if len(matcher.filters) != 3 {
-		t.Fatalf("filter system size mismatch: have %d, want %d", len(matcher.filters), 3)
-	}
-	if len(matcher.filters[0]) != 2 {
-		t.Fatalf("address clause size mismatch: have %d, want %d", len(matcher.filters[0]), 2)
-	}
-	if len(matcher.filters[1]) != 2 {
-		t.Fatalf("combo topic clause size mismatch: have %d, want %d", len(matcher.filters[1]), 2)
-	}
-	if len(matcher.filters[2]) != 1 {
-		t.Fatalf("singletone topic clause size mismatch: have %d, want %d", len(matcher.filters[2]), 1)
-	}
-}
-
-// Tests the matcher pipeline on a single continuous workflow without interrupts.
-func TestMatcherContinuous(t *testing.T) {
-	t.Parallel()
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, false, 75)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, false, 81)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, false, 36)
-}
-
-// Tests the matcher pipeline on a constantly interrupted and resumed work pattern
-// with the aim of ensuring data items are requested only once.
-func TestMatcherIntermittent(t *testing.T) {
-	t.Parallel()
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, true, 75)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, true, 81)
-	testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, true, 36)
-}
-
-// Tests the matcher pipeline on random input to hopefully catch anomalies.
-func TestMatcherRandom(t *testing.T) {
-	t.Parallel()
-	for i := 0; i < 10; i++ {
-		testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 0, 10000, 0)
-		testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 0, 10000, 0)
-	}
-}
-
-// Tests that the matcher can properly find matches if the starting block is
-// shifted from a multiple of 8. This is needed to cover an optimisation with
-// bitset matching https://github.com/ethereum/go-ethereum/issues/15309.
-func TestMatcherShifted(t *testing.T) {
-	t.Parallel()
-	// Block 0 always matches in the tests, skip ahead of first 8 blocks with the
-	// start to get a potential zero byte in the matcher bitset.
-
-	// To keep the second bitset byte zero, the filter must only match for the first
-	// time in block 16, so doing an all-16 bit filter should suffice.
-
-	// To keep the starting block non divisible by 8, block number 9 is the first
-	// that would introduce a shift and not match block 0.
-	testMatcherBothModes(t, [][]bloomIndexes{{{16, 16, 16}}}, 9, 64, 0)
-}
-
-// Tests that matching on everything doesn't crash (special case internally).
-func TestWildcardMatcher(t *testing.T) {
-	t.Parallel()
-	testMatcherBothModes(t, nil, 0, 10000, 0)
-}
-
-// makeRandomIndexes generates a random filter system, composed of multiple filter
-// criteria, each having one bloom list component for the address and arbitrarily
-// many topic bloom list components.
-func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes {
-	res := make([][]bloomIndexes, len(lengths))
-	for i, topics := range lengths {
-		res[i] = make([]bloomIndexes, topics)
-		for j := 0; j < topics; j++ {
-			for k := 0; k < len(res[i][j]); k++ {
-				res[i][j][k] = uint(rand.Intn(max-1) + 2)
-			}
-		}
-	}
-	return res
-}
-
-// testMatcherDiffBatches runs the given matches test in single-delivery and also
-// in batches delivery mode, verifying that all kinds of deliveries are handled
-// correctly within.
-func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) {
-	singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1)
-	batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16)
-
-	if singleton != batched {
-		t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in singleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched)
-	}
-}
-
-// testMatcherBothModes runs the given matcher test in both continuous as well as
-// in intermittent mode, verifying that the request counts match each other.
-func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, retrievals uint32) {
-	continuous := testMatcher(t, filter, start, blocks, false, retrievals, 16)
-	intermittent := testMatcher(t, filter, start, blocks, true, retrievals, 16)
-
-	if continuous != intermittent {
-		t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent)
-	}
-}
-
-// testMatcher is a generic tester to run the given matcher test and return the
-// number of requests made for cross validation between different modes.
-func testMatcher(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 {
-	// Create a new matcher an simulate our explicit random bitsets
-	matcher := NewMatcher(testSectionSize, nil)
-	matcher.filters = filter
-
-	for _, rule := range filter {
-		for _, topic := range rule {
-			for _, bit := range topic {
-				matcher.addScheduler(bit)
-			}
-		}
-	}
-	// Track the number of retrieval requests made
-	var requested atomic.Uint32
-
-	// Start the matching session for the filter and the retriever goroutines
-	quit := make(chan struct{})
-	matches := make(chan uint64, 16)
-
-	session, err := matcher.Start(context.Background(), start, blocks-1, matches)
-	if err != nil {
-		t.Fatalf("failed to stat matcher session: %v", err)
-	}
-	startRetrievers(session, quit, &requested, maxReqCount)
-
-	// Iterate over all the blocks and verify that the pipeline produces the correct matches
-	for i := start; i < blocks; i++ {
-		if expMatch3(filter, i) {
-			match, ok := <-matches
-			if !ok {
-				t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i)
-				return 0
-			}
-			if match != i {
-				t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match)
-			}
-			// If we're testing intermittent mode, abort and restart the pipeline
-			if intermittent {
-				session.Close()
-				close(quit)
-
-				quit = make(chan struct{})
-				matches = make(chan uint64, 16)
-
-				session, err = matcher.Start(context.Background(), i+1, blocks-1, matches)
-				if err != nil {
-					t.Fatalf("failed to stat matcher session: %v", err)
-				}
-				startRetrievers(session, quit, &requested, maxReqCount)
-			}
-		}
-	}
-	// Ensure the result channel is torn down after the last block
-	match, ok := <-matches
-	if ok {
-		t.Errorf("filter = %v  blocks = %v  intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match)
-	}
-	// Clean up the session and ensure we match the expected retrieval count
-	session.Close()
-	close(quit)
-
-	if retrievals != 0 && requested.Load() != retrievals {
-		t.Errorf("filter = %v  blocks = %v  intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested.Load(), retrievals)
-	}
-	return requested.Load()
-}
-
-// startRetrievers starts a batch of goroutines listening for section requests
-// and serving them.
-func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *atomic.Uint32, batch int) {
-	requests := make(chan chan *Retrieval)
-
-	for i := 0; i < 10; i++ {
-		// Start a multiplexer to test multiple threaded execution
-		go session.Multiplex(batch, 100*time.Microsecond, requests)
-
-		// Start a services to match the above multiplexer
-		go func() {
-			for {
-				// Wait for a service request or a shutdown
-				select {
-				case <-quit:
-					return
-
-				case request := <-requests:
-					task := <-request
-
-					task.Bitsets = make([][]byte, len(task.Sections))
-					for i, section := range task.Sections {
-						if rand.Int()%4 != 0 { // Handle occasional missing deliveries
-							task.Bitsets[i] = generateBitset(task.Bit, section)
-							retrievals.Add(1)
-						}
-					}
-					request <- task
-				}
-			}
-		}()
-	}
-}
-
-// generateBitset generates the rotated bitset for the given bloom bit and section
-// numbers.
-func generateBitset(bit uint, section uint64) []byte {
-	bitset := make([]byte, testSectionSize/8)
-	for i := 0; i < len(bitset); i++ {
-		for b := 0; b < 8; b++ {
-			blockIdx := section*testSectionSize + uint64(i*8+b)
-			bitset[i] += bitset[i]
-			if (blockIdx % uint64(bit)) == 0 {
-				bitset[i]++
-			}
-		}
-	}
-	return bitset
-}
-
-func expMatch1(filter bloomIndexes, i uint64) bool {
-	for _, ii := range filter {
-		if (i % uint64(ii)) != 0 {
-			return false
-		}
-	}
-	return true
-}
-
-func expMatch2(filter []bloomIndexes, i uint64) bool {
-	for _, ii := range filter {
-		if expMatch1(ii, i) {
-			return true
-		}
-	}
-	return false
-}
-
-func expMatch3(filter [][]bloomIndexes, i uint64) bool {
-	for _, ii := range filter {
-		if !expMatch2(ii, i) {
-			return false
-		}
-	}
-	return true
-}

core/bloombits/scheduler.go

@@ -1,181 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"sync"
-)
-
-// request represents a bloom retrieval task to prioritize and pull from the local
-// database or remotely from the network.
-type request struct {
-	section uint64 // Section index to retrieve the bit-vector from
-	bit     uint   // Bit index within the section to retrieve the vector of
-}
-
-// response represents the state of a requested bit-vector through a scheduler.
-type response struct {
-	cached []byte        // Cached bits to dedup multiple requests
-	done   chan struct{} // Channel to allow waiting for completion
-}
-
-// scheduler handles the scheduling of bloom-filter retrieval operations for
-// entire section-batches belonging to a single bloom bit. Beside scheduling the
-// retrieval operations, this struct also deduplicates the requests and caches
-// the results to minimize network/database overhead even in complex filtering
-// scenarios.
-type scheduler struct {
-	bit       uint                 // Index of the bit in the bloom filter this scheduler is responsible for
-	responses map[uint64]*response // Currently pending retrieval requests or already cached responses
-	lock      sync.Mutex           // Lock protecting the responses from concurrent access
-}
-
-// newScheduler creates a new bloom-filter retrieval scheduler for a specific
-// bit index.
-func newScheduler(idx uint) *scheduler {
-	return &scheduler{
-		bit:       idx,
-		responses: make(map[uint64]*response),
-	}
-}
-
-// run creates a retrieval pipeline, receiving section indexes from sections and
-// returning the results in the same order through the done channel. Concurrent
-// runs of the same scheduler are allowed, leading to retrieval task deduplication.
-func (s *scheduler) run(sections chan uint64, dist chan *request, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) {
-	// Create a forwarder channel between requests and responses of the same size as
-	// the distribution channel (since that will block the pipeline anyway).
-	pend := make(chan uint64, cap(dist))
-
-	// Start the pipeline schedulers to forward between user -> distributor -> user
-	wg.Add(2)
-	go s.scheduleRequests(sections, dist, pend, quit, wg)
-	go s.scheduleDeliveries(pend, done, quit, wg)
-}
-
-// reset cleans up any leftovers from previous runs. This is required before a
-// restart to ensure the no previously requested but never delivered state will
-// cause a lockup.
-func (s *scheduler) reset() {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	for section, res := range s.responses {
-		if res.cached == nil {
-			delete(s.responses, section)
-		}
-	}
-}
-
-// scheduleRequests reads section retrieval requests from the input channel,
-// deduplicates the stream and pushes unique retrieval tasks into the distribution
-// channel for a database or network layer to honour.
-func (s *scheduler) scheduleRequests(reqs chan uint64, dist chan *request, pend chan uint64, quit chan struct{}, wg *sync.WaitGroup) {
-	// Clean up the goroutine and pipeline when done
-	defer wg.Done()
-	defer close(pend)
-
-	// Keep reading and scheduling section requests
-	for {
-		select {
-		case <-quit:
-			return
-
-		case section, ok := <-reqs:
-			// New section retrieval requested
-			if !ok {
-				return
-			}
-			// Deduplicate retrieval requests
-			unique := false
-
-			s.lock.Lock()
-			if s.responses[section] == nil {
-				s.responses[section] = &response{
-					done: make(chan struct{}),
-				}
-				unique = true
-			}
-			s.lock.Unlock()
-
-			// Schedule the section for retrieval and notify the deliverer to expect this section
-			if unique {
-				select {
-				case <-quit:
-					return
-				case dist <- &request{bit: s.bit, section: section}:
-				}
-			}
-			select {
-			case <-quit:
-				return
-			case pend <- section:
-			}
-		}
-	}
-}
-
-// scheduleDeliveries reads section acceptance notifications and waits for them
-// to be delivered, pushing them into the output data buffer.
-func (s *scheduler) scheduleDeliveries(pend chan uint64, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) {
-	// Clean up the goroutine and pipeline when done
-	defer wg.Done()
-	defer close(done)
-
-	// Keep reading notifications and scheduling deliveries
-	for {
-		select {
-		case <-quit:
-			return
-
-		case idx, ok := <-pend:
-			// New section retrieval pending
-			if !ok {
-				return
-			}
-			// Wait until the request is honoured
-			s.lock.Lock()
-			res := s.responses[idx]
-			s.lock.Unlock()
-
-			select {
-			case <-quit:
-				return
-			case <-res.done:
-			}
-			// Deliver the result
-			select {
-			case <-quit:
-				return
-			case done <- res.cached:
-			}
-		}
-	}
-}
-
-// deliver is called by the request distributor when a reply to a request arrives.
-func (s *scheduler) deliver(sections []uint64, data [][]byte) {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-
-	for i, section := range sections {
-		if res := s.responses[section]; res != nil && res.cached == nil { // Avoid non-requests and double deliveries
-			res.cached = data[i]
-			close(res.done)
-		}
-	}
-}

core/bloombits/scheduler_test.go

@@ -1,103 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package bloombits
-
-import (
-	"bytes"
-	"math/big"
-	"sync"
-	"sync/atomic"
-	"testing"
-)
-
-// Tests that the scheduler can deduplicate and forward retrieval requests to
-// underlying fetchers and serve responses back, irrelevant of the concurrency
-// of the requesting clients or serving data fetchers.
-func TestSchedulerSingleClientSingleFetcher(t *testing.T) { testScheduler(t, 1, 1, 5000) }
-func TestSchedulerSingleClientMultiFetcher(t *testing.T)  { testScheduler(t, 1, 10, 5000) }
-func TestSchedulerMultiClientSingleFetcher(t *testing.T)  { testScheduler(t, 10, 1, 5000) }
-func TestSchedulerMultiClientMultiFetcher(t *testing.T)   { testScheduler(t, 10, 10, 5000) }
-
-func testScheduler(t *testing.T, clients int, fetchers int, requests int) {
-	t.Parallel()
-	f := newScheduler(0)
-
-	// Create a batch of handler goroutines that respond to bloom bit requests and
-	// deliver them to the scheduler.
-	var fetchPend sync.WaitGroup
-	fetchPend.Add(fetchers)
-	defer fetchPend.Wait()
-
-	fetch := make(chan *request, 16)
-	defer close(fetch)
-
-	var delivered atomic.Uint32
-	for i := 0; i < fetchers; i++ {
-		go func() {
-			defer fetchPend.Done()
-
-			for req := range fetch {
-				delivered.Add(1)
-
-				f.deliver([]uint64{
-					req.section + uint64(requests), // Non-requested data (ensure it doesn't go out of bounds)
-					req.section,                    // Requested data
-					req.section,                    // Duplicated data (ensure it doesn't double close anything)
-				}, [][]byte{
-					{},
-					new(big.Int).SetUint64(req.section).Bytes(),
-					new(big.Int).SetUint64(req.section).Bytes(),
-				})
-			}
-		}()
-	}
-	// Start a batch of goroutines to concurrently run scheduling tasks
-	quit := make(chan struct{})
-
-	var pend sync.WaitGroup
-	pend.Add(clients)
-
-	for i := 0; i < clients; i++ {
-		go func() {
-			defer pend.Done()
-
-			in := make(chan uint64, 16)
-			out := make(chan []byte, 16)
-
-			f.run(in, fetch, out, quit, &pend)
-
-			go func() {
-				for j := 0; j < requests; j++ {
-					in <- uint64(j)
-				}
-				close(in)
-			}()
-			b := new(big.Int)
-			for j := 0; j < requests; j++ {
-				bits := <-out
-				if want := b.SetUint64(uint64(j)).Bytes(); !bytes.Equal(bits, want) {
-					t.Errorf("vector %d: delivered content mismatch: have %x, want %x", j, bits, want)
-				}
-			}
-		}()
-	}
-	pend.Wait()
-
-	if have := delivered.Load(); int(have) != requests {
-		t.Errorf("request count mismatch: have %v, want %v", have, requests)
-	}
-}

core/chain_indexer.go

@@ -1,522 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package core
-
-import (
-	"context"
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/event"
-	"github.com/ethereum/go-ethereum/log"
-)
-
-// ChainIndexerBackend defines the methods needed to process chain segments in
-// the background and write the segment results into the database. These can be
-// used to create filter blooms or CHTs.
-type ChainIndexerBackend interface {
-	// Reset initiates the processing of a new chain segment, potentially terminating
-	// any partially completed operations (in case of a reorg).
-	Reset(ctx context.Context, section uint64, prevHead common.Hash) error
-
-	// Process crunches through the next header in the chain segment. The caller
-	// will ensure a sequential order of headers.
-	Process(ctx context.Context, header *types.Header) error
-
-	// Commit finalizes the section metadata and stores it into the database.
-	Commit() error
-
-	// Prune deletes the chain index older than the given threshold.
-	Prune(threshold uint64) error
-}
-
-// ChainIndexerChain interface is used for connecting the indexer to a blockchain
-type ChainIndexerChain interface {
-	// CurrentHeader retrieves the latest locally known header.
-	CurrentHeader() *types.Header
-
-	// SubscribeChainHeadEvent subscribes to new head header notifications.
-	SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription
-}
-
-// ChainIndexer does a post-processing job for equally sized sections of the
-// canonical chain (like BlooomBits and CHT structures). A ChainIndexer is
-// connected to the blockchain through the event system by starting a
-// ChainHeadEventLoop in a goroutine.
-//
-// Further child ChainIndexers can be added which use the output of the parent
-// section indexer. These child indexers receive new head notifications only
-// after an entire section has been finished or in case of rollbacks that might
-// affect already finished sections.
-type ChainIndexer struct {
-	chainDb  ethdb.Database      // Chain database to index the data from
-	indexDb  ethdb.Database      // Prefixed table-view of the db to write index metadata into
-	backend  ChainIndexerBackend // Background processor generating the index data content
-	children []*ChainIndexer     // Child indexers to cascade chain updates to
-
-	active    atomic.Bool     // Flag whether the event loop was started
-	update    chan struct{}   // Notification channel that headers should be processed
-	quit      chan chan error // Quit channel to tear down running goroutines
-	ctx       context.Context
-	ctxCancel func()
-
-	sectionSize uint64 // Number of blocks in a single chain segment to process
-	confirmsReq uint64 // Number of confirmations before processing a completed segment
-
-	storedSections uint64 // Number of sections successfully indexed into the database
-	knownSections  uint64 // Number of sections known to be complete (block wise)
-	cascadedHead   uint64 // Block number of the last completed section cascaded to subindexers
-
-	checkpointSections uint64      // Number of sections covered by the checkpoint
-	checkpointHead     common.Hash // Section head belonging to the checkpoint
-
-	throttling time.Duration // Disk throttling to prevent a heavy upgrade from hogging resources
-
-	log  log.Logger
-	lock sync.Mutex
-}
-
-// NewChainIndexer creates a new chain indexer to do background processing on
-// chain segments of a given size after certain number of confirmations passed.
-// The throttling parameter might be used to prevent database thrashing.
-func NewChainIndexer(chainDb ethdb.Database, indexDb ethdb.Database, backend ChainIndexerBackend, section, confirm uint64, throttling time.Duration, kind string) *ChainIndexer {
-	c := &ChainIndexer{
-		chainDb:     chainDb,
-		indexDb:     indexDb,
-		backend:     backend,
-		update:      make(chan struct{}, 1),
-		quit:        make(chan chan error),
-		sectionSize: section,
-		confirmsReq: confirm,
-		throttling:  throttling,
-		log:         log.New("type", kind),
-	}
-	// Initialize database dependent fields and start the updater
-	c.loadValidSections()
-	c.ctx, c.ctxCancel = context.WithCancel(context.Background())
-
-	go c.updateLoop()
-
-	return c
-}
-
-// AddCheckpoint adds a checkpoint. Sections are never processed and the chain
-// is not expected to be available before this point. The indexer assumes that
-// the backend has sufficient information available to process subsequent sections.
-//
-// Note: knownSections == 0 and storedSections == checkpointSections until
-// syncing reaches the checkpoint
-func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	// Short circuit if the given checkpoint is below than local's.
-	if c.checkpointSections >= section+1 || section < c.storedSections {
-		return
-	}
-	c.checkpointSections = section + 1
-	c.checkpointHead = shead
-
-	c.setSectionHead(section, shead)
-	c.setValidSections(section + 1)
-}
-
-// Start creates a goroutine to feed chain head events into the indexer for
-// cascading background processing. Children do not need to be started, they
-// are notified about new events by their parents.
-func (c *ChainIndexer) Start(chain ChainIndexerChain) {
-	events := make(chan ChainHeadEvent, 10)
-	sub := chain.SubscribeChainHeadEvent(events)
-
-	go c.eventLoop(chain.CurrentHeader(), events, sub)
-}
-
-// Close tears down all goroutines belonging to the indexer and returns any error
-// that might have occurred internally.
-func (c *ChainIndexer) Close() error {
-	var errs []error
-
-	c.ctxCancel()
-
-	// Tear down the primary update loop
-	errc := make(chan error)
-	c.quit <- errc
-	if err := <-errc; err != nil {
-		errs = append(errs, err)
-	}
-	// If needed, tear down the secondary event loop
-	if c.active.Load() {
-		c.quit <- errc
-		if err := <-errc; err != nil {
-			errs = append(errs, err)
-		}
-	}
-	// Close all children
-	for _, child := range c.children {
-		if err := child.Close(); err != nil {
-			errs = append(errs, err)
-		}
-	}
-	// Return any failures
-	switch {
-	case len(errs) == 0:
-		return nil
-
-	case len(errs) == 1:
-		return errs[0]
-
-	default:
-		return fmt.Errorf("%v", errs)
-	}
-}
-
-// eventLoop is a secondary - optional - event loop of the indexer which is only
-// started for the outermost indexer to push chain head events into a processing
-// queue.
-func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainHeadEvent, sub event.Subscription) {
-	// Mark the chain indexer as active, requiring an additional teardown
-	c.active.Store(true)
-
-	defer sub.Unsubscribe()
-
-	// Fire the initial new head event to start any outstanding processing
-	c.newHead(currentHeader.Number.Uint64(), false)
-
-	var (
-		prevHeader = currentHeader
-		prevHash   = currentHeader.Hash()
-	)
-	for {
-		select {
-		case errc := <-c.quit:
-			// Chain indexer terminating, report no failure and abort
-			errc <- nil
-			return
-
-		case ev, ok := <-events:
-			// Received a new event, ensure it's not nil (closing) and update
-			if !ok {
-				errc := <-c.quit
-				errc <- nil
-				return
-			}
-			if ev.Header.ParentHash != prevHash {
-				// Reorg to the common ancestor if needed (might not exist in light sync mode, skip reorg then)
-				// TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly?
-
-				if rawdb.ReadCanonicalHash(c.chainDb, prevHeader.Number.Uint64()) != prevHash {
-					if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, ev.Header); h != nil {
-						c.newHead(h.Number.Uint64(), true)
-					}
-				}
-			}
-			c.newHead(ev.Header.Number.Uint64(), false)
-
-			prevHeader, prevHash = ev.Header, ev.Header.Hash()
-		}
-	}
-}
-
-// newHead notifies the indexer about new chain heads and/or reorgs.
-func (c *ChainIndexer) newHead(head uint64, reorg bool) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	// If a reorg happened, invalidate all sections until that point
-	if reorg {
-		// Revert the known section number to the reorg point
-		known := (head + 1) / c.sectionSize
-		stored := known
-		if known < c.checkpointSections {
-			known = 0
-		}
-		if stored < c.checkpointSections {
-			stored = c.checkpointSections
-		}
-		if known < c.knownSections {
-			c.knownSections = known
-		}
-		// Revert the stored sections from the database to the reorg point
-		if stored < c.storedSections {
-			c.setValidSections(stored)
-		}
-		// Update the new head number to the finalized section end and notify children
-		head = known * c.sectionSize
-
-		if head < c.cascadedHead {
-			c.cascadedHead = head
-			for _, child := range c.children {
-				child.newHead(c.cascadedHead, true)
-			}
-		}
-		return
-	}
-	// No reorg, calculate the number of newly known sections and update if high enough
-	var sections uint64
-	if head >= c.confirmsReq {
-		sections = (head + 1 - c.confirmsReq) / c.sectionSize
-		if sections < c.checkpointSections {
-			sections = 0
-		}
-		if sections > c.knownSections {
-			if c.knownSections < c.checkpointSections {
-				// syncing reached the checkpoint, verify section head
-				syncedHead := rawdb.ReadCanonicalHash(c.chainDb, c.checkpointSections*c.sectionSize-1)
-				if syncedHead != c.checkpointHead {
-					c.log.Error("Synced chain does not match checkpoint", "number", c.checkpointSections*c.sectionSize-1, "expected", c.checkpointHead, "synced", syncedHead)
-					return
-				}
-			}
-			c.knownSections = sections
-
-			select {
-			case c.update <- struct{}{}:
-			default:
-			}
-		}
-	}
-}
-
-// updateLoop is the main event loop of the indexer which pushes chain segments
-// down into the processing backend.
-func (c *ChainIndexer) updateLoop() {
-	var (
-		updating bool
-		updated  time.Time
-	)
-
-	for {
-		select {
-		case errc := <-c.quit:
-			// Chain indexer terminating, report no failure and abort
-			errc <- nil
-			return
-
-		case <-c.update:
-			// Section headers completed (or rolled back), update the index
-			c.lock.Lock()
-			if c.knownSections > c.storedSections {
-				// Periodically print an upgrade log message to the user
-				if time.Since(updated) > 8*time.Second {
-					if c.knownSections > c.storedSections+1 {
-						updating = true
-						c.log.Info("Upgrading chain index", "percentage", c.storedSections*100/c.knownSections)
-					}
-					updated = time.Now()
-				}
-				// Cache the current section count and head to allow unlocking the mutex
-				c.verifyLastHead()
-				section := c.storedSections
-				var oldHead common.Hash
-				if section > 0 {
-					oldHead = c.SectionHead(section - 1)
-				}
-				// Process the newly defined section in the background
-				c.lock.Unlock()
-				newHead, err := c.processSection(section, oldHead)
-				if err != nil {
-					select {
-					case <-c.ctx.Done():
-						<-c.quit <- nil
-						return
-					default:
-					}
-					c.log.Error("Section processing failed", "error", err)
-				}
-				c.lock.Lock()
-
-				// If processing succeeded and no reorgs occurred, mark the section completed
-				if err == nil && (section == 0 || oldHead == c.SectionHead(section-1)) {
-					c.setSectionHead(section, newHead)
-					c.setValidSections(section + 1)
-					if c.storedSections == c.knownSections && updating {
-						updating = false
-						c.log.Info("Finished upgrading chain index")
-					}
-					c.cascadedHead = c.storedSections*c.sectionSize - 1
-					for _, child := range c.children {
-						c.log.Trace("Cascading chain index update", "head", c.cascadedHead)
-						child.newHead(c.cascadedHead, false)
-					}
-				} else {
-					// If processing failed, don't retry until further notification
-					c.log.Debug("Chain index processing failed", "section", section, "err", err)
-					c.verifyLastHead()
-					c.knownSections = c.storedSections
-				}
-			}
-			// If there are still further sections to process, reschedule
-			if c.knownSections > c.storedSections {
-				time.AfterFunc(c.throttling, func() {
-					select {
-					case c.update <- struct{}{}:
-					default:
-					}
-				})
-			}
-			c.lock.Unlock()
-		}
-	}
-}
-
-// processSection processes an entire section by calling backend functions while
-// ensuring the continuity of the passed headers. Since the chain mutex is not
-// held while processing, the continuity can be broken by a long reorg, in which
-// case the function returns with an error.
-func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (common.Hash, error) {
-	c.log.Trace("Processing new chain section", "section", section)
-
-	// Reset and partial processing
-	if err := c.backend.Reset(c.ctx, section, lastHead); err != nil {
-		c.setValidSections(0)
-		return common.Hash{}, err
-	}
-
-	for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ {
-		hash := rawdb.ReadCanonicalHash(c.chainDb, number)
-		if hash == (common.Hash{}) {
-			return common.Hash{}, fmt.Errorf("canonical block #%d unknown", number)
-		}
-		header := rawdb.ReadHeader(c.chainDb, hash, number)
-		if header == nil {
-			return common.Hash{}, fmt.Errorf("block #%d [%x..] not found", number, hash[:4])
-		} else if header.ParentHash != lastHead {
-			return common.Hash{}, errors.New("chain reorged during section processing")
-		}
-		if err := c.backend.Process(c.ctx, header); err != nil {
-			return common.Hash{}, err
-		}
-		lastHead = header.Hash()
-	}
-	if err := c.backend.Commit(); err != nil {
-		return common.Hash{}, err
-	}
-	return lastHead, nil
-}
-
-// verifyLastHead compares last stored section head with the corresponding block hash in the
-// actual canonical chain and rolls back reorged sections if necessary to ensure that stored
-// sections are all valid
-func (c *ChainIndexer) verifyLastHead() {
-	for c.storedSections > 0 && c.storedSections > c.checkpointSections {
-		if c.SectionHead(c.storedSections-1) == rawdb.ReadCanonicalHash(c.chainDb, c.storedSections*c.sectionSize-1) {
-			return
-		}
-		c.setValidSections(c.storedSections - 1)
-	}
-}
-
-// Sections returns the number of processed sections maintained by the indexer
-// and also the information about the last header indexed for potential canonical
-// verifications.
-func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) {
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	c.verifyLastHead()
-	return c.storedSections, c.storedSections*c.sectionSize - 1, c.SectionHead(c.storedSections - 1)
-}
-
-// AddChildIndexer adds a child ChainIndexer that can use the output of this one
-func (c *ChainIndexer) AddChildIndexer(indexer *ChainIndexer) {
-	if indexer == c {
-		panic("can't add indexer as a child of itself")
-	}
-	c.lock.Lock()
-	defer c.lock.Unlock()
-
-	c.children = append(c.children, indexer)
-
-	// Cascade any pending updates to new children too
-	sections := c.storedSections
-	if c.knownSections < sections {
-		// if a section is "stored" but not "known" then it is a checkpoint without
-		// available chain data so we should not cascade it yet
-		sections = c.knownSections
-	}
-	if sections > 0 {
-		indexer.newHead(sections*c.sectionSize-1, false)
-	}
-}
-
-// Prune deletes all chain data older than given threshold.
-func (c *ChainIndexer) Prune(threshold uint64) error {
-	return c.backend.Prune(threshold)
-}
-
-// loadValidSections reads the number of valid sections from the index database
-// and caches is into the local state.
-func (c *ChainIndexer) loadValidSections() {
-	data, _ := c.indexDb.Get([]byte("count"))
-	if len(data) == 8 {
-		c.storedSections = binary.BigEndian.Uint64(data)
-	}
-}
-
-// setValidSections writes the number of valid sections to the index database
-func (c *ChainIndexer) setValidSections(sections uint64) {
-	// Set the current number of valid sections in the database
-	var data [8]byte
-	binary.BigEndian.PutUint64(data[:], sections)
-	c.indexDb.Put([]byte("count"), data[:])
-
-	// Remove any reorged sections, caching the valids in the mean time
-	for c.storedSections > sections {
-		c.storedSections--
-		c.removeSectionHead(c.storedSections)
-	}
-	c.storedSections = sections // needed if new > old
-}
-
-// SectionHead retrieves the last block hash of a processed section from the
-// index database.
-func (c *ChainIndexer) SectionHead(section uint64) common.Hash {
-	var data [8]byte
-	binary.BigEndian.PutUint64(data[:], section)
-
-	hash, _ := c.indexDb.Get(append([]byte("shead"), data[:]...))
-	if len(hash) == len(common.Hash{}) {
-		return common.BytesToHash(hash)
-	}
-	return common.Hash{}
-}
-
-// setSectionHead writes the last block hash of a processed section to the index
-// database.
-func (c *ChainIndexer) setSectionHead(section uint64, hash common.Hash) {
-	var data [8]byte
-	binary.BigEndian.PutUint64(data[:], section)
-
-	c.indexDb.Put(append([]byte("shead"), data[:]...), hash.Bytes())
-}
-
-// removeSectionHead removes the reference to a processed section from the index
-// database.
-func (c *ChainIndexer) removeSectionHead(section uint64) {
-	var data [8]byte
-	binary.BigEndian.PutUint64(data[:], section)
-
-	c.indexDb.Delete(append([]byte("shead"), data[:]...))
-}

core/chain_indexer_test.go

@@ -1,246 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package core
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"math/big"
-	"math/rand"
-	"testing"
-	"time"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/core/types"
-)
-
-// Runs multiple tests with randomized parameters.
-func TestChainIndexerSingle(t *testing.T) {
-	for i := 0; i < 10; i++ {
-		testChainIndexer(t, 1)
-	}
-}
-
-// Runs multiple tests with randomized parameters and different number of
-// chain backends.
-func TestChainIndexerWithChildren(t *testing.T) {
-	for i := 2; i < 8; i++ {
-		testChainIndexer(t, i)
-	}
-}
-
-// testChainIndexer runs a test with either a single chain indexer or a chain of
-// multiple backends. The section size and required confirmation count parameters
-// are randomized.
-func testChainIndexer(t *testing.T, count int) {
-	db := rawdb.NewMemoryDatabase()
-	defer db.Close()
-
-	// Create a chain of indexers and ensure they all report empty
-	backends := make([]*testChainIndexBackend, count)
-	for i := 0; i < count; i++ {
-		var (
-			sectionSize = uint64(rand.Intn(100) + 1)
-			confirmsReq = uint64(rand.Intn(10))
-		)
-		backends[i] = &testChainIndexBackend{t: t, processCh: make(chan uint64)}
-		backends[i].indexer = NewChainIndexer(db, rawdb.NewTable(db, string([]byte{byte(i)})), backends[i], sectionSize, confirmsReq, 0, fmt.Sprintf("indexer-%d", i))
-
-		if sections, _, _ := backends[i].indexer.Sections(); sections != 0 {
-			t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, 0)
-		}
-		if i > 0 {
-			backends[i-1].indexer.AddChildIndexer(backends[i].indexer)
-		}
-	}
-	defer backends[0].indexer.Close() // parent indexer shuts down children
-	// notify pings the root indexer about a new head or reorg, then expect
-	// processed blocks if a section is processable
-	notify := func(headNum, failNum uint64, reorg bool) {
-		backends[0].indexer.newHead(headNum, reorg)
-		if reorg {
-			for _, backend := range backends {
-				headNum = backend.reorg(headNum)
-				backend.assertSections()
-			}
-			return
-		}
-		var cascade bool
-		for _, backend := range backends {
-			headNum, cascade = backend.assertBlocks(headNum, failNum)
-			if !cascade {
-				break
-			}
-			backend.assertSections()
-		}
-	}
-	// inject inserts a new random canonical header into the database directly
-	inject := func(number uint64) {
-		header := &types.Header{Number: big.NewInt(int64(number)), Extra: big.NewInt(rand.Int63()).Bytes()}
-		if number > 0 {
-			header.ParentHash = rawdb.ReadCanonicalHash(db, number-1)
-		}
-		rawdb.WriteHeader(db, header)
-		rawdb.WriteCanonicalHash(db, header.Hash(), number)
-	}
-	// Start indexer with an already existing chain
-	for i := uint64(0); i <= 100; i++ {
-		inject(i)
-	}
-	notify(100, 100, false)
-
-	// Add new blocks one by one
-	for i := uint64(101); i <= 1000; i++ {
-		inject(i)
-		notify(i, i, false)
-	}
-	// Do a reorg
-	notify(500, 500, true)
-
-	// Create new fork
-	for i := uint64(501); i <= 1000; i++ {
-		inject(i)
-		notify(i, i, false)
-	}
-	for i := uint64(1001); i <= 1500; i++ {
-		inject(i)
-	}
-	// Failed processing scenario where less blocks are available than notified
-	notify(2000, 1500, false)
-
-	// Notify about a reorg (which could have caused the missing blocks if happened during processing)
-	notify(1500, 1500, true)
-
-	// Create new fork
-	for i := uint64(1501); i <= 2000; i++ {
-		inject(i)
-		notify(i, i, false)
-	}
-}
-
-// testChainIndexBackend implements ChainIndexerBackend
-type testChainIndexBackend struct {
-	t                          *testing.T
-	indexer                    *ChainIndexer
-	section, headerCnt, stored uint64
-	processCh                  chan uint64
-}
-
-// assertSections verifies if a chain indexer has the correct number of section.
-func (b *testChainIndexBackend) assertSections() {
-	// Keep trying for 3 seconds if it does not match
-	var sections uint64
-	for i := 0; i < 300; i++ {
-		sections, _, _ = b.indexer.Sections()
-		if sections == b.stored {
-			return
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	b.t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, b.stored)
-}
-
-// assertBlocks expects processing calls after new blocks have arrived. If the
-// failNum < headNum then we are simulating a scenario where a reorg has happened
-// after the processing has started and the processing of a section fails.
-func (b *testChainIndexBackend) assertBlocks(headNum, failNum uint64) (uint64, bool) {
-	var sections uint64
-	if headNum >= b.indexer.confirmsReq {
-		sections = (headNum + 1 - b.indexer.confirmsReq) / b.indexer.sectionSize
-		if sections > b.stored {
-			// expect processed blocks
-			for expectd := b.stored * b.indexer.sectionSize; expectd < sections*b.indexer.sectionSize; expectd++ {
-				if expectd > failNum {
-					// rolled back after processing started, no more process calls expected
-					// wait until updating is done to make sure that processing actually fails
-					var updating bool
-					for i := 0; i < 300; i++ {
-						b.indexer.lock.Lock()
-						updating = b.indexer.knownSections > b.indexer.storedSections
-						b.indexer.lock.Unlock()
-						if !updating {
-							break
-						}
-						time.Sleep(10 * time.Millisecond)
-					}
-					if updating {
-						b.t.Fatalf("update did not finish")
-					}
-					sections = expectd / b.indexer.sectionSize
-					break
-				}
-				select {
-				case <-time.After(10 * time.Second):
-					b.t.Fatalf("Expected processed block #%d, got nothing", expectd)
-				case processed := <-b.processCh:
-					if processed != expectd {
-						b.t.Errorf("Expected processed block #%d, got #%d", expectd, processed)
-					}
-				}
-			}
-			b.stored = sections
-		}
-	}
-	if b.stored == 0 {
-		return 0, false
-	}
-	return b.stored*b.indexer.sectionSize - 1, true
-}
-
-func (b *testChainIndexBackend) reorg(headNum uint64) uint64 {
-	firstChanged := (headNum + 1) / b.indexer.sectionSize
-	if firstChanged < b.stored {
-		b.stored = firstChanged
-	}
-	return b.stored * b.indexer.sectionSize
-}
-
-func (b *testChainIndexBackend) Reset(ctx context.Context, section uint64, prevHead common.Hash) error {
-	b.section = section
-	b.headerCnt = 0
-	return nil
-}
-
-func (b *testChainIndexBackend) Process(ctx context.Context, header *types.Header) error {
-	b.headerCnt++
-	if b.headerCnt > b.indexer.sectionSize {
-		b.t.Error("Processing too many headers")
-	}
-	//t.processCh <- header.Number.Uint64()
-	select {
-	case <-time.After(10 * time.Second):
-		b.t.Error("Unexpected call to Process")
-		// Can't use Fatal since this is not the test's goroutine.
-		// Returning error stops the chainIndexer's updateLoop
-		return errors.New("unexpected call to Process")
-	case b.processCh <- header.Number.Uint64():
-	}
-	return nil
-}
-
-func (b *testChainIndexBackend) Commit() error {
-	if b.headerCnt != b.indexer.sectionSize {
-		b.t.Error("Not enough headers processed")
-	}
-	return nil
-}
-
-func (b *testChainIndexBackend) Prune(threshold uint64) error {
-	return nil
-}

core/chain_makers.go

@@ -504,6 +504,13 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine
 		if gen != nil {
 			gen(i, b)
 		}
+
+		requests := b.collectRequests(false)
+		if requests != nil {
+			reqHash := types.CalcRequestsHash(requests)
+			b.header.RequestsHash = &reqHash
+		}
+
 		body := &types.Body{
 			Transactions: b.txs,
 			Uncles:       b.uncles,

core/filtermaps/chain_view.go

@@ -83,7 +83,11 @@ func (cv *ChainView) getReceipts(number uint64) types.Receipts {
 	if number > cv.headNumber {
 		panic("invalid block number")
 	}
-	return cv.chain.GetReceiptsByHash(cv.blockHash(number))
+	blockHash := cv.blockHash(number)
+	if blockHash == (common.Hash{}) {
+		log.Error("Chain view: block hash unavailable", "number", number, "head", cv.headNumber)
+	}
+	return cv.chain.GetReceiptsByHash(blockHash)
 }
 
 // limitedView returns a new chain view that is a truncated version of the parent view.

core/filtermaps/filtermaps.go

@@ -17,7 +17,6 @@
 package filtermaps
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"os"
@@ -30,8 +29,24 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/ethdb/leveldb"
 	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/metrics"
+)
+
+var (
+	mapCountGauge           = metrics.NewRegisteredGauge("filtermaps/maps/count", nil)          // actual number of rendered maps
+	mapLogValueMeter        = metrics.NewRegisteredMeter("filtermaps/maps/logvalues", nil)      // number of log values processed
+	mapBlockMeter           = metrics.NewRegisteredMeter("filtermaps/maps/blocks", nil)         // number of block delimiters processed
+	mapRenderTimer          = metrics.NewRegisteredTimer("filtermaps/maps/rendertime", nil)     // time elapsed while rendering a single map
+	mapWriteTimer           = metrics.NewRegisteredTimer("filtermaps/maps/writetime", nil)      // time elapsed while writing a batch of finished maps to db
+	matchRequestTimer       = metrics.NewRegisteredTimer("filtermaps/match/requesttime", nil)   // processing time a matching request in a single epoch
+	matchEpochTimer         = metrics.NewRegisteredTimer("filtermaps/match/epochtime", nil)     // total processing time a matching request
+	matchBaseRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowaccess", nil) // number of accessed rows on layer 0
+	matchBaseRowSizeMeter   = metrics.NewRegisteredMeter("filtermaps/match/baserowsize", nil)   // size of accessed rows on layer 0
+	matchExtRowAccessMeter  = metrics.NewRegisteredMeter("filtermaps/match/extrowaccess", nil)  // number of accessed rows on higher layers
+	matchExtRowSizeMeter    = metrics.NewRegisteredMeter("filtermaps/match/extrowsize", nil)    // size of accessed rows on higher layers
+	matchLogLookup          = metrics.NewRegisteredMeter("filtermaps/match/loglookup", nil)     // number of log lookups based on potential matches
+	matchAllMeter           = metrics.NewRegisteredMeter("filtermaps/match/matchall", nil)      // number of requests returned with ErrMatchAll
 )
 
 const (
@@ -55,10 +70,12 @@ type FilterMaps struct {
 	// We chose to implement disabling this way because it requires less special
 	// case logic in eth/filters.
 	disabled   bool
+	disabledCh chan struct{} // closed by indexer if disabled
 
 	closeCh        chan struct{}
 	closeWg        sync.WaitGroup
 	history        uint64
+	hashScheme     bool // use hashdb-safe delete range method
 	exportFileName string
 	Params
 
@@ -69,6 +86,7 @@ type FilterMaps struct {
 	// Matcher backend can read them under indexLock read lock.
 	indexLock           sync.RWMutex
 	indexedRange        filterMapsRange
+	cleanedEpochsBefore uint32     // all unindexed data cleaned before this point
 	indexedView         *ChainView // always consistent with the log index
 	hasTempRange        bool
 
@@ -180,6 +198,10 @@ type Config struct {
 	// This option enables the checkpoint JSON file generator.
 	// If set, the given file will be updated with checkpoint information.
 	ExportFileName string
+
+	// expect trie nodes of hash based state scheme in the filtermaps key range;
+	// use safe iterator based implementation of DeleteRange that skips them
+	HashScheme bool
 }
 
 // NewFilterMaps creates a new FilterMaps and starts the indexer.
@@ -197,6 +219,8 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f
 		blockProcessingCh: make(chan bool, 1),
 		history:           config.History,
 		disabled:          config.Disabled,
+		hashScheme:        config.HashScheme,
+		disabledCh:        make(chan struct{}),
 		exportFileName:    config.ExportFileName,
 		Params:            params,
 		indexedRange: filterMapsRange{
@@ -207,6 +231,10 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f
 			maps:             common.NewRange(rs.MapsFirst, rs.MapsAfterLast-rs.MapsFirst),
 			tailPartialEpoch: rs.TailPartialEpoch,
 		},
+		// deleting last unindexed epoch might have been interrupted by shutdown
+		cleanedEpochsBefore: max(rs.MapsFirst>>params.logMapsPerEpoch, 1) - 1,
+		historyCutoff:       historyCutoff,
+		finalBlock:          finalBlock,
 		matcherSyncCh:       make(chan *FilterMapsMatcherBackend),
 		matchers:            make(map[*FilterMapsMatcherBackend]struct{}),
 		filterMapCache:      lru.NewCache[uint32, filterMap](cachedFilterMaps),
@@ -242,7 +270,8 @@ func (f *FilterMaps) Start() {
 			log.Error("Could not load head filter map snapshot", "error", err)
 		}
 	}
-	f.closeWg.Add(1)
+	f.closeWg.Add(2)
+	go f.removeBloomBits()
 	go f.indexerLoop()
 }
 
@@ -278,8 +307,13 @@ func (f *FilterMaps) initChainView(chainView *ChainView) *ChainView {
 }
 
 // reset un-initializes the FilterMaps structure and removes all related data from
-// the database. The function returns true if everything was successfully removed.
+// the database.
-func (f *FilterMaps) reset() bool {
+// Note that in case of leveldb database the fallback implementation of DeleteRange
+// might take a long time to finish and deleting the entire database may be
+// interrupted by a shutdown. Deleting the filterMapsRange entry first does
+// guarantee though that the next init() will not return successfully until the
+// entire database has been cleaned.
+func (f *FilterMaps) reset() {
 	f.indexLock.Lock()
 	f.indexedRange = filterMapsRange{}
 	f.indexedView = nil
@@ -292,11 +326,26 @@ func (f *FilterMaps) reset() bool {
 	// deleting the range first ensures that resetDb will be called again at next
 	// startup and any leftover data will be removed even if it cannot finish now.
 	rawdb.DeleteFilterMapsRange(f.db)
-	return f.removeDbWithPrefix([]byte(rawdb.FilterMapsPrefix), "Resetting log index database")
+	f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown)
+}
+
+// isShuttingDown returns true if FilterMaps is shutting down.
+func (f *FilterMaps) isShuttingDown() bool {
+	select {
+	case <-f.closeCh:
+		return true
+	default:
+		return false
+	}
 }
 
 // init initializes an empty log index according to the current targetView.
 func (f *FilterMaps) init() error {
+	// ensure that there is no remaining data in the filter maps key range
+	if err := f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown); err != nil {
+		return err
+	}
+
 	f.indexLock.Lock()
 	defer f.indexLock.Unlock()
 
@@ -317,6 +366,13 @@ func (f *FilterMaps) init() error {
 			bestIdx, bestLen = idx, max
 		}
 	}
+	var initBlockNumber uint64
+	if bestLen > 0 {
+		initBlockNumber = checkpoints[bestIdx][bestLen-1].BlockNumber
+	}
+	if initBlockNumber < f.historyCutoff {
+		return errors.New("cannot start indexing before history cutoff point")
+	}
 	batch := f.db.NewBatch()
 	for epoch := range bestLen {
 		cp := checkpoints[bestIdx][epoch]
@@ -335,39 +391,39 @@ func (f *FilterMaps) init() error {
 	return batch.Write()
 }
 
-// removeDbWithPrefix removes data with the given prefix from the database and
+// removeBloomBits removes old bloom bits data from the database.
-// returns true if everything was successfully removed.
+func (f *FilterMaps) removeBloomBits() {
-func (f *FilterMaps) removeDbWithPrefix(prefix []byte, action string) bool {
+	f.safeDeleteWithLogs(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database", f.isShuttingDown)
-	it := f.db.NewIterator(prefix, nil)
+	f.closeWg.Done()
-	hasData := it.Next()
-	it.Release()
-	if !hasData {
-		return true
 }
 
-	end := bytes.Clone(prefix)
+// safeDeleteWithLogs is a wrapper for a function that performs a safe range
-	end[len(end)-1]++
+// delete operation using rawdb.SafeDeleteRange. It emits log messages if the
-	start := time.Now()
+// process takes long enough to call the stop callback.
-	var retry bool
+func (f *FilterMaps) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error, action string, stopCb func() bool) error {
-	for {
+	var (
-		err := f.db.DeleteRange(prefix, end)
+		start          = time.Now()
-		if err == nil {
+		logPrinted     bool
-			log.Info(action+" finished", "elapsed", time.Since(start))
+		lastLogPrinted = start
-			return true
+	)
+	switch err := deleteFn(f.db, f.hashScheme, func(deleted bool) bool {
+		if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 {
+			log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start)))
+			logPrinted, lastLogPrinted = true, time.Now()
 		}
-		if err != leveldb.ErrTooManyKeys {
+		return stopCb()
-			log.Error(action+" failed", "error", err)
+	}); {
-			return false
+	case err == nil:
+		if logPrinted {
+			log.Info(action+" finished", "elapsed", common.PrettyDuration(time.Since(start)))
 		}
-		select {
+		return nil
-		case <-f.closeCh:
+	case errors.Is(err, rawdb.ErrDeleteRangeInterrupted):
-			return false
+		log.Warn(action+" interrupted", "elapsed", common.PrettyDuration(time.Since(start)))
+		return err
 	default:
-		}
+		log.Error(action+" failed", "error", err)
-		if !retry {
+		return err
-			log.Info(action + " in progress...")
-			retry = true
-		}
 	}
 }
 
@@ -390,8 +446,12 @@ func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, ne
 			TailPartialEpoch: newRange.tailPartialEpoch,
 		}
 		rawdb.WriteFilterMapsRange(batch, rs)
+		if !isTempRange {
+			mapCountGauge.Update(int64(newRange.maps.Count() + newRange.tailPartialEpoch))
+		}
 	} else {
 		rawdb.DeleteFilterMapsRange(batch)
+		mapCountGauge.Update(0)
 	}
 }
 
@@ -636,54 +696,97 @@ func (f *FilterMaps) deleteLastBlockOfMap(batch ethdb.Batch, mapIndex uint32) {
 	rawdb.DeleteFilterMapLastBlock(batch, mapIndex)
 }
 
-// deleteTailEpoch deletes index data from the earliest, either fully or partially
+// deleteTailEpoch deletes index data from the specified epoch. The last block
-// indexed epoch. The last block pointer for the last map of the epoch and the
+// pointer for the last map of the epoch and the corresponding block log value
-// corresponding block log value pointer are retained as these are always assumed
+// pointer are retained as these are always assumed to be available for each
-// to be available for each epoch.
+// epoch as boundary markers.
-func (f *FilterMaps) deleteTailEpoch(epoch uint32) error {
+// The function returns true if all index data related to the epoch (except for
+// the boundary markers) has been fully removed.
+func (f *FilterMaps) deleteTailEpoch(epoch uint32) (bool, error) {
 	f.indexLock.Lock()
 	defer f.indexLock.Unlock()
 
+	// determine epoch boundaries
 	firstMap := epoch << f.logMapsPerEpoch
 	lastBlock, _, err := f.getLastBlockOfMap(firstMap + f.mapsPerEpoch - 1)
 	if err != nil {
-		return fmt.Errorf("failed to retrieve last block of deleted epoch %d: %v", epoch, err)
+		return false, fmt.Errorf("failed to retrieve last block of deleted epoch %d: %v", epoch, err)
 	}
 	var firstBlock uint64
 	if epoch > 0 {
 		firstBlock, _, err = f.getLastBlockOfMap(firstMap - 1)
 		if err != nil {
-			return fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err)
+			return false, fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err)
 		}
 		firstBlock++
 	}
-	fmr := f.indexedRange
+	// update rendered range if necessary
-	if f.indexedRange.maps.First() == firstMap &&
+	var (
-		f.indexedRange.maps.AfterLast() > firstMap+f.mapsPerEpoch &&
+		fmr            = f.indexedRange
-		f.indexedRange.tailPartialEpoch == 0 {
+		firstEpoch     = f.indexedRange.maps.First() >> f.logMapsPerEpoch
-		fmr.maps.SetFirst(firstMap + f.mapsPerEpoch)
+		afterLastEpoch = (f.indexedRange.maps.AfterLast() + f.mapsPerEpoch - 1) >> f.logMapsPerEpoch
-		fmr.blocks.SetFirst(lastBlock + 1)
+	)
-	} else if f.indexedRange.maps.First() == firstMap+f.mapsPerEpoch {
+	if f.indexedRange.tailPartialEpoch != 0 && firstEpoch > 0 {
-		fmr.tailPartialEpoch = 0
+		firstEpoch--
-	} else {
-		return errors.New("invalid tail epoch number")
 	}
+	switch {
+	case epoch < firstEpoch:
+	// cleanup of already unindexed epoch; range not affected
+	case epoch == firstEpoch && epoch+1 < afterLastEpoch:
+		// first fully or partially rendered epoch and there is at least one
+		// rendered map in the next epoch; remove from indexed range
+		fmr.tailPartialEpoch = 0
+		fmr.maps.SetFirst((epoch + 1) << f.logMapsPerEpoch)
+		fmr.blocks.SetFirst(lastBlock + 1)
 		f.setRange(f.db, f.indexedView, fmr, false)
+	default:
+		// cannot be cleaned or unindexed; return with error
+		return false, errors.New("invalid tail epoch number")
+	}
+	// remove index data
+	if err := f.safeDeleteWithLogs(func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error {
 		first := f.mapRowIndex(firstMap, 0)
 		count := f.mapRowIndex(firstMap+f.mapsPerEpoch, 0) - first
-	rawdb.DeleteFilterMapRows(f.db, common.NewRange(first, count))
+		if err := rawdb.DeleteFilterMapRows(f.db, common.NewRange(first, count), hashScheme, stopCb); err != nil {
+			return err
+		}
 		for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch; mapIndex++ {
 			f.filterMapCache.Remove(mapIndex)
 		}
-	rawdb.DeleteFilterMapLastBlocks(f.db, common.NewRange(firstMap, f.mapsPerEpoch-1)) // keep last enrty
+		delMapRange := common.NewRange(firstMap, f.mapsPerEpoch-1) // keep last entry
+		if err := rawdb.DeleteFilterMapLastBlocks(f.db, delMapRange, hashScheme, stopCb); err != nil {
+			return err
+		}
 		for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch-1; mapIndex++ {
 			f.lastBlockCache.Remove(mapIndex)
 		}
-	rawdb.DeleteBlockLvPointers(f.db, common.NewRange(firstBlock, lastBlock-firstBlock)) // keep last enrty
+		delBlockRange := common.NewRange(firstBlock, lastBlock-firstBlock) // keep last entry
+		if err := rawdb.DeleteBlockLvPointers(f.db, delBlockRange, hashScheme, stopCb); err != nil {
+			return err
+		}
 		for blockNumber := firstBlock; blockNumber < lastBlock; blockNumber++ {
 			f.lvPointerCache.Remove(blockNumber)
 		}
 		return nil
+	}, fmt.Sprintf("Deleting tail epoch #%d", epoch), func() bool {
+		f.processEvents()
+		return f.stop || !f.targetHeadIndexed()
+	}); err == nil {
+		// everything removed; mark as cleaned and report success
+		if f.cleanedEpochsBefore == epoch {
+			f.cleanedEpochsBefore = epoch + 1
+		}
+		return true, nil
+	} else {
+		// more data left in epoch range; mark as dirty and report unfinished
+		if f.cleanedEpochsBefore > epoch {
+			f.cleanedEpochsBefore = epoch
+		}
+		if errors.Is(err, rawdb.ErrDeleteRangeInterrupted) {
+			return false, nil
+		}
+		return false, err
+	}
 }
 
 // exportCheckpoints exports epoch checkpoints in the format used by checkpoints.go.

core/filtermaps/indexer.go

@@ -17,6 +17,7 @@
 package filtermaps
 
 import (
+	"errors"
 	"math"
 	"time"
 
@@ -36,21 +37,28 @@ func (f *FilterMaps) indexerLoop() {
 
 	if f.disabled {
 		f.reset()
+		close(f.disabledCh)
 		return
 	}
 	log.Info("Started log indexer")
 
 	for !f.stop {
 		if !f.indexedRange.initialized {
-			if err := f.init(); err != nil {
+			if f.targetView.headNumber == 0 {
-				log.Error("Error initializing log index", "error", err)
+				// initialize when chain head is available
-				f.waitForEvent()
+				f.processSingleEvent(true)
 				continue
 			}
+			if err := f.init(); err != nil {
+				f.disableForError("initialization", err)
+				f.reset() // remove broken index from DB
+				return
+			}
 		}
 		if !f.targetHeadIndexed() {
-			if !f.tryIndexHead() {
+			if err := f.tryIndexHead(); err != nil && err != errChainUpdate {
-				f.waitForEvent()
+				f.disableForError("head rendering", err)
+				return
 			}
 		} else {
 			if f.finalBlock != f.lastFinal {
@@ -59,11 +67,37 @@ func (f *FilterMaps) indexerLoop() {
 				}
 				f.lastFinal = f.finalBlock
 			}
-			if f.tryIndexTail() && f.tryUnindexTail() {
+			// always attempt unindexing before indexing the tail in order to
-				f.waitForEvent()
+			// ensure that a potentially dirty previously unindexed epoch is
+			// always cleaned up before any new maps are rendered.
+			if done, err := f.tryUnindexTail(); err != nil {
+				f.disableForError("tail unindexing", err)
+				return
+			} else if !done {
+				continue
+			}
+			if done, err := f.tryIndexTail(); err != nil {
+				f.disableForError("tail rendering", err)
+				return
+			} else if !done {
+				continue
+			}
+			// tail indexing/unindexing is done; if head is also indexed then
+			// wait here until there is a new head
+			f.waitForNewHead()
 		}
 	}
 }
+
+// disableForError is called when the indexer encounters a database error, for example a
+// missing receipt. We can't continue operating when the database is broken, so the
+// indexer goes into disabled state.
+// Note that the partial index is left in disk; maybe a client update can fix the
+// issue without reindexing.
+func (f *FilterMaps) disableForError(op string, err error) {
+	log.Error("Log index "+op+" failed, reverting to unindexed mode", "error", err)
+	f.disabled = true
+	close(f.disabledCh)
 }
 
 type targetUpdate struct {
@@ -106,21 +140,23 @@ func (f *FilterMaps) SetBlockProcessing(blockProcessing bool) {
 // WaitIdle blocks until the indexer is in an idle state while synced up to the
 // latest targetView.
 func (f *FilterMaps) WaitIdle() {
-	if f.disabled {
-		f.closeWg.Wait()
-		return
-	}
 	for {
 		ch := make(chan bool)
-		f.waitIdleCh <- ch
+		select {
+		case f.waitIdleCh <- ch:
 			if <-ch {
 				return
 			}
+		case <-f.disabledCh:
+			f.closeWg.Wait()
+			return
+		}
 	}
 }
 
-// waitForEvent blocks until an event happens that the indexer might react to.
+// waitForNewHead blocks until there is a new target head to index and block
-func (f *FilterMaps) waitForEvent() {
+// processing has been finished.
+func (f *FilterMaps) waitForNewHead() {
 	for !f.stop && (f.blockProcessing || f.targetHeadIndexed()) {
 		f.processSingleEvent(true)
 	}
@@ -129,13 +165,16 @@ func (f *FilterMaps) waitForEvent() {
 // processEvents processes all events, blocking only if a block processing is
 // happening and indexing should be suspended.
 func (f *FilterMaps) processEvents() {
-	for !f.stop && f.processSingleEvent(f.blockProcessing) {
+	for f.processSingleEvent(f.blockProcessing) {
 	}
 }
 
 // processSingleEvent processes a single event either in a blocking or
-// non-blocking manner.
+// non-blocking manner. It returns true if it did process an event.
 func (f *FilterMaps) processSingleEvent(blocking bool) bool {
+	if f.stop {
+		return false
+	}
 	if !f.hasTempRange {
 		for _, mb := range f.matcherSyncRequests {
 			mb.synced()
@@ -182,16 +221,16 @@ func (f *FilterMaps) setTarget(target targetUpdate) {
 	f.finalBlock = target.finalBlock
 }
 
-// tryIndexHead tries to render head maps according to the current targetView
+// tryIndexHead tries to render head maps according to the current targetView.
-// and returns true if successful.
+// Should be called when targetHeadIndexed returns false. If this function
-func (f *FilterMaps) tryIndexHead() bool {
+// returns no error then either stop is true or head indexing is finished.
+func (f *FilterMaps) tryIndexHead() error {
 	headRenderer, err := f.renderMapsBefore(math.MaxUint32)
 	if err != nil {
-		log.Error("Error creating log index head renderer", "error", err)
+		return err
-		return false
 	}
 	if headRenderer == nil {
-		return true
+		return errors.New("head indexer has nothing to do") // tryIndexHead should be called when head is not indexed
 	}
 	if !f.startedHeadIndex {
 		f.lastLogHeadIndex = time.Now()
@@ -216,8 +255,7 @@ func (f *FilterMaps) tryIndexHead() bool {
 			f.lastLogHeadIndex = time.Now()
 		}
 	}); err != nil {
-		log.Error("Log index head rendering failed", "error", err)
+		return err
-		return false
 	}
 	if f.loggedHeadIndex && f.indexedRange.hasIndexedBlocks() {
 		log.Info("Log index head rendering finished",
@@ -226,7 +264,7 @@ func (f *FilterMaps) tryIndexHead() bool {
 			"elapsed", common.PrettyDuration(time.Since(f.startedHeadIndexAt)))
 	}
 	f.loggedHeadIndex, f.startedHeadIndex = false, false
-	return true
+	return nil
 }
 
 // tryIndexTail tries to render tail epochs until the tail target block is
@@ -234,7 +272,7 @@ func (f *FilterMaps) tryIndexHead() bool {
 // Note that tail indexing is only started if the log index head is fully
 // rendered according to targetView and is suspended as soon as the targetView
 // is changed.
-func (f *FilterMaps) tryIndexTail() bool {
+func (f *FilterMaps) tryIndexTail() (bool, error) {
 	for {
 		firstEpoch := f.indexedRange.maps.First() >> f.logMapsPerEpoch
 		if firstEpoch == 0 || !f.needTailEpoch(firstEpoch-1) {
@@ -242,7 +280,7 @@ func (f *FilterMaps) tryIndexTail() bool {
 		}
 		f.processEvents()
 		if f.stop || !f.targetHeadIndexed() {
-			return false
+			return false, nil
 		}
 		// resume process if tail rendering was interrupted because of head rendering
 		tailRenderer := f.tailRenderer
@@ -254,8 +292,7 @@ func (f *FilterMaps) tryIndexTail() bool {
 			var err error
 			tailRenderer, err = f.renderMapsBefore(f.indexedRange.maps.First())
 			if err != nil {
-				log.Error("Error creating log index tail renderer", "error", err)
+				return false, err
-				return false
 			}
 		}
 		if tailRenderer == nil {
@@ -288,13 +325,16 @@ func (f *FilterMaps) tryIndexTail() bool {
 				f.lastLogTailIndex = time.Now()
 			}
 		})
-		if err != nil && f.needTailEpoch(firstEpoch-1) {
+		if err != nil && !f.needTailEpoch(firstEpoch-1) {
 			// stop silently if cutoff point has move beyond epoch boundary while rendering
-			log.Error("Log index tail rendering failed", "error", err)
+			return true, nil
+		}
+		if err != nil {
+			return false, err
 		}
 		if !done {
 			f.tailRenderer = tailRenderer // only keep tail renderer if interrupted by stopCb
-			return false
+			return false, nil
 		}
 	}
 	if f.loggedTailIndex && f.indexedRange.hasIndexedBlocks() {
@@ -304,32 +344,31 @@ func (f *FilterMaps) tryIndexTail() bool {
 			"elapsed", common.PrettyDuration(time.Since(f.startedTailIndexAt)))
 		f.loggedTailIndex = false
 	}
-	return true
+	return true, nil
 }
 
 // tryUnindexTail removes entire epochs of log index data as long as the first
 // fully indexed block is at least as old as the tail target.
 // Note that unindexing is very quick as it only removes continuous ranges of
 // data from the database and is also called while running head indexing.
-func (f *FilterMaps) tryUnindexTail() bool {
+func (f *FilterMaps) tryUnindexTail() (bool, error) {
-	for {
+	firstEpoch := f.indexedRange.maps.First() >> f.logMapsPerEpoch
-		firstEpoch := (f.indexedRange.maps.First() - f.indexedRange.tailPartialEpoch) >> f.logMapsPerEpoch
+	if f.indexedRange.tailPartialEpoch > 0 && firstEpoch > 0 {
-		if f.needTailEpoch(firstEpoch) {
+		firstEpoch--
-			break
-		}
-		f.processEvents()
-		if f.stop {
-			return false
 	}
+	for epoch := min(firstEpoch, f.cleanedEpochsBefore); !f.needTailEpoch(epoch); epoch++ {
 		if !f.startedTailUnindex {
 			f.startedTailUnindexAt = time.Now()
 			f.startedTailUnindex = true
 			f.ptrTailUnindexMap = f.indexedRange.maps.First() - f.indexedRange.tailPartialEpoch
 			f.ptrTailUnindexBlock = f.indexedRange.blocks.First() - f.tailPartialBlocks()
 		}
-		if err := f.deleteTailEpoch(firstEpoch); err != nil {
+		if done, err := f.deleteTailEpoch(epoch); !done {
-			log.Error("Log index tail epoch unindexing failed", "error", err)
+			return false, err
-			return false
+		}
+		f.processEvents()
+		if f.stop || !f.targetHeadIndexed() {
+			return false, nil
 		}
 	}
 	if f.startedTailUnindex && f.indexedRange.hasIndexedBlocks() {
@@ -340,7 +379,7 @@ func (f *FilterMaps) tryUnindexTail() bool {
 			"elapsed", common.PrettyDuration(time.Since(f.startedTailUnindexAt)))
 		f.startedTailUnindex = false
 	}
-	return true
+	return true, nil
 }
 
 // needTailEpoch returns true if the given tail epoch needs to be kept
@@ -356,16 +395,18 @@ func (f *FilterMaps) needTailEpoch(epoch uint32) bool {
 	if epoch+1 < firstEpoch {
 		return false
 	}
+	var lastBlockOfPrevEpoch uint64
 	if epoch > 0 {
-		lastBlockOfPrevEpoch, _, err := f.getLastBlockOfMap(epoch<<f.logMapsPerEpoch - 1)
+		var err error
+		lastBlockOfPrevEpoch, _, err = f.getLastBlockOfMap(epoch<<f.logMapsPerEpoch - 1)
 		if err != nil {
 			log.Error("Could not get last block of previous epoch", "epoch", epoch-1, "error", err)
 			return epoch >= firstEpoch
 		}
+	}
 	if f.historyCutoff > lastBlockOfPrevEpoch {
 		return false
 	}
-	}
 	lastBlockOfEpoch, _, err := f.getLastBlockOfMap((epoch+1)<<f.logMapsPerEpoch - 1)
 	if err != nil {
 		log.Error("Could not get last block of epoch", "epoch", epoch, "error", err)

core/filtermaps/map_renderer.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"math"
 	"sort"
+	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/lru"
@@ -301,6 +302,11 @@ func (r *mapRenderer) run(stopCb func() bool, writeCb func()) (bool, error) {
 
 // renderCurrentMap renders a single map.
 func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) {
+	var (
+		totalTime                           time.Duration
+		logValuesProcessed, blocksProcessed int64
+	)
+	start := time.Now()
 	if !r.iterator.updateChainView(r.f.targetView) {
 		return false, errChainUpdate
 	}
@@ -316,9 +322,11 @@ func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) {
 	for r.iterator.lvIndex < uint64(r.currentMap.mapIndex+1)<<r.f.logValuesPerMap && !r.iterator.finished {
 		waitCnt++
 		if waitCnt >= valuesPerCallback {
+			totalTime += time.Since(start)
 			if stopCb() {
 				return false, nil
 			}
+			start = time.Now()
 			if !r.iterator.updateChainView(r.f.targetView) {
 				return false, errChainUpdate
 			}
@@ -343,8 +351,10 @@ func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) {
 			return false, fmt.Errorf("failed to advance log iterator at %d while rendering map %d: %v", r.iterator.lvIndex, r.currentMap.mapIndex, err)
 		}
 		if !r.iterator.skipToBoundary {
+			logValuesProcessed++
 			r.currentMap.lastBlock = r.iterator.blockNumber
 			if r.iterator.blockStart {
+				blocksProcessed++
 				r.currentMap.blockLvPtrs = append(r.currentMap.blockLvPtrs, r.iterator.lvIndex)
 			}
 			if !r.f.testDisableSnapshots && r.renderBefore >= r.f.indexedRange.maps.AfterLast() &&
@@ -358,12 +368,18 @@ func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) {
 		r.currentMap.headDelimiter = r.iterator.lvIndex
 	}
 	r.currentMap.lastBlockId = r.f.targetView.getBlockId(r.currentMap.lastBlock)
+	totalTime += time.Since(start)
+	mapRenderTimer.Update(totalTime)
+	mapLogValueMeter.Mark(logValuesProcessed)
+	mapBlockMeter.Mark(blocksProcessed)
 	return true, nil
 }
 
 // writeFinishedMaps writes rendered maps to the database and updates
 // filterMapsRange and indexedView accordingly.
 func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error {
+	var totalTime time.Duration
+	start := time.Now()
 	if len(r.finishedMaps) == 0 {
 		return nil
 	}
@@ -379,7 +395,7 @@ func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error {
 	if err != nil {
 		return fmt.Errorf("failed to get updated rendered range: %v", err)
 	}
-	renderedView := r.f.targetView // stopCb callback might still change targetView while writing finished maps
+	renderedView := r.f.targetView // pauseCb callback might still change targetView while writing finished maps
 
 	batch := r.f.db.NewBatch()
 	var writeCnt int
@@ -393,7 +409,9 @@ func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error {
 			// do not exit while in partially written state but do allow processing
 			// events and pausing while block processing is in progress
 			r.f.indexLock.Unlock()
+			totalTime += time.Since(start)
 			pauseCb()
+			start = time.Now()
 			r.f.indexLock.Lock()
 			batch = r.f.db.NewBatch()
 		}
@@ -477,6 +495,8 @@ func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error {
 	if err := batch.Write(); err != nil {
 		log.Crit("Error writing log index update batch", "error", err)
 	}
+	totalTime += time.Since(start)
+	mapWriteTimer.Update(totalTime)
 	return nil
 }
 

core/filtermaps/matcher.go

@@ -125,6 +125,7 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock
 
 	start := time.Now()
 	res, err := m.process()
+	matchRequestTimer.Update(time.Since(start))
 
 	if doRuntimeStats {
 		log.Info("Log search finished", "elapsed", time.Since(start))
@@ -202,6 +203,7 @@ func (m *matcherEnv) process() ([]*types.Log, error) {
 			logs = append(logs, tasks[waitEpoch].logs...)
 			if err := tasks[waitEpoch].err; err != nil {
 				if err == ErrMatchAll {
+					matchAllMeter.Mark(1)
 					return logs, err
 				}
 				return logs, fmt.Errorf("failed to process log index epoch %d: %v", waitEpoch, err)
@@ -220,6 +222,7 @@ func (m *matcherEnv) process() ([]*types.Log, error) {
 
 // processEpoch returns the potentially matching logs from the given epoch.
 func (m *matcherEnv) processEpoch(epochIndex uint32) ([]*types.Log, error) {
+	start := time.Now()
 	var logs []*types.Log
 	// create a list of map indices to process
 	fm, lm := epochIndex<<m.params.logMapsPerEpoch, (epochIndex+1)<<m.params.logMapsPerEpoch-1
@@ -254,6 +257,7 @@ func (m *matcherEnv) processEpoch(epochIndex uint32) ([]*types.Log, error) {
 		logs = append(logs, mlogs...)
 	}
 	m.getLogStats.addAmount(st, int64(len(logs)))
+	matchEpochTimer.Update(time.Since(start))
 	return logs, nil
 }
 
@@ -273,6 +277,7 @@ func (m *matcherEnv) getLogsFromMatches(matches potentialMatches) ([]*types.Log,
 		if log != nil {
 			logs = append(logs, log)
 		}
+		matchLogLookup.Mark(1)
 	}
 	return logs, nil
 }
@@ -381,6 +386,13 @@ func (m *singleMatcherInstance) getMatchesForLayer(ctx context.Context, layerInd
 			m.stats.setState(&st, stNone)
 			return nil, fmt.Errorf("failed to retrieve filter map %d row %d: %v", mapIndex, rowIndex, err)
 		}
+		if layerIndex == 0 {
+			matchBaseRowAccessMeter.Mark(1)
+			matchBaseRowSizeMeter.Mark(int64(len(filterRow)))
+		} else {
+			matchExtRowAccessMeter.Mark(1)
+			matchExtRowSizeMeter.Mark(int64(len(filterRow)))
+		}
 		m.stats.addAmount(st, int64(len(filterRow)))
 		m.stats.setState(&st, stOther)
 		filterRows = append(filterRows, filterRow)

core/filtermaps/matcher_backend.go

@@ -141,10 +141,6 @@ func (fm *FilterMapsMatcherBackend) synced() {
 // range that has not been changed and has been consistent with all states of the
 // chain since the previous SyncLogIndex or the creation of the matcher backend.
 func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange, error) {
-	if fm.f.disabled {
-		return SyncRange{HeadNumber: fm.f.targetView.headNumber}, nil
-	}
-
 	syncCh := make(chan SyncRange, 1)
 	fm.f.matchersLock.Lock()
 	fm.syncCh = syncCh
@@ -154,12 +150,16 @@ func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange
 	case fm.f.matcherSyncCh <- fm:
 	case <-ctx.Done():
 		return SyncRange{}, ctx.Err()
+	case <-fm.f.disabledCh:
+		return SyncRange{HeadNumber: fm.f.targetView.headNumber}, nil
 	}
 	select {
 	case vr := <-syncCh:
 		return vr, nil
 	case <-ctx.Done():
 		return SyncRange{}, ctx.Err()
+	case <-fm.f.disabledCh:
+		return SyncRange{HeadNumber: fm.f.targetView.headNumber}, nil
 	}
 }
 

core/rawdb/accessors_chain.go

@@ -277,6 +277,14 @@ func WriteTxIndexTail(db ethdb.KeyValueWriter, number uint64) {
 	}
 }
 
+// DeleteTxIndexTail deletes the number of oldest indexed block
+// from database.
+func DeleteTxIndexTail(db ethdb.KeyValueWriter) {
+	if err := db.Delete(txIndexTailKey); err != nil {
+		log.Crit("Failed to delete the transaction index tail", "err", err)
+	}
+}
+
 // ReadHeaderRange returns the rlp-encoded headers, starting at 'number', and going
 // backwards towards genesis. This method assumes that the caller already has
 // placed a cap on count, to prevent DoS issues.

core/rawdb/accessors_indexes.go

@@ -30,13 +30,8 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
-// ReadTxLookupEntry retrieves the positional metadata associated with a transaction
+// DecodeTxLookupEntry decodes the supplied tx lookup data.
-// hash to allow retrieving the transaction or receipt by hash.
+func DecodeTxLookupEntry(data []byte, db ethdb.Reader) *uint64 {
-func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 {
-	data, _ := db.Get(txLookupKey(hash))
-	if len(data) == 0 {
-		return nil
-	}
 	// Database v6 tx lookup just stores the block number
 	if len(data) < common.HashLength {
 		number := new(big.Int).SetBytes(data).Uint64()
@@ -49,12 +44,22 @@ func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 {
 	// Finally try database v3 tx lookup format
 	var entry LegacyTxLookupEntry
 	if err := rlp.DecodeBytes(data, &entry); err != nil {
-		log.Error("Invalid transaction lookup entry RLP", "hash", hash, "blob", data, "err", err)
+		log.Error("Invalid transaction lookup entry RLP", "blob", data, "err", err)
 		return nil
 	}
 	return &entry.BlockIndex
 }
 
+// ReadTxLookupEntry retrieves the positional metadata associated with a transaction
+// hash to allow retrieving the transaction or receipt by hash.
+func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 {
+	data, _ := db.Get(txLookupKey(hash))
+	if len(data) == 0 {
+		return nil
+	}
+	return DecodeTxLookupEntry(data, db)
+}
+
 // writeTxLookupEntry stores a positional metadata for a transaction,
 // enabling hash based transaction and receipt lookups.
 func writeTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash, numberBytes []byte) {
@@ -95,6 +100,34 @@ func DeleteTxLookupEntries(db ethdb.KeyValueWriter, hashes []common.Hash) {
 	}
 }
 
+// DeleteAllTxLookupEntries purges all the transaction indexes in the database.
+// If condition is specified, only the entry with condition as True will be
+// removed; If condition is not specified, the entry is deleted.
+func DeleteAllTxLookupEntries(db ethdb.KeyValueStore, condition func(common.Hash, []byte) bool) {
+	iter := NewKeyLengthIterator(db.NewIterator(txLookupPrefix, nil), common.HashLength+len(txLookupPrefix))
+	defer iter.Release()
+
+	batch := db.NewBatch()
+	for iter.Next() {
+		txhash := common.Hash(iter.Key()[1:])
+		if condition == nil || condition(txhash, iter.Value()) {
+			batch.Delete(iter.Key())
+		}
+		if batch.ValueSize() >= ethdb.IdealBatchSize {
+			if err := batch.Write(); err != nil {
+				log.Crit("Failed to delete transaction lookup entries", "err", err)
+			}
+			batch.Reset()
+		}
+	}
+	if batch.ValueSize() > 0 {
+		if err := batch.Write(); err != nil {
+			log.Crit("Failed to delete transaction lookup entries", "err", err)
+		}
+		batch.Reset()
+	}
+}
+
 // ReadTransaction retrieves a specific transaction from the database, along with
 // its added positional metadata.
 func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) {
@@ -147,41 +180,6 @@ func ReadReceipt(db ethdb.Reader, hash common.Hash, config *params.ChainConfig)
 	return nil, common.Hash{}, 0, 0
 }
 
-// ReadBloomBits retrieves the compressed bloom bit vector belonging to the given
-// section and bit index from the.
-func ReadBloomBits(db ethdb.KeyValueReader, bit uint, section uint64, head common.Hash) ([]byte, error) {
-	return db.Get(bloomBitsKey(bit, section, head))
-}
-
-// WriteBloomBits stores the compressed bloom bits vector belonging to the given
-// section and bit index.
-func WriteBloomBits(db ethdb.KeyValueWriter, bit uint, section uint64, head common.Hash, bits []byte) {
-	if err := db.Put(bloomBitsKey(bit, section, head), bits); err != nil {
-		log.Crit("Failed to store bloom bits", "err", err)
-	}
-}
-
-// DeleteBloombits removes all compressed bloom bits vector belonging to the
-// given section range and bit index.
-func DeleteBloombits(db ethdb.Database, bit uint, from uint64, to uint64) {
-	start, end := bloomBitsKey(bit, from, common.Hash{}), bloomBitsKey(bit, to, common.Hash{})
-	it := db.NewIterator(nil, start)
-	defer it.Release()
-
-	for it.Next() {
-		if bytes.Compare(it.Key(), end) >= 0 {
-			break
-		}
-		if len(it.Key()) != len(bloomBitsPrefix)+2+8+32 {
-			continue
-		}
-		db.Delete(it.Key())
-	}
-	if it.Error() != nil {
-		log.Crit("Failed to delete bloom bits", "err", it.Error())
-	}
-}
-
 // ReadFilterMapRow retrieves a filter map row at the given mapRowIndex
 // (see filtermaps.mapRowIndex for the storage index encoding).
 // Note that zero length rows are not stored in the database and therefore all
@@ -356,10 +354,8 @@ func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows []
 	}
 }
 
-func DeleteFilterMapRows(db ethdb.KeyValueRangeDeleter, mapRows common.Range[uint64]) {
+func DeleteFilterMapRows(db ethdb.KeyValueStore, mapRows common.Range[uint64], hashScheme bool, stopCallback func(bool) bool) error {
-	if err := db.DeleteRange(filterMapRowKey(mapRows.First(), false), filterMapRowKey(mapRows.AfterLast(), false)); err != nil {
+	return SafeDeleteRange(db, filterMapRowKey(mapRows.First(), false), filterMapRowKey(mapRows.AfterLast(), false), hashScheme, stopCallback)
-		log.Crit("Failed to delete range of filter map rows", "err", err)
-	}
 }
 
 // ReadFilterMapLastBlock retrieves the number of the block that generated the
@@ -370,7 +366,7 @@ func ReadFilterMapLastBlock(db ethdb.KeyValueReader, mapIndex uint32) (uint64, c
 		return 0, common.Hash{}, err
 	}
 	if len(enc) != 40 {
-		return 0, common.Hash{}, errors.New("Invalid block number and id encoding")
+		return 0, common.Hash{}, errors.New("invalid block number and id encoding")
 	}
 	var id common.Hash
 	copy(id[:], enc[8:])
@@ -396,10 +392,8 @@ func DeleteFilterMapLastBlock(db ethdb.KeyValueWriter, mapIndex uint32) {
 	}
 }
 
-func DeleteFilterMapLastBlocks(db ethdb.KeyValueRangeDeleter, maps common.Range[uint32]) {
+func DeleteFilterMapLastBlocks(db ethdb.KeyValueStore, maps common.Range[uint32], hashScheme bool, stopCallback func(bool) bool) error {
-	if err := db.DeleteRange(filterMapLastBlockKey(maps.First()), filterMapLastBlockKey(maps.AfterLast())); err != nil {
+	return SafeDeleteRange(db, filterMapLastBlockKey(maps.First()), filterMapLastBlockKey(maps.AfterLast()), hashScheme, stopCallback)
-		log.Crit("Failed to delete range of filter map last block pointers", "err", err)
-	}
 }
 
 // ReadBlockLvPointer retrieves the starting log value index where the log values
@@ -410,7 +404,7 @@ func ReadBlockLvPointer(db ethdb.KeyValueReader, blockNumber uint64) (uint64, er
 		return 0, err
 	}
 	if len(encPtr) != 8 {
-		return 0, errors.New("Invalid log value pointer encoding")
+		return 0, errors.New("invalid log value pointer encoding")
 	}
 	return binary.BigEndian.Uint64(encPtr), nil
 }
@@ -433,10 +427,8 @@ func DeleteBlockLvPointer(db ethdb.KeyValueWriter, blockNumber uint64) {
 	}
 }
 
-func DeleteBlockLvPointers(db ethdb.KeyValueRangeDeleter, blocks common.Range[uint64]) {
+func DeleteBlockLvPointers(db ethdb.KeyValueStore, blocks common.Range[uint64], hashScheme bool, stopCallback func(bool) bool) error {
-	if err := db.DeleteRange(filterMapBlockLVKey(blocks.First()), filterMapBlockLVKey(blocks.AfterLast())); err != nil {
+	return SafeDeleteRange(db, filterMapBlockLVKey(blocks.First()), filterMapBlockLVKey(blocks.AfterLast()), hashScheme, stopCallback)
-		log.Crit("Failed to delete range of block log value pointers", "err", err)
-	}
 }
 
 // FilterMapsRange is a storage representation of the block range covered by the
@@ -485,3 +477,24 @@ func DeleteFilterMapsRange(db ethdb.KeyValueWriter) {
 		log.Crit("Failed to delete filter maps range", "err", err)
 	}
 }
+
+// deletePrefixRange deletes everything with the given prefix from the database.
+func deletePrefixRange(db ethdb.KeyValueStore, prefix []byte, hashScheme bool, stopCallback func(bool) bool) error {
+	end := bytes.Clone(prefix)
+	end[len(end)-1]++
+	return SafeDeleteRange(db, prefix, end, hashScheme, stopCallback)
+}
+
+// DeleteFilterMapsDb removes the entire filter maps database
+func DeleteFilterMapsDb(db ethdb.KeyValueStore, hashScheme bool, stopCallback func(bool) bool) error {
+	return deletePrefixRange(db, []byte(filterMapsPrefix), hashScheme, stopCallback)
+}
+
+// DeleteBloomBitsDb removes the old bloombits database and the associated
+// chain indexer database.
+func DeleteBloomBitsDb(db ethdb.KeyValueStore, hashScheme bool, stopCallback func(bool) bool) error {
+	if err := deletePrefixRange(db, bloomBitsPrefix, hashScheme, stopCallback); err != nil {
+		return err
+	}
+	return deletePrefixRange(db, bloomBitsMetaPrefix, hashScheme, stopCallback)
+}

core/rawdb/accessors_indexes_test.go

@@ -17,7 +17,6 @@
 package rawdb
 
 import (
-	"bytes"
 	"math/big"
 	"testing"
 
@@ -25,7 +24,6 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/internal/blocktest"
-	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
@@ -111,54 +109,3 @@ func TestLookupStorage(t *testing.T) {
 		})
 	}
 }
-
-func TestDeleteBloomBits(t *testing.T) {
-	// Prepare testing data
-	db := NewMemoryDatabase()
-	for i := uint(0); i < 2; i++ {
-		for s := uint64(0); s < 2; s++ {
-			WriteBloomBits(db, i, s, params.MainnetGenesisHash, []byte{0x01, 0x02})
-			WriteBloomBits(db, i, s, params.SepoliaGenesisHash, []byte{0x01, 0x02})
-			WriteBloomBits(db, i, s, params.HoodiGenesisHash, []byte{0x01, 0x02})
-		}
-	}
-	check := func(bit uint, section uint64, head common.Hash, exist bool) {
-		bits, _ := ReadBloomBits(db, bit, section, head)
-		if exist && !bytes.Equal(bits, []byte{0x01, 0x02}) {
-			t.Fatalf("Bloombits mismatch")
-		}
-		if !exist && len(bits) > 0 {
-			t.Fatalf("Bloombits should be removed")
-		}
-	}
-	// Check the existence of written data.
-	check(0, 0, params.MainnetGenesisHash, true)
-	check(0, 0, params.SepoliaGenesisHash, true)
-	check(0, 0, params.HoodiGenesisHash, true)
-
-	// Check the existence of deleted data.
-	DeleteBloombits(db, 0, 0, 1)
-	check(0, 0, params.MainnetGenesisHash, false)
-	check(0, 0, params.SepoliaGenesisHash, false)
-	check(0, 0, params.HoodiGenesisHash, false)
-	check(0, 1, params.MainnetGenesisHash, true)
-	check(0, 1, params.SepoliaGenesisHash, true)
-	check(0, 1, params.HoodiGenesisHash, true)
-
-	// Check the existence of deleted data.
-	DeleteBloombits(db, 0, 0, 2)
-	check(0, 0, params.MainnetGenesisHash, false)
-	check(0, 0, params.SepoliaGenesisHash, false)
-	check(0, 0, params.HoodiGenesisHash, false)
-	check(0, 1, params.MainnetGenesisHash, false)
-	check(0, 1, params.SepoliaGenesisHash, false)
-	check(0, 1, params.HoodiGenesisHash, false)
-
-	// Bit1 shouldn't be affect.
-	check(1, 0, params.MainnetGenesisHash, true)
-	check(1, 0, params.SepoliaGenesisHash, true)
-	check(1, 0, params.HoodiGenesisHash, true)
-	check(1, 1, params.MainnetGenesisHash, true)
-	check(1, 1, params.SepoliaGenesisHash, true)
-	check(1, 1, params.HoodiGenesisHash, true)
-}

core/rawdb/chain_iterator.go

@@ -17,6 +17,7 @@
 package rawdb
 
 import (
+	"encoding/binary"
 	"runtime"
 	"sync/atomic"
 	"time"
@@ -361,3 +362,38 @@ func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch
 func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) {
 	unindexTransactions(db, from, to, interrupt, hook, false)
 }
+
+// PruneTransactionIndex removes all tx index entries below a certain block number.
+func PruneTransactionIndex(db ethdb.Database, pruneBlock uint64) {
+	tail := ReadTxIndexTail(db)
+	if tail == nil || *tail > pruneBlock {
+		return // no index, or index ends above pruneBlock
+	}
+	// There are blocks below pruneBlock in the index. Iterate the entire index to remove
+	// their entries. Note if this fails, the index is messed up, but tail still points to
+	// the old tail.
+	var count, removed int
+	DeleteAllTxLookupEntries(db, func(txhash common.Hash, v []byte) bool {
+		count++
+		if count%10000000 == 0 {
+			log.Info("Pruning tx index", "count", count, "removed", removed)
+		}
+		if len(v) > 8 {
+			log.Error("Skipping legacy tx index entry", "hash", txhash)
+			return false
+		}
+		bn := decodeNumber(v)
+		if bn < pruneBlock {
+			removed++
+			return true
+		}
+		return false
+	})
+	WriteTxIndexTail(db, pruneBlock)
+}
+
+func decodeNumber(b []byte) uint64 {
+	var numBuffer [8]byte
+	copy(numBuffer[8-len(b):], b)
+	return binary.BigEndian.Uint64(numBuffer[:])
+}

core/rawdb/chain_iterator_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
 )
 
 func TestChainIterator(t *testing.T) {
@@ -102,19 +103,18 @@ func TestChainIterator(t *testing.T) {
 	}
 }
 
-func TestIndexTransactions(t *testing.T) {
+func initDatabaseWithTransactions(db ethdb.Database) ([]*types.Block, []*types.Transaction) {
-	// Construct test chain db
+	var blocks []*types.Block
-	chainDb := NewMemoryDatabase()
-
-	var block *types.Block
 	var txs []*types.Transaction
 	to := common.BytesToAddress([]byte{0x11})
 
 	// Write empty genesis block
-	block = types.NewBlock(&types.Header{Number: big.NewInt(int64(0))}, nil, nil, newTestHasher())
+	block := types.NewBlock(&types.Header{Number: big.NewInt(int64(0))}, nil, nil, newTestHasher())
-	WriteBlock(chainDb, block)
+	WriteBlock(db, block)
-	WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64())
+	WriteCanonicalHash(db, block.Hash(), block.NumberU64())
+	blocks = append(blocks, block)
 
+	// Create transactions.
 	for i := uint64(1); i <= 10; i++ {
 		var tx *types.Transaction
 		if i%2 == 0 {
@@ -138,10 +138,21 @@ func TestIndexTransactions(t *testing.T) {
 			})
 		}
 		txs = append(txs, tx)
-		block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, &types.Body{Transactions: types.Transactions{tx}}, nil, newTestHasher())
+		block := types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, &types.Body{Transactions: types.Transactions{tx}}, nil, newTestHasher())
-		WriteBlock(chainDb, block)
+		WriteBlock(db, block)
-		WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64())
+		WriteCanonicalHash(db, block.Hash(), block.NumberU64())
+		blocks = append(blocks, block)
 	}
+
+	return blocks, txs
+}
+
+func TestIndexTransactions(t *testing.T) {
+	// Construct test chain db
+	chainDB := NewMemoryDatabase()
+
+	_, txs := initDatabaseWithTransactions(chainDB)
+
 	// verify checks whether the tx indices in the range [from, to)
 	// is expected.
 	verify := func(from, to int, exist bool, tail uint64) {
@@ -149,7 +160,7 @@ func TestIndexTransactions(t *testing.T) {
 			if i == 0 {
 				continue
 			}
-			number := ReadTxLookupEntry(chainDb, txs[i-1].Hash())
+			number := ReadTxLookupEntry(chainDB, txs[i-1].Hash())
 			if exist && number == nil {
 				t.Fatalf("Transaction index %d missing", i)
 			}
@@ -157,29 +168,29 @@ func TestIndexTransactions(t *testing.T) {
 				t.Fatalf("Transaction index %d is not deleted", i)
 			}
 		}
-		number := ReadTxIndexTail(chainDb)
+		number := ReadTxIndexTail(chainDB)
 		if number == nil || *number != tail {
 			t.Fatalf("Transaction tail mismatch")
 		}
 	}
-	IndexTransactions(chainDb, 5, 11, nil, false)
+	IndexTransactions(chainDB, 5, 11, nil, false)
 	verify(5, 11, true, 5)
 	verify(0, 5, false, 5)
 
-	IndexTransactions(chainDb, 0, 5, nil, false)
+	IndexTransactions(chainDB, 0, 5, nil, false)
 	verify(0, 11, true, 0)
 
-	UnindexTransactions(chainDb, 0, 5, nil, false)
+	UnindexTransactions(chainDB, 0, 5, nil, false)
 	verify(5, 11, true, 5)
 	verify(0, 5, false, 5)
 
-	UnindexTransactions(chainDb, 5, 11, nil, false)
+	UnindexTransactions(chainDB, 5, 11, nil, false)
 	verify(0, 11, false, 11)
 
 	// Testing corner cases
 	signal := make(chan struct{})
 	var once sync.Once
-	indexTransactionsForTesting(chainDb, 5, 11, signal, func(n uint64) bool {
+	indexTransactionsForTesting(chainDB, 5, 11, signal, func(n uint64) bool {
 		if n <= 8 {
 			once.Do(func() {
 				close(signal)
@@ -190,11 +201,11 @@ func TestIndexTransactions(t *testing.T) {
 	})
 	verify(9, 11, true, 9)
 	verify(0, 9, false, 9)
-	IndexTransactions(chainDb, 0, 9, nil, false)
+	IndexTransactions(chainDB, 0, 9, nil, false)
 
 	signal = make(chan struct{})
 	var once2 sync.Once
-	unindexTransactionsForTesting(chainDb, 0, 11, signal, func(n uint64) bool {
+	unindexTransactionsForTesting(chainDB, 0, 11, signal, func(n uint64) bool {
 		if n >= 8 {
 			once2.Do(func() {
 				close(signal)
@@ -206,3 +217,37 @@ func TestIndexTransactions(t *testing.T) {
 	verify(8, 11, true, 8)
 	verify(0, 8, false, 8)
 }
+
+func TestPruneTransactionIndex(t *testing.T) {
+	chainDB := NewMemoryDatabase()
+	blocks, _ := initDatabaseWithTransactions(chainDB)
+	lastBlock := blocks[len(blocks)-1].NumberU64()
+	pruneBlock := lastBlock - 3
+
+	IndexTransactions(chainDB, 0, lastBlock+1, nil, false)
+
+	// Check all transactions are in index.
+	for _, block := range blocks {
+		for _, tx := range block.Transactions() {
+			num := ReadTxLookupEntry(chainDB, tx.Hash())
+			if num == nil || *num != block.NumberU64() {
+				t.Fatalf("wrong TxLookup entry: %x -> %v", tx.Hash(), num)
+			}
+		}
+	}
+
+	PruneTransactionIndex(chainDB, pruneBlock)
+
+	// Check transactions from old blocks not included.
+	for _, block := range blocks {
+		for _, tx := range block.Transactions() {
+			num := ReadTxLookupEntry(chainDB, tx.Hash())
+			if block.NumberU64() < pruneBlock && num != nil {
+				t.Fatalf("TxLookup entry not removed: %x -> %v", tx.Hash(), num)
+			}
+			if block.NumberU64() >= pruneBlock && (num == nil || *num != block.NumberU64()) {
+				t.Fatalf("wrong TxLookup entry after pruning: %x -> %v", tx.Hash(), num)
+			}
+		}
+	}
+}

core/rawdb/database.go

@@ -20,18 +20,23 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"maps"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/ethdb/memorydb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/olekukonko/tablewriter"
 )
 
+var ErrDeleteRangeInterrupted = errors.New("safe delete range operation interrupted")
+
 // freezerdb is a database wrapper that enables ancient chain segment freezing.
 type freezerdb struct {
 	ethdb.KeyValueStore
@@ -375,25 +380,28 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		accountSnaps       stat
 		storageSnaps       stat
 		preimages          stat
-		bloomBits       stat
-		filterMaps      stat
 		beaconHeaders      stat
 		cliqueSnaps        stat
+		bloomBits          stat
+		filterMapRows      stat
+		filterMapLastBlock stat
+		filterMapBlockLV   stat
 
 		// Verkle statistics
 		verkleTries        stat
 		verkleStateLookups stat
 
-		// Les statistic
-		chtTrieNodes   stat
-		bloomTrieNodes stat
-
 		// Meta- and unaccounted data
 		metadata    stat
 		unaccounted stat
 
 		// Totals
 		total common.StorageSize
+
+		// This map tracks example keys for unaccounted data.
+		// For each unique two-byte prefix, the first unaccounted key encountered
+		// by the iterator will be stored.
+		unaccountedKeys = make(map[[2]byte][]byte)
 	)
 	// Inspect key-value database first.
 	for it.Next() {
@@ -437,24 +445,24 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			metadata.Add(size)
 		case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength):
 			metadata.Add(size)
-		case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
-			bloomBits.Add(size)
-		case bytes.HasPrefix(key, BloomBitsIndexPrefix):
-			bloomBits.Add(size)
-		case bytes.HasPrefix(key, []byte(FilterMapsPrefix)):
-			filterMaps.Add(size)
 		case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8):
 			beaconHeaders.Add(size)
 		case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength:
 			cliqueSnaps.Add(size)
-		case bytes.HasPrefix(key, ChtTablePrefix) ||
+
-			bytes.HasPrefix(key, ChtIndexTablePrefix) ||
+		// new log index
-			bytes.HasPrefix(key, ChtPrefix): // Canonical hash trie
+		case bytes.HasPrefix(key, filterMapRowPrefix) && len(key) <= len(filterMapRowPrefix)+9:
-			chtTrieNodes.Add(size)
+			filterMapRows.Add(size)
-		case bytes.HasPrefix(key, BloomTrieTablePrefix) ||
+		case bytes.HasPrefix(key, filterMapLastBlockPrefix) && len(key) == len(filterMapLastBlockPrefix)+4:
-			bytes.HasPrefix(key, BloomTrieIndexPrefix) ||
+			filterMapLastBlock.Add(size)
-			bytes.HasPrefix(key, BloomTriePrefix): // Bloomtrie sub
+		case bytes.HasPrefix(key, filterMapBlockLVPrefix) && len(key) == len(filterMapBlockLVPrefix)+8:
-			bloomTrieNodes.Add(size)
+			filterMapBlockLV.Add(size)
+
+		// old log index (deprecated)
+		case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
+			bloomBits.Add(size)
+		case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8:
+			bloomBits.Add(size)
 
 		// Verkle trie data is detected, determine the sub-category
 		case bytes.HasPrefix(key, VerklePrefix):
@@ -473,23 +481,18 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			default:
 				unaccounted.Add(size)
 			}
-		default:
+
-			var accounted bool
+		// Metadata keys
-			for _, meta := range [][]byte{
+		case slices.ContainsFunc(knownMetadataKeys, func(x []byte) bool { return bytes.Equal(x, key) }):
-				databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, headFinalizedBlockKey,
-				lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey,
-				snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey,
-				uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey,
-				persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey,
-			} {
-				if bytes.Equal(key, meta) {
 			metadata.Add(size)
-					accounted = true
+
-					break
+		default:
-				}
-			}
-			if !accounted {
 			unaccounted.Add(size)
+			if len(key) >= 2 {
+				prefix := [2]byte(key[:2])
+				if _, ok := unaccountedKeys[prefix]; !ok {
+					unaccountedKeys[prefix] = bytes.Clone(key)
+				}
 			}
 		}
 		count++
@@ -507,8 +510,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		{"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()},
 		{"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()},
 		{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
-		{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
+		{"Key-Value store", "Log index filter-map rows", filterMapRows.Size(), filterMapRows.Count()},
-		{"Key-Value store", "Log search index", filterMaps.Size(), filterMaps.Count()},
+		{"Key-Value store", "Log index last-block-of-map", filterMapLastBlock.Size(), filterMapLastBlock.Count()},
+		{"Key-Value store", "Log index block-lv", filterMapBlockLV.Size(), filterMapBlockLV.Count()},
+		{"Key-Value store", "Log bloombits (deprecated)", bloomBits.Size(), bloomBits.Count()},
 		{"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
 		{"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()},
 		{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
@@ -522,8 +527,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		{"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()},
 		{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
 		{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
-		{"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()},
-		{"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()},
 	}
 	// Inspect all registered append-only file store then.
 	ancients, err := inspectFreezers(db)
@@ -549,10 +552,23 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 
 	if unaccounted.size > 0 {
 		log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count)
+		for _, e := range slices.SortedFunc(maps.Values(unaccountedKeys), bytes.Compare) {
+			log.Error(fmt.Sprintf("   example key: %x", e))
+		}
 	}
 	return nil
 }
 
+// This is the list of known 'metadata' keys stored in the databasse.
+var knownMetadataKeys = [][]byte{
+	databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, headFinalizedBlockKey,
+	lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey,
+	snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey,
+	uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey,
+	persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey,
+	filterMapsRangeKey,
+}
+
 // printChainMetadata prints out chain metadata to stderr.
 func printChainMetadata(db ethdb.KeyValueStore) {
 	fmt.Fprintf(os.Stderr, "Chain metadata\n")
@@ -572,6 +588,7 @@ func ReadChainMetadata(db ethdb.KeyValueStore) [][]string {
 		}
 		return fmt.Sprintf("%d (%#x)", *val, *val)
 	}
+
 	data := [][]string{
 		{"databaseVersion", pp(ReadDatabaseVersion(db))},
 		{"headBlockHash", fmt.Sprintf("%v", ReadHeadBlockHash(db))},
@@ -588,5 +605,78 @@ func ReadChainMetadata(db ethdb.KeyValueStore) [][]string {
 	if b := ReadSkeletonSyncStatus(db); b != nil {
 		data = append(data, []string{"SkeletonSyncStatus", string(b)})
 	}
+	if fmr, ok, _ := ReadFilterMapsRange(db); ok {
+		data = append(data, []string{"filterMapsRange", fmt.Sprintf("%+v", fmr)})
+	}
 	return data
 }
+
+// SafeDeleteRange deletes all of the keys (and values) in the range
+// [start,end) (inclusive on start, exclusive on end).
+// If hashScheme is true then it always uses an iterator and skips hashdb trie
+// node entries. If it is false and the backing db is pebble db then it uses
+// the fast native range delete.
+// In case of fallback mode (hashdb or leveldb) the range deletion might be
+// very slow depending on the number of entries. In this case stopCallback
+// is periodically called and if it returns an error then SafeDeleteRange
+// stops and also returns that error. The callback is not called if native
+// range delete is used or there are a small number of keys only. The bool
+// argument passed to the callback is true if enrties have actually been
+// deleted already.
+func SafeDeleteRange(db ethdb.KeyValueStore, start, end []byte, hashScheme bool, stopCallback func(bool) bool) error {
+	if !hashScheme {
+		// delete entire range; use fast native range delete on pebble db
+		for {
+			switch err := db.DeleteRange(start, end); {
+			case err == nil:
+				return nil
+			case errors.Is(err, ethdb.ErrTooManyKeys):
+				if stopCallback(true) {
+					return ErrDeleteRangeInterrupted
+				}
+			default:
+				return err
+			}
+		}
+	}
+
+	var (
+		count, deleted, skipped int
+		buff                    = crypto.NewKeccakState()
+		startTime               = time.Now()
+	)
+
+	batch := db.NewBatch()
+	it := db.NewIterator(nil, start)
+	defer func() {
+		it.Release() // it might be replaced during the process
+		log.Debug("SafeDeleteRange finished", "deleted", deleted, "skipped", skipped, "elapsed", common.PrettyDuration(time.Since(startTime)))
+	}()
+
+	for it.Next() && bytes.Compare(end, it.Key()) > 0 {
+		// Prevent deletion for trie nodes in hash mode
+		if len(it.Key()) != 32 || crypto.HashData(buff, it.Value()) != common.BytesToHash(it.Key()) {
+			if err := batch.Delete(it.Key()); err != nil {
+				return err
+			}
+			deleted++
+		} else {
+			skipped++
+		}
+		count++
+		if count > 10000 { // should not block for more than a second
+			if err := batch.Write(); err != nil {
+				return err
+			}
+			if stopCallback(deleted != 0) {
+				return ErrDeleteRangeInterrupted
+			}
+			start = append(bytes.Clone(it.Key()), 0) // appending a zero gives us the next possible key
+			it.Release()
+			batch = db.NewBatch()
+			it = db.NewIterator(nil, start)
+			count = 0
+		}
+	}
+	return batch.Write()
+}

core/rawdb/schema.go

@@ -128,28 +128,21 @@ var (
 	configPrefix   = []byte("ethereum-config-")  // config prefix for the db
 	genesisPrefix  = []byte("ethereum-genesis-") // genesis state prefix for the db
 
-	// BloomBitsIndexPrefix is the data table of a chain indexer to track its progress
-	BloomBitsIndexPrefix = []byte("iB")
-
-	ChtPrefix           = []byte("chtRootV2-") // ChtPrefix + chtNum (uint64 big endian) -> trie root hash
-	ChtTablePrefix      = []byte("cht-")
-	ChtIndexTablePrefix = []byte("chtIndexV2-")
-
-	BloomTriePrefix      = []byte("bltRoot-") // BloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash
-	BloomTrieTablePrefix = []byte("blt-")
-	BloomTrieIndexPrefix = []byte("bltIndex-")
-
 	CliqueSnapshotPrefix = []byte("clique-")
 
 	BestUpdateKey         = []byte("update-")    // bigEndian64(syncPeriod) -> RLP(types.LightClientUpdate)  (nextCommittee only referenced by root hash)
 	FixedCommitteeRootKey = []byte("fixedRoot-") // bigEndian64(syncPeriod) -> committee root hash
 	SyncCommitteeKey      = []byte("committee-") // bigEndian64(syncPeriod) -> serialized committee
 
-	FilterMapsPrefix         = "fm-"
+	// new log index
-	filterMapsRangeKey       = []byte(FilterMapsPrefix + "R")
+	filterMapsPrefix         = "fm-"
-	filterMapRowPrefix       = []byte(FilterMapsPrefix + "r") // filterMapRowPrefix + mapRowIndex (uint64 big endian) -> filter row
+	filterMapsRangeKey       = []byte(filterMapsPrefix + "R")
-	filterMapLastBlockPrefix = []byte(FilterMapsPrefix + "b") // filterMapLastBlockPrefix + mapIndex (uint32 big endian) -> block number (uint64 big endian)
+	filterMapRowPrefix       = []byte(filterMapsPrefix + "r") // filterMapRowPrefix + mapRowIndex (uint64 big endian) -> filter row
-	filterMapBlockLVPrefix   = []byte(FilterMapsPrefix + "p") // filterMapBlockLVPrefix + num (uint64 big endian) -> log value pointer (uint64 big endian)
+	filterMapLastBlockPrefix = []byte(filterMapsPrefix + "b") // filterMapLastBlockPrefix + mapIndex (uint32 big endian) -> block number (uint64 big endian)
+	filterMapBlockLVPrefix   = []byte(filterMapsPrefix + "p") // filterMapBlockLVPrefix + num (uint64 big endian) -> log value pointer (uint64 big endian)
+
+	// old log index
+	bloomBitsMetaPrefix = []byte("iB")
 
 	preimageCounter     = metrics.NewRegisteredCounter("db/preimage/total", nil)
 	preimageHitsCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)
@@ -225,16 +218,6 @@ func storageSnapshotsKey(accountHash common.Hash) []byte {
 	return append(SnapshotStoragePrefix, accountHash.Bytes()...)
 }
 
-// bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash
-func bloomBitsKey(bit uint, section uint64, hash common.Hash) []byte {
-	key := append(append(bloomBitsPrefix, make([]byte, 10)...), hash.Bytes()...)
-
-	binary.BigEndian.PutUint16(key[1:], uint16(bit))
-	binary.BigEndian.PutUint64(key[3:], section)
-
-	return key
-}
-
 // skeletonHeaderKey = skeletonHeaderPrefix + num (uint64 big endian)
 func skeletonHeaderKey(number uint64) []byte {
 	return append(skeletonHeaderPrefix, encodeBlockNumber(number)...)

core/state/snapshot/generate_test.go

@@ -655,7 +655,7 @@ func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) {
 		for i := 0; i < 1000; i++ {
 			acc := &types.StateAccount{Balance: uint256.NewInt(uint64(i)), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()}
 			val, _ := rlp.EncodeToBytes(acc)
-			key := hashData([]byte(fmt.Sprintf("acc-%d", i)))
+			key := hashData(fmt.Appendf(nil, "acc-%d", i))
 			rawdb.WriteAccountSnapshot(helper.diskdb, key, val)
 		}
 	}

core/state/snapshot/iterator_test.go

@@ -329,27 +329,27 @@ func TestAccountIteratorTraversalValues(t *testing.T) {
 		h = make(map[common.Hash][]byte)
 	)
 	for i := byte(2); i < 0xff; i++ {
-		a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i))
+		a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i)
 		if i > 20 && i%2 == 0 {
-			b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i))
+			b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i)
 		}
 		if i%4 == 0 {
-			c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i))
+			c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i)
 		}
 		if i%7 == 0 {
-			d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i))
+			d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i)
 		}
 		if i%8 == 0 {
-			e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i))
+			e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i)
 		}
 		if i > 50 || i < 85 {
-			f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i))
+			f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i)
 		}
 		if i%64 == 0 {
-			g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i))
+			g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i)
 		}
 		if i%128 == 0 {
-			h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i))
+			h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i)
 		}
 	}
 	// Assemble a stack of snapshots from the account layers
@@ -428,27 +428,27 @@ func TestStorageIteratorTraversalValues(t *testing.T) {
 		h = make(map[common.Hash][]byte)
 	)
 	for i := byte(2); i < 0xff; i++ {
-		a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i))
+		a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i)
 		if i > 20 && i%2 == 0 {
-			b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i))
+			b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i)
 		}
 		if i%4 == 0 {
-			c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i))
+			c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i)
 		}
 		if i%7 == 0 {
-			d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i))
+			d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i)
 		}
 		if i%8 == 0 {
-			e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i))
+			e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i)
 		}
 		if i > 50 || i < 85 {
-			f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i))
+			f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i)
 		}
 		if i%64 == 0 {
-			g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i))
+			g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i)
 		}
 		if i%128 == 0 {
-			h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i))
+			h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i)
 		}
 	}
 	// Assemble a stack of snapshots from the account layers

core/state/statedb.go

@@ -600,7 +600,6 @@ func (s *StateDB) getStateObject(addr common.Address) *stateObject {
 	// Insert into the live set
 	obj := newObject(s, addr, acct)
 	s.setStateObject(obj)
-	s.AccountLoaded++
 	return obj
 }
 

core/txindexer.go

@@ -20,6 +20,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
@@ -45,6 +46,10 @@ type txIndexer struct {
 	//  * N: means the latest N blocks [HEAD-N+1, HEAD] should be indexed
 	//       and all others shouldn't.
 	limit uint64
+
+	// cutoff denotes the block number before which the chain segment should
+	// be pruned and not available locally.
+	cutoff   uint64
 	db       ethdb.Database
 	progress chan chan TxIndexProgress
 	term     chan chan struct{}
@@ -55,6 +60,7 @@ type txIndexer struct {
 func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer {
 	indexer := &txIndexer{
 		limit:    limit,
+		cutoff:   chain.HistoryPruningCutoff(),
 		db:       chain.db,
 		progress: make(chan chan TxIndexProgress),
 		term:     make(chan chan struct{}),
@@ -64,7 +70,11 @@ func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer {
 
 	var msg string
 	if limit == 0 {
+		if indexer.cutoff == 0 {
 			msg = "entire chain"
+		} else {
+			msg = fmt.Sprintf("blocks since #%d", indexer.cutoff)
+		}
 	} else {
 		msg = fmt.Sprintf("last %d blocks", limit)
 	}
@@ -74,23 +84,31 @@ func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer {
 }
 
 // run executes the scheduled indexing/unindexing task in a separate thread.
-// If the stop channel is closed, the task should be terminated as soon as
+// If the stop channel is closed, the task should terminate as soon as possible.
-// possible, the done channel will be closed once the task is finished.
+// The done channel will be closed once the task is complete.
-func (indexer *txIndexer) run(tail *uint64, head uint64, stop chan struct{}, done chan struct{}) {
+//
+// Existing transaction indexes are assumed to be valid, with both the head
+// and tail above the configured cutoff.
+func (indexer *txIndexer) run(head uint64, stop chan struct{}, done chan struct{}) {
 	defer func() { close(done) }()
 
-	// Short circuit if chain is empty and nothing to index.
+	// Short circuit if the chain is either empty, or entirely below the
-	if head == 0 {
+	// cutoff point.
+	if head == 0 || head < indexer.cutoff {
 		return
 	}
 	// The tail flag is not existent, it means the node is just initialized
 	// and all blocks in the chain (part of them may from ancient store) are
 	// not indexed yet, index the chain according to the configured limit.
+	tail := rawdb.ReadTxIndexTail(indexer.db)
 	if tail == nil {
+		// Determine the first block for transaction indexing, taking the
+		// configured cutoff point into account.
 		from := uint64(0)
 		if indexer.limit != 0 && head >= indexer.limit {
 			from = head - indexer.limit + 1
 		}
+		from = max(from, indexer.cutoff)
 		rawdb.IndexTransactions(indexer.db, from, head+1, stop, true)
 		return
 	}
@@ -98,25 +116,82 @@ func (indexer *txIndexer) run(tail *uint64, head uint64, stop chan struct{}, don
 	// present), while the whole chain are requested for indexing.
 	if indexer.limit == 0 || head < indexer.limit {
 		if *tail > 0 {
-			// It can happen when chain is rewound to a historical point which
+			from := max(uint64(0), indexer.cutoff)
-			// is even lower than the indexes tail, recap the indexing target
+			rawdb.IndexTransactions(indexer.db, from, *tail, stop, true)
-			// to new head to avoid reading non-existent block bodies.
-			end := *tail
-			if end > head+1 {
-				end = head + 1
-			}
-			rawdb.IndexTransactions(indexer.db, 0, end, stop, true)
 		}
 		return
 	}
 	// The tail flag is existent, adjust the index range according to configured
 	// limit and the latest chain head.
-	if head-indexer.limit+1 < *tail {
+	from := head - indexer.limit + 1
+	from = max(from, indexer.cutoff)
+	if from < *tail {
 		// Reindex a part of missing indices and rewind index tail to HEAD-limit
-		rawdb.IndexTransactions(indexer.db, head-indexer.limit+1, *tail, stop, true)
+		rawdb.IndexTransactions(indexer.db, from, *tail, stop, true)
 	} else {
 		// Unindex a part of stale indices and forward index tail to HEAD-limit
-		rawdb.UnindexTransactions(indexer.db, *tail, head-indexer.limit+1, stop, false)
+		rawdb.UnindexTransactions(indexer.db, *tail, from, stop, false)
+	}
+}
+
+// repair ensures that transaction indexes are in a valid state and invalidates
+// them if they are not. The following cases are considered invalid:
+// * The index tail is higher than the chain head.
+// * The chain head is below the configured cutoff, but the index tail is not empty.
+// * The index tail is below the configured cutoff, but it is not empty.
+func (indexer *txIndexer) repair(head uint64) {
+	// If the transactions haven't been indexed yet, nothing to repair
+	tail := rawdb.ReadTxIndexTail(indexer.db)
+	if tail == nil {
+		return
+	}
+	// The transaction index tail is higher than the chain head, which may occur
+	// when the chain is rewound to a historical height below the index tail.
+	// Purge the transaction indexes from the database. **It's not a common case
+	// to rewind the chain head below the index tail**.
+	if *tail > head {
+		// A crash may occur between the two delete operations,
+		// potentially leaving dangling indexes in the database.
+		// However, this is considered acceptable.
+		rawdb.DeleteTxIndexTail(indexer.db)
+		rawdb.DeleteAllTxLookupEntries(indexer.db, nil)
+		log.Warn("Purge transaction indexes", "head", head, "tail", *tail)
+		return
+	}
+
+	// If the entire chain is below the configured cutoff point,
+	// removing the tail of transaction indexing and purges the
+	// transaction indexes. **It's not a common case, as the cutoff
+	// is usually defined below the chain head**.
+	if head < indexer.cutoff {
+		// A crash may occur between the two delete operations,
+		// potentially leaving dangling indexes in the database.
+		// However, this is considered acceptable.
+		//
+		// The leftover indexes can't be unindexed by scanning
+		// the blocks as they are not guaranteed to be available.
+		// Traversing the database directly within the transaction
+		// index namespace might be slow and expensive, but we
+		// have no choice.
+		rawdb.DeleteTxIndexTail(indexer.db)
+		rawdb.DeleteAllTxLookupEntries(indexer.db, nil)
+		log.Warn("Purge transaction indexes", "head", head, "cutoff", indexer.cutoff)
+		return
+	}
+
+	// The chain head is above the cutoff while the tail is below the
+	// cutoff. Shift the tail to the cutoff point and remove the indexes
+	// below.
+	if *tail < indexer.cutoff {
+		// A crash may occur between the two delete operations,
+		// potentially leaving dangling indexes in the database.
+		// However, this is considered acceptable.
+		rawdb.WriteTxIndexTail(indexer.db, indexer.cutoff)
+		rawdb.DeleteAllTxLookupEntries(indexer.db, func(txhash common.Hash, blob []byte) bool {
+			n := rawdb.DecodeTxLookupEntry(blob, indexer.db)
+			return n != nil && *n < indexer.cutoff
+		})
+		log.Warn("Purge transaction indexes below cutoff", "tail", *tail, "cutoff", indexer.cutoff)
 	}
 }
 
@@ -127,39 +202,39 @@ func (indexer *txIndexer) loop(chain *BlockChain) {
 
 	// Listening to chain events and manipulate the transaction indexes.
 	var (
-		stop     chan struct{}                       // Non-nil if background routine is active.
+		stop chan struct{}                                 // Non-nil if background routine is active
-		done     chan struct{}                       // Non-nil if background routine is active.
+		done chan struct{}                                 // Non-nil if background routine is active
-		lastHead uint64                              // The latest announced chain head (whose tx indexes are assumed created)
+		head = rawdb.ReadHeadBlock(indexer.db).NumberU64() // The latest announced chain head
-		lastTail = rawdb.ReadTxIndexTail(indexer.db) // The oldest indexed block, nil means nothing indexed
 
 		headCh = make(chan ChainHeadEvent)
 		sub    = chain.SubscribeChainHeadEvent(headCh)
 	)
 	defer sub.Unsubscribe()
 
+	// Validate the transaction indexes and repair if necessary
+	indexer.repair(head)
+
 	// Launch the initial processing if chain is not empty (head != genesis).
 	// This step is useful in these scenarios that chain has no progress.
-	if head := rawdb.ReadHeadBlock(indexer.db); head != nil && head.Number().Uint64() != 0 {
+	if head != 0 {
 		stop = make(chan struct{})
 		done = make(chan struct{})
-		lastHead = head.Number().Uint64()
+		go indexer.run(head, stop, done)
-		go indexer.run(rawdb.ReadTxIndexTail(indexer.db), head.NumberU64(), stop, done)
 	}
 	for {
 		select {
-		case head := <-headCh:
+		case h := <-headCh:
 			if done == nil {
 				stop = make(chan struct{})
 				done = make(chan struct{})
-				go indexer.run(rawdb.ReadTxIndexTail(indexer.db), head.Header.Number.Uint64(), stop, done)
+				go indexer.run(h.Header.Number.Uint64(), stop, done)
 			}
-			lastHead = head.Header.Number.Uint64()
+			head = h.Header.Number.Uint64()
 		case <-done:
 			stop = nil
 			done = nil
-			lastTail = rawdb.ReadTxIndexTail(indexer.db)
 		case ch := <-indexer.progress:
-			ch <- indexer.report(lastHead, lastTail)
+			ch <- indexer.report(head)
 		case ch := <-indexer.term:
 			if stop != nil {
 				close(stop)
@@ -175,12 +250,27 @@ func (indexer *txIndexer) loop(chain *BlockChain) {
 }
 
 // report returns the tx indexing progress.
-func (indexer *txIndexer) report(head uint64, tail *uint64) TxIndexProgress {
+func (indexer *txIndexer) report(head uint64) TxIndexProgress {
+	// Special case if the head is even below the cutoff,
+	// nothing to index.
+	if head < indexer.cutoff {
+		return TxIndexProgress{
+			Indexed:   0,
+			Remaining: 0,
+		}
+	}
+	// Compute how many blocks are supposed to be indexed
 	total := indexer.limit
 	if indexer.limit == 0 || total > head {
 		total = head + 1 // genesis included
 	}
+	length := head - indexer.cutoff + 1 // all available chain for indexing
+	if total > length {
+		total = length
+	}
+	// Compute how many blocks have been indexed
 	var indexed uint64
+	tail := rawdb.ReadTxIndexTail(indexer.db)
 	if tail != nil {
 		indexed = head - *tail + 1
 	}

core/txindexer_test.go

@@ -29,6 +29,46 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 )
 
+func verifyIndexes(t *testing.T, db ethdb.Database, block *types.Block, exist bool) {
+	for _, tx := range block.Transactions() {
+		lookup := rawdb.ReadTxLookupEntry(db, tx.Hash())
+		if exist && lookup == nil {
+			t.Fatalf("missing %d %x", block.NumberU64(), tx.Hash().Hex())
+		}
+		if !exist && lookup != nil {
+			t.Fatalf("unexpected %d %x", block.NumberU64(), tx.Hash().Hex())
+		}
+	}
+}
+
+func verify(t *testing.T, db ethdb.Database, blocks []*types.Block, expTail uint64) {
+	tail := rawdb.ReadTxIndexTail(db)
+	if tail == nil {
+		t.Fatal("Failed to write tx index tail")
+		return
+	}
+	if *tail != expTail {
+		t.Fatalf("Unexpected tx index tail, want %v, got %d", expTail, *tail)
+	}
+	for _, b := range blocks {
+		if b.Number().Uint64() < *tail {
+			verifyIndexes(t, db, b, false)
+		} else {
+			verifyIndexes(t, db, b, true)
+		}
+	}
+}
+
+func verifyNoIndex(t *testing.T, db ethdb.Database, blocks []*types.Block) {
+	tail := rawdb.ReadTxIndexTail(db)
+	if tail != nil {
+		t.Fatalf("Unexpected tx index tail %d", *tail)
+	}
+	for _, b := range blocks {
+		verifyIndexes(t, db, b, false)
+	}
+}
+
 // TestTxIndexer tests the functionalities for managing transaction indexes.
 func TestTxIndexer(t *testing.T) {
 	var (
@@ -50,163 +90,29 @@ func TestTxIndexer(t *testing.T) {
 		gen.AddTx(tx)
 		nonce += 1
 	})
-
-	// verifyIndexes checks if the transaction indexes are present or not
-	// of the specified block.
-	verifyIndexes := func(db ethdb.Database, number uint64, exist bool) {
-		if number == 0 {
-			return
-		}
-		block := blocks[number-1]
-		for _, tx := range block.Transactions() {
-			lookup := rawdb.ReadTxLookupEntry(db, tx.Hash())
-			if exist && lookup == nil {
-				t.Fatalf("missing %d %x", number, tx.Hash().Hex())
-			}
-			if !exist && lookup != nil {
-				t.Fatalf("unexpected %d %x", number, tx.Hash().Hex())
-			}
-		}
-	}
-	verify := func(db ethdb.Database, expTail uint64, indexer *txIndexer) {
-		tail := rawdb.ReadTxIndexTail(db)
-		if tail == nil {
-			t.Fatal("Failed to write tx index tail")
-		}
-		if *tail != expTail {
-			t.Fatalf("Unexpected tx index tail, want %v, got %d", expTail, *tail)
-		}
-		if *tail != 0 {
-			for number := uint64(0); number < *tail; number += 1 {
-				verifyIndexes(db, number, false)
-			}
-		}
-		for number := *tail; number <= chainHead; number += 1 {
-			verifyIndexes(db, number, true)
-		}
-		progress := indexer.report(chainHead, tail)
-		if !progress.Done() {
-			t.Fatalf("Expect fully indexed")
-		}
-	}
-
 	var cases = []struct {
-		limitA uint64
+		limits []uint64
-		tailA  uint64
+		tails  []uint64
-		limitB uint64
-		tailB  uint64
-		limitC uint64
-		tailC  uint64
 	}{
 		{
-			// LimitA: 0
+			limits: []uint64{0, 1, 64, 129, 0},
-			// TailA:  0
+			tails:  []uint64{0, 128, 65, 0, 0},
-			//
-			// all blocks are indexed
-			limitA: 0,
-			tailA:  0,
-
-			// LimitB: 1
-			// TailB:  128
-			//
-			// block-128 is indexed
-			limitB: 1,
-			tailB:  128,
-
-			// LimitB: 64
-			// TailB:  65
-			//
-			// block [65, 128] are indexed
-			limitC: 64,
-			tailC:  65,
 		},
 		{
-			// LimitA: 64
+			limits: []uint64{64, 1, 64, 0},
-			// TailA:  65
+			tails:  []uint64{65, 128, 65, 0},
-			//
-			// block [65, 128] are indexed
-			limitA: 64,
-			tailA:  65,
-
-			// LimitB: 1
-			// TailB:  128
-			//
-			// block-128 is indexed
-			limitB: 1,
-			tailB:  128,
-
-			// LimitB: 64
-			// TailB:  65
-			//
-			// block [65, 128] are indexed
-			limitC: 64,
-			tailC:  65,
 		},
 		{
-			// LimitA: 127
+			limits: []uint64{127, 1, 64, 0},
-			// TailA:  2
+			tails:  []uint64{2, 128, 65, 0},
-			//
-			// block [2, 128] are indexed
-			limitA: 127,
-			tailA:  2,
-
-			// LimitB: 1
-			// TailB:  128
-			//
-			// block-128 is indexed
-			limitB: 1,
-			tailB:  128,
-
-			// LimitB: 64
-			// TailB:  65
-			//
-			// block [65, 128] are indexed
-			limitC: 64,
-			tailC:  65,
 		},
 		{
-			// LimitA: 128
+			limits: []uint64{128, 1, 64, 0},
-			// TailA:  1
+			tails:  []uint64{1, 128, 65, 0},
-			//
-			// block [2, 128] are indexed
-			limitA: 128,
-			tailA:  1,
-
-			// LimitB: 1
-			// TailB:  128
-			//
-			// block-128 is indexed
-			limitB: 1,
-			tailB:  128,
-
-			// LimitB: 64
-			// TailB:  65
-			//
-			// block [65, 128] are indexed
-			limitC: 64,
-			tailC:  65,
 		},
 		{
-			// LimitA: 129
+			limits: []uint64{129, 1, 64, 0},
-			// TailA:  0
+			tails:  []uint64{0, 128, 65, 0},
-			//
-			// block [0, 128] are indexed
-			limitA: 129,
-			tailA:  0,
-
-			// LimitB: 1
-			// TailB:  128
-			//
-			// block-128 is indexed
-			limitB: 1,
-			tailB:  128,
-
-			// LimitB: 64
-			// TailB:  65
-			//
-			// block [65, 128] are indexed
-			limitC: 64,
-			tailC:  65,
 		},
 	}
 	for _, c := range cases {
@@ -215,26 +121,332 @@ func TestTxIndexer(t *testing.T) {
 
 		// Index the initial blocks from ancient store
 		indexer := &txIndexer{
-			limit:    c.limitA,
+			limit:    0,
 			db:       db,
 			progress: make(chan chan TxIndexProgress),
 		}
-		indexer.run(nil, 128, make(chan struct{}), make(chan struct{}))
+		for i, limit := range c.limits {
-		verify(db, c.tailA, indexer)
+			indexer.limit = limit
-
+			indexer.run(chainHead, make(chan struct{}), make(chan struct{}))
-		indexer.limit = c.limitB
+			verify(t, db, blocks, c.tails[i])
-		indexer.run(rawdb.ReadTxIndexTail(db), 128, make(chan struct{}), make(chan struct{}))
+		}
-		verify(db, c.tailB, indexer)
+		db.Close()
-
+	}
-		indexer.limit = c.limitC
+}
-		indexer.run(rawdb.ReadTxIndexTail(db), 128, make(chan struct{}), make(chan struct{}))
+
-		verify(db, c.tailC, indexer)
+func TestTxIndexerRepair(t *testing.T) {
-
+	var (
-		// Recover all indexes
+		testBankKey, _  = crypto.GenerateKey()
-		indexer.limit = 0
+		testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey)
-		indexer.run(rawdb.ReadTxIndexTail(db), 128, make(chan struct{}), make(chan struct{}))
+		testBankFunds   = big.NewInt(1000000000000000000)
-		verify(db, 0, indexer)
+
-
+		gspec = &Genesis{
+			Config:  params.TestChainConfig,
+			Alloc:   types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}},
+			BaseFee: big.NewInt(params.InitialBaseFee),
+		}
+		engine    = ethash.NewFaker()
+		nonce     = uint64(0)
+		chainHead = uint64(128)
+	)
+	_, blocks, receipts := GenerateChainWithGenesis(gspec, engine, int(chainHead), func(i int, gen *BlockGen) {
+		tx, _ := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("0xdeadbeef"), big.NewInt(1000), params.TxGas, big.NewInt(10*params.InitialBaseFee), nil), types.HomesteadSigner{}, testBankKey)
+		gen.AddTx(tx)
+		nonce += 1
+	})
+	tailPointer := func(n uint64) *uint64 {
+		return &n
+	}
+	var cases = []struct {
+		limit   uint64
+		head    uint64
+		cutoff  uint64
+		expTail *uint64
+	}{
+		// if *tail > head => purge indexes
+		{
+			limit:   0,
+			head:    chainHead / 2,
+			cutoff:  0,
+			expTail: tailPointer(0),
+		},
+		{
+			limit:   1,             // tail = 128
+			head:    chainHead / 2, // newhead = 64
+			cutoff:  0,
+			expTail: nil,
+		},
+		{
+			limit:   64,            // tail = 65
+			head:    chainHead / 2, // newhead = 64
+			cutoff:  0,
+			expTail: nil,
+		},
+		{
+			limit:   65,            // tail = 64
+			head:    chainHead / 2, // newhead = 64
+			cutoff:  0,
+			expTail: tailPointer(64),
+		},
+		{
+			limit:   66,            // tail = 63
+			head:    chainHead / 2, // newhead = 64
+			cutoff:  0,
+			expTail: tailPointer(63),
+		},
+
+		// if tail < cutoff => remove indexes below cutoff
+		{
+			limit:   0,         // tail = 0
+			head:    chainHead, // head = 128
+			cutoff:  chainHead, // cutoff = 128
+			expTail: tailPointer(chainHead),
+		},
+		{
+			limit:   1,         // tail = 128
+			head:    chainHead, // head = 128
+			cutoff:  chainHead, // cutoff = 128
+			expTail: tailPointer(128),
+		},
+		{
+			limit:   2,         // tail = 127
+			head:    chainHead, // head = 128
+			cutoff:  chainHead, // cutoff = 128
+			expTail: tailPointer(chainHead),
+		},
+		{
+			limit:   2,             // tail = 127
+			head:    chainHead,     // head = 128
+			cutoff:  chainHead / 2, // cutoff = 64
+			expTail: tailPointer(127),
+		},
+
+		// if head < cutoff => purge indexes
+		{
+			limit:   0,             // tail = 0
+			head:    chainHead,     // head = 128
+			cutoff:  2 * chainHead, // cutoff = 256
+			expTail: nil,
+		},
+		{
+			limit:   64,            // tail = 65
+			head:    chainHead,     // head = 128
+			cutoff:  chainHead / 2, // cutoff = 64
+			expTail: tailPointer(65),
+		},
+	}
+	for _, c := range cases {
+		db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
+		rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...))
+
+		// Index the initial blocks from ancient store
+		indexer := &txIndexer{
+			limit:    c.limit,
+			db:       db,
+			progress: make(chan chan TxIndexProgress),
+		}
+		indexer.run(chainHead, make(chan struct{}), make(chan struct{}))
+
+		indexer.cutoff = c.cutoff
+		indexer.repair(c.head)
+
+		if c.expTail == nil {
+			verifyNoIndex(t, db, blocks)
+		} else {
+			verify(t, db, blocks, *c.expTail)
+		}
+		db.Close()
+	}
+}
+
+func TestTxIndexerReport(t *testing.T) {
+	var (
+		testBankKey, _  = crypto.GenerateKey()
+		testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey)
+		testBankFunds   = big.NewInt(1000000000000000000)
+
+		gspec = &Genesis{
+			Config:  params.TestChainConfig,
+			Alloc:   types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}},
+			BaseFee: big.NewInt(params.InitialBaseFee),
+		}
+		engine    = ethash.NewFaker()
+		nonce     = uint64(0)
+		chainHead = uint64(128)
+	)
+	_, blocks, receipts := GenerateChainWithGenesis(gspec, engine, int(chainHead), func(i int, gen *BlockGen) {
+		tx, _ := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("0xdeadbeef"), big.NewInt(1000), params.TxGas, big.NewInt(10*params.InitialBaseFee), nil), types.HomesteadSigner{}, testBankKey)
+		gen.AddTx(tx)
+		nonce += 1
+	})
+	tailPointer := func(n uint64) *uint64 {
+		return &n
+	}
+	var cases = []struct {
+		head         uint64
+		limit        uint64
+		cutoff       uint64
+		tail         *uint64
+		expIndexed   uint64
+		expRemaining uint64
+	}{
+		// The entire chain is supposed to be indexed
+		{
+			// head = 128, limit = 0, cutoff = 0 => all: 129
+			head:   chainHead,
+			limit:  0,
+			cutoff: 0,
+
+			// tail = 0
+			tail:         tailPointer(0),
+			expIndexed:   129,
+			expRemaining: 0,
+		},
+		{
+			// head = 128, limit = 0, cutoff = 0 => all: 129
+			head:   chainHead,
+			limit:  0,
+			cutoff: 0,
+
+			// tail = 1
+			tail:         tailPointer(1),
+			expIndexed:   128,
+			expRemaining: 1,
+		},
+		{
+			// head = 128, limit = 0, cutoff = 0 => all: 129
+			head:   chainHead,
+			limit:  0,
+			cutoff: 0,
+
+			// tail = 128
+			tail:         tailPointer(chainHead),
+			expIndexed:   1,
+			expRemaining: 128,
+		},
+		{
+			// head = 128, limit = 256, cutoff = 0 => all: 129
+			head:   chainHead,
+			limit:  256,
+			cutoff: 0,
+
+			// tail = 0
+			tail:         tailPointer(0),
+			expIndexed:   129,
+			expRemaining: 0,
+		},
+
+		// The chain with specific range is supposed to be indexed
+		{
+			// head = 128, limit = 64, cutoff = 0 => index: [65, 128]
+			head:   chainHead,
+			limit:  64,
+			cutoff: 0,
+
+			// tail = 0, part of them need to be unindexed
+			tail:         tailPointer(0),
+			expIndexed:   129,
+			expRemaining: 0,
+		},
+		{
+			// head = 128, limit = 64, cutoff = 0 => index: [65, 128]
+			head:   chainHead,
+			limit:  64,
+			cutoff: 0,
+
+			// tail = 64, one of them needs to be unindexed
+			tail:         tailPointer(64),
+			expIndexed:   65,
+			expRemaining: 0,
+		},
+		{
+			// head = 128, limit = 64, cutoff = 0 => index: [65, 128]
+			head:   chainHead,
+			limit:  64,
+			cutoff: 0,
+
+			// tail = 65, all of them have been indexed
+			tail:         tailPointer(65),
+			expIndexed:   64,
+			expRemaining: 0,
+		},
+		{
+			// head = 128, limit = 64, cutoff = 0 => index: [65, 128]
+			head:   chainHead,
+			limit:  64,
+			cutoff: 0,
+
+			// tail = 66, one of them has to be indexed
+			tail:         tailPointer(66),
+			expIndexed:   63,
+			expRemaining: 1,
+		},
+
+		// The chain with configured cutoff, the chain range could be capped
+		{
+			// head = 128, limit = 64, cutoff = 66 => index: [66, 128]
+			head:   chainHead,
+			limit:  64,
+			cutoff: 66,
+
+			// tail = 0, part of them need to be unindexed
+			tail:         tailPointer(0),
+			expIndexed:   129,
+			expRemaining: 0,
+		},
+		{
+			// head = 128, limit = 64, cutoff = 66 => index: [66, 128]
+			head:   chainHead,
+			limit:  64,
+			cutoff: 66,
+
+			// tail = 66, all of them have been indexed
+			tail:         tailPointer(66),
+			expIndexed:   63,
+			expRemaining: 0,
+		},
+		{
+			// head = 128, limit = 64, cutoff = 66 => index: [66, 128]
+			head:   chainHead,
+			limit:  64,
+			cutoff: 66,
+
+			// tail = 67, one of them has to be indexed
+			tail:         tailPointer(67),
+			expIndexed:   62,
+			expRemaining: 1,
+		},
+		{
+			// head = 128, limit = 64, cutoff = 256 => index: [66, 128]
+			head:         chainHead,
+			limit:        0,
+			cutoff:       256,
+			tail:         nil,
+			expIndexed:   0,
+			expRemaining: 0,
+		},
+	}
+	for _, c := range cases {
+		db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
+		rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...))
+
+		// Index the initial blocks from ancient store
+		indexer := &txIndexer{
+			limit:    c.limit,
+			cutoff:   c.cutoff,
+			db:       db,
+			progress: make(chan chan TxIndexProgress),
+		}
+		if c.tail != nil {
+			rawdb.WriteTxIndexTail(db, *c.tail)
+		}
+		p := indexer.report(c.head)
+		if p.Indexed != c.expIndexed {
+			t.Fatalf("Unexpected indexed: %d, expected: %d", p.Indexed, c.expIndexed)
+		}
+		if p.Remaining != c.expRemaining {
+			t.Fatalf("Unexpected remaining: %d, expected: %d", p.Remaining, c.expRemaining)
+		}
 		db.Close()
 	}
 }

core/txpool/blobpool/blobpool.go

@@ -88,7 +88,8 @@ type blobTxMeta struct {
 	vhashes []common.Hash // Blob versioned hashes to maintain the lookup table
 
 	id          uint64 // Storage ID in the pool's persistent store
-	size uint32 // Byte size in the pool's persistent store
+	storageSize uint32 // Byte size in the pool's persistent store
+	size        uint64 // RLP-encoded size of transaction including the attached blob
 
 	nonce      uint64       // Needed to prioritize inclusion order within an account
 	costCap    *uint256.Int // Needed to validate cumulative balance sufficiency
@@ -108,11 +109,12 @@ type blobTxMeta struct {
 
 // newBlobTxMeta retrieves the indexed metadata fields from a blob transaction
 // and assembles a helper struct to track in memory.
-func newBlobTxMeta(id uint64, size uint32, tx *types.Transaction) *blobTxMeta {
+func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transaction) *blobTxMeta {
 	meta := &blobTxMeta{
 		hash:        tx.Hash(),
 		vhashes:     tx.BlobHashes(),
 		id:          id,
+		storageSize: storageSize,
 		size:        size,
 		nonce:       tx.Nonce(),
 		costCap:     uint256.MustFromBig(tx.Cost()),
@@ -480,7 +482,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error {
 		return errors.New("missing blob sidecar")
 	}
 
-	meta := newBlobTxMeta(id, size, tx)
+	meta := newBlobTxMeta(id, tx.Size(), size, tx)
 	if p.lookup.exists(meta.hash) {
 		// This path is only possible after a crash, where deleted items are not
 		// removed via the normal shutdown-startup procedure and thus may get
@@ -507,7 +509,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error {
 	p.spent[sender] = new(uint256.Int).Add(p.spent[sender], meta.costCap)
 
 	p.lookup.track(meta)
-	p.stored += uint64(meta.size)
+	p.stored += uint64(meta.storageSize)
 	return nil
 }
 
@@ -539,7 +541,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
 			ids = append(ids, txs[i].id)
 			nonces = append(nonces, txs[i].nonce)
 
-			p.stored -= uint64(txs[i].size)
+			p.stored -= uint64(txs[i].storageSize)
 			p.lookup.untrack(txs[i])
 
 			// Included transactions blobs need to be moved to the limbo
@@ -580,7 +582,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
 			nonces = append(nonces, txs[0].nonce)
 
 			p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[0].costCap)
-			p.stored -= uint64(txs[0].size)
+			p.stored -= uint64(txs[0].storageSize)
 			p.lookup.untrack(txs[0])
 
 			// Included transactions blobs need to be moved to the limbo
@@ -636,7 +638,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
 			dropRepeatedMeter.Mark(1)
 
 			p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[i].costCap)
-			p.stored -= uint64(txs[i].size)
+			p.stored -= uint64(txs[i].storageSize)
 			p.lookup.untrack(txs[i])
 
 			if err := p.store.Delete(id); err != nil {
@@ -658,7 +660,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
 			nonces = append(nonces, txs[j].nonce)
 
 			p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[j].costCap)
-			p.stored -= uint64(txs[j].size)
+			p.stored -= uint64(txs[j].storageSize)
 			p.lookup.untrack(txs[j])
 		}
 		txs = txs[:i]
@@ -696,7 +698,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
 			nonces = append(nonces, last.nonce)
 
 			p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], last.costCap)
-			p.stored -= uint64(last.size)
+			p.stored -= uint64(last.storageSize)
 			p.lookup.untrack(last)
 		}
 		if len(txs) == 0 {
@@ -736,7 +738,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
 			nonces = append(nonces, last.nonce)
 
 			p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], last.costCap)
-			p.stored -= uint64(last.size)
+			p.stored -= uint64(last.storageSize)
 			p.lookup.untrack(last)
 		}
 		p.index[addr] = txs
@@ -1002,7 +1004,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error {
 	}
 
 	// Update the indices and metrics
-	meta := newBlobTxMeta(id, p.store.Size(id), tx)
+	meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), tx)
 	if _, ok := p.index[addr]; !ok {
 		if err := p.reserve(addr, true); err != nil {
 			log.Warn("Failed to reserve account for blob pool", "tx", tx.Hash(), "from", addr, "err", err)
@@ -1016,7 +1018,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error {
 		p.spent[addr] = new(uint256.Int).Add(p.spent[addr], meta.costCap)
 	}
 	p.lookup.track(meta)
-	p.stored += uint64(meta.size)
+	p.stored += uint64(meta.storageSize)
 	return nil
 }
 
@@ -1041,7 +1043,7 @@ func (p *BlobPool) SetGasTip(tip *big.Int) {
 						nonces = []uint64{tx.nonce}
 					)
 					p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[i].costCap)
-					p.stored -= uint64(tx.size)
+					p.stored -= uint64(tx.storageSize)
 					p.lookup.untrack(tx)
 					txs[i] = nil
 
@@ -1051,7 +1053,7 @@ func (p *BlobPool) SetGasTip(tip *big.Int) {
 						nonces = append(nonces, tx.nonce)
 
 						p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], tx.costCap)
-						p.stored -= uint64(tx.size)
+						p.stored -= uint64(tx.storageSize)
 						p.lookup.untrack(tx)
 						txs[i+1+j] = nil
 					}
@@ -1236,6 +1238,25 @@ func (p *BlobPool) GetRLP(hash common.Hash) []byte {
 	return p.getRLP(hash)
 }
 
+// GetMetadata returns the transaction type and transaction size with the
+// given transaction hash.
+//
+// The size refers the length of the 'rlp encoding' of a blob transaction
+// including the attached blobs.
+func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+
+	size, ok := p.lookup.sizeOfTx(hash)
+	if !ok {
+		return nil
+	}
+	return &txpool.TxMetadata{
+		Type: types.BlobTxType,
+		Size: size,
+	}
+}
+
 // GetBlobs returns a number of blobs are proofs for the given versioned hashes.
 // This is a utility method for the engine API, enabling consensus clients to
 // retrieve blobs from the pools directly instead of the network.
@@ -1375,7 +1396,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
 	if err != nil {
 		return err
 	}
-	meta := newBlobTxMeta(id, p.store.Size(id), tx)
+	meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), tx)
 
 	var (
 		next   = p.state.GetNonce(from)
@@ -1403,7 +1424,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
 
 		p.lookup.untrack(prev)
 		p.lookup.track(meta)
-		p.stored += uint64(meta.size) - uint64(prev.size)
+		p.stored += uint64(meta.storageSize) - uint64(prev.storageSize)
 	} else {
 		// Transaction extends previously scheduled ones
 		p.index[from] = append(p.index[from], meta)
@@ -1413,7 +1434,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
 		}
 		p.spent[from] = new(uint256.Int).Add(p.spent[from], meta.costCap)
 		p.lookup.track(meta)
-		p.stored += uint64(meta.size)
+		p.stored += uint64(meta.storageSize)
 	}
 	// Recompute the rolling eviction fields. In case of a replacement, this will
 	// recompute all subsequent fields. In case of an append, this will only do
@@ -1500,7 +1521,7 @@ func (p *BlobPool) drop() {
 		p.index[from] = txs
 		p.spent[from] = new(uint256.Int).Sub(p.spent[from], drop.costCap)
 	}
-	p.stored -= uint64(drop.size)
+	p.stored -= uint64(drop.storageSize)
 	p.lookup.untrack(drop)
 
 	// Remove the transaction from the pool's eviction heap:

core/txpool/blobpool/blobpool_test.go

@@ -376,7 +376,7 @@ func verifyPoolInternals(t *testing.T, pool *BlobPool) {
 	var stored uint64
 	for _, txs := range pool.index {
 		for _, tx := range txs {
-			stored += uint64(tx.size)
+			stored += uint64(tx.storageSize)
 		}
 	}
 	if pool.stored != stored {
@@ -1553,6 +1553,16 @@ func TestAdd(t *testing.T) {
 			if err := pool.add(signed); !errors.Is(err, add.err) {
 				t.Errorf("test %d, tx %d: adding transaction error mismatch: have %v, want %v", i, j, err, add.err)
 			}
+			if add.err == nil {
+				size, exist := pool.lookup.sizeOfTx(signed.Hash())
+				if !exist {
+					t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j)
+				}
+				if size != signed.Size() {
+					t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v",
+						i, j, size, signed.Size())
+				}
+			}
 			verifyPoolInternals(t, pool)
 		}
 		verifyPoolInternals(t, pool)

core/txpool/blobpool/evictheap_test.go

@@ -146,7 +146,7 @@ func TestPriceHeapSorting(t *testing.T) {
 			)
 			index[addr] = []*blobTxMeta{{
 				id:                   uint64(j),
-				size:                 128 * 1024,
+				storageSize:          128 * 1024,
 				nonce:                0,
 				execTipCap:           execTip,
 				execFeeCap:           execFee,
@@ -205,7 +205,7 @@ func benchmarkPriceHeapReinit(b *testing.B, datacap uint64) {
 		)
 		index[addr] = []*blobTxMeta{{
 			id:                   uint64(i),
-			size:                 128 * 1024,
+			storageSize:          128 * 1024,
 			nonce:                0,
 			execTipCap:           execTip,
 			execFeeCap:           execFee,
@@ -281,7 +281,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) {
 		)
 		index[addr] = []*blobTxMeta{{
 			id:                   uint64(i),
-			size:                 128 * 1024,
+			storageSize:          128 * 1024,
 			nonce:                0,
 			execTipCap:           execTip,
 			execFeeCap:           execFee,
@@ -312,7 +312,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) {
 		)
 		metas[i] = &blobTxMeta{
 			id:                   uint64(int(blobs) + i),
-			size:                 128 * 1024,
+			storageSize:          128 * 1024,
 			nonce:                0,
 			execTipCap:           execTip,
 			execFeeCap:           execFee,

core/txpool/blobpool/lookup.go

@@ -20,18 +20,24 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 )
 
+type txMetadata struct {
+	id   uint64 // the billy id of transction
+	size uint64 // the RLP encoded size of transaction (blobs are included)
+}
+
 // lookup maps blob versioned hashes to transaction hashes that include them,
-// and transaction hashes to billy entries that include them.
+// transaction hashes to billy entries that include them, transaction hashes
+// to the transaction size
 type lookup struct {
 	blobIndex map[common.Hash]map[common.Hash]struct{}
-	txIndex   map[common.Hash]uint64
+	txIndex   map[common.Hash]*txMetadata
 }
 
 // newLookup creates a new index for tracking blob to tx; and tx to billy mappings.
 func newLookup() *lookup {
 	return &lookup{
 		blobIndex: make(map[common.Hash]map[common.Hash]struct{}),
-		txIndex:   make(map[common.Hash]uint64),
+		txIndex:   make(map[common.Hash]*txMetadata),
 	}
 }
 
@@ -43,8 +49,11 @@ func (l *lookup) exists(txhash common.Hash) bool {
 
 // storeidOfTx returns the datastore storage item id of a transaction.
 func (l *lookup) storeidOfTx(txhash common.Hash) (uint64, bool) {
-	id, ok := l.txIndex[txhash]
+	meta, ok := l.txIndex[txhash]
-	return id, ok
+	if !ok {
+		return 0, false
+	}
+	return meta.id, true
 }
 
 // storeidOfBlob returns the datastore storage item id of a blob.
@@ -61,6 +70,15 @@ func (l *lookup) storeidOfBlob(vhash common.Hash) (uint64, bool) {
 	return 0, false // Weird, don't choke
 }
 
+// sizeOfTx returns the RLP-encoded size of transaction
+func (l *lookup) sizeOfTx(txhash common.Hash) (uint64, bool) {
+	meta, ok := l.txIndex[txhash]
+	if !ok {
+		return 0, false
+	}
+	return meta.size, true
+}
+
 // track inserts a new set of mappings from blob versioned hashes to transaction
 // hashes; and from transaction hashes to datastore storage item ids.
 func (l *lookup) track(tx *blobTxMeta) {
@@ -71,8 +89,11 @@ func (l *lookup) track(tx *blobTxMeta) {
 		}
 		l.blobIndex[vhash][tx.hash] = struct{}{} // may be double mapped if a tx contains the same blob twice
 	}
-	// Map the transaction hash to the datastore id
+	// Map the transaction hash to the datastore id and RLP-encoded transaction size
-	l.txIndex[tx.hash] = tx.id
+	l.txIndex[tx.hash] = &txMetadata{
+		id:   tx.id,
+		size: tx.size,
+	}
 }
 
 // untrack removes a set of mappings from blob versioned hashes to transaction

core/txpool/legacypool/legacypool.go

@@ -618,7 +618,7 @@ func (pool *LegacyPool) checkDelegationLimit(tx *types.Transaction) error {
 	from, _ := types.Sender(pool.signer, tx) // validated
 
 	// Short circuit if the sender has neither delegation nor pending delegation.
-	if pool.currentState.GetCodeHash(from) == types.EmptyCodeHash && len(pool.all.auths[from]) == 0 {
+	if pool.currentState.GetCodeHash(from) == types.EmptyCodeHash && pool.all.delegationTxsCount(from) == 0 {
 		return nil
 	}
 	pending := pool.pending[from]
@@ -1035,6 +1035,19 @@ func (pool *LegacyPool) GetRLP(hash common.Hash) []byte {
 	return encoded
 }
 
+// GetMetadata returns the transaction type and transaction size with the
+// given transaction hash.
+func (pool *LegacyPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
+	tx := pool.all.Get(hash)
+	if tx == nil {
+		return nil
+	}
+	return &txpool.TxMetadata{
+		Type: tx.Type(),
+		Size: tx.Size(),
+	}
+}
+
 // GetBlobs is not supported by the legacy transaction pool, it is just here to
 // implement the txpool.SubPool interface.
 func (pool *LegacyPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Proof) {
@@ -1849,6 +1862,13 @@ func (t *lookup) removeAuthorities(tx *types.Transaction) {
 	}
 }
 
+// delegationTxsCount returns the number of pending authorizations for the specified address.
+func (t *lookup) delegationTxsCount(addr common.Address) int {
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+	return len(t.auths[addr])
+}
+
 // numSlots calculates the number of slots needed for a single transaction.
 func numSlots(tx *types.Transaction) int {
 	return int((tx.Size() + txSlotSize - 1) / txSlotSize)

core/txpool/locals/tx_tracker_test.go

@@ -0,0 +1,218 @@
+// Copyright 2025 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package locals
+
+import (
+	"errors"
+	"math/big"
+	"testing"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/ethash"
+	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/txpool"
+	"github.com/ethereum/go-ethereum/core/txpool/legacypool"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/params"
+)
+
+var (
+	key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	address = crypto.PubkeyToAddress(key.PublicKey)
+	funds   = big.NewInt(1000000000000000)
+	gspec   = &core.Genesis{
+		Config: params.TestChainConfig,
+		Alloc: types.GenesisAlloc{
+			address: {Balance: funds},
+		},
+		BaseFee: big.NewInt(params.InitialBaseFee),
+	}
+	signer = types.LatestSigner(gspec.Config)
+)
+
+type testEnv struct {
+	chain   *core.BlockChain
+	pool    *txpool.TxPool
+	tracker *TxTracker
+	genDb   ethdb.Database
+}
+
+func newTestEnv(t *testing.T, n int, gasTip uint64, journal string) *testEnv {
+	genDb, blocks, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), n, func(i int, gen *core.BlockGen) {
+		tx, err := types.SignTx(types.NewTransaction(gen.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, gen.BaseFee(), nil), signer, key)
+		if err != nil {
+			panic(err)
+		}
+		gen.AddTx(tx)
+	})
+
+	db := rawdb.NewMemoryDatabase()
+	chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil)
+
+	legacyPool := legacypool.New(legacypool.DefaultConfig, chain)
+	pool, err := txpool.New(gasTip, chain, []txpool.SubPool{legacyPool})
+	if err != nil {
+		t.Fatalf("Failed to create tx pool: %v", err)
+	}
+	if n, err := chain.InsertChain(blocks); err != nil {
+		t.Fatalf("Failed to process block %d: %v", n, err)
+	}
+	if err := pool.Sync(); err != nil {
+		t.Fatalf("Failed to sync the txpool, %v", err)
+	}
+	return &testEnv{
+		chain:   chain,
+		pool:    pool,
+		tracker: New(journal, time.Minute, gspec.Config, pool),
+		genDb:   genDb,
+	}
+}
+
+func (env *testEnv) close() {
+	env.chain.Stop()
+}
+
+func (env *testEnv) setGasTip(gasTip uint64) {
+	env.pool.SetGasTip(new(big.Int).SetUint64(gasTip))
+}
+
+func (env *testEnv) makeTx(nonce uint64, gasPrice *big.Int) *types.Transaction {
+	if nonce == 0 {
+		head := env.chain.CurrentHeader()
+		state, _ := env.chain.StateAt(head.Root)
+		nonce = state.GetNonce(address)
+	}
+	if gasPrice == nil {
+		gasPrice = big.NewInt(params.GWei)
+	}
+	tx, _ := types.SignTx(types.NewTransaction(nonce, common.Address{0x00}, big.NewInt(1000), params.TxGas, gasPrice, nil), signer, key)
+	return tx
+}
+
+func (env *testEnv) makeTxs(n int) []*types.Transaction {
+	head := env.chain.CurrentHeader()
+	state, _ := env.chain.StateAt(head.Root)
+	nonce := state.GetNonce(address)
+
+	var txs []*types.Transaction
+	for i := 0; i < n; i++ {
+		tx, _ := types.SignTx(types.NewTransaction(nonce+uint64(i), common.Address{0x00}, big.NewInt(1000), params.TxGas, big.NewInt(params.GWei), nil), signer, key)
+		txs = append(txs, tx)
+	}
+	return txs
+}
+
+func (env *testEnv) commit() {
+	head := env.chain.CurrentBlock()
+	block := env.chain.GetBlock(head.Hash(), head.Number.Uint64())
+	blocks, _ := core.GenerateChain(env.chain.Config(), block, ethash.NewFaker(), env.genDb, 1, func(i int, gen *core.BlockGen) {
+		tx, err := types.SignTx(types.NewTransaction(gen.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, gen.BaseFee(), nil), signer, key)
+		if err != nil {
+			panic(err)
+		}
+		gen.AddTx(tx)
+	})
+	env.chain.InsertChain(blocks)
+	if err := env.pool.Sync(); err != nil {
+		panic(err)
+	}
+}
+
+func TestRejectInvalids(t *testing.T) {
+	env := newTestEnv(t, 10, 0, "")
+	defer env.close()
+
+	var cases = []struct {
+		gasTip uint64
+		tx     *types.Transaction
+		expErr error
+		commit bool
+	}{
+		{
+			tx:     env.makeTx(5, nil), // stale
+			expErr: core.ErrNonceTooLow,
+		},
+		{
+			tx:     env.makeTx(11, nil), // future transaction
+			expErr: nil,
+		},
+		{
+			gasTip: params.GWei,
+			tx:     env.makeTx(0, new(big.Int).SetUint64(params.GWei/2)), // low price
+			expErr: txpool.ErrUnderpriced,
+		},
+		{
+			tx:     types.NewTransaction(10, common.Address{0x00}, big.NewInt(1000), params.TxGas, big.NewInt(params.GWei), nil), // invalid signature
+			expErr: types.ErrInvalidSig,
+		},
+		{
+			commit: true,
+			tx:     env.makeTx(10, nil), // stale
+			expErr: core.ErrNonceTooLow,
+		},
+		{
+			tx:     env.makeTx(11, nil),
+			expErr: nil,
+		},
+	}
+	for i, c := range cases {
+		if c.gasTip != 0 {
+			env.setGasTip(c.gasTip)
+		}
+		if c.commit {
+			env.commit()
+		}
+		gotErr := env.tracker.Track(c.tx)
+		if c.expErr == nil && gotErr != nil {
+			t.Fatalf("%d, unexpected error: %v", i, gotErr)
+		}
+		if c.expErr != nil && !errors.Is(gotErr, c.expErr) {
+			t.Fatalf("%d, unexpected error, want: %v, got: %v", i, c.expErr, gotErr)
+		}
+	}
+}
+
+func TestResubmit(t *testing.T) {
+	env := newTestEnv(t, 10, 0, "")
+	defer env.close()
+
+	txs := env.makeTxs(10)
+	txsA := txs[:len(txs)/2]
+	txsB := txs[len(txs)/2:]
+	env.pool.Add(txsA, true)
+	pending, queued := env.pool.ContentFrom(address)
+	if len(pending) != len(txsA) || len(queued) != 0 {
+		t.Fatalf("Unexpected txpool content: %d, %d", len(pending), len(queued))
+	}
+	env.tracker.TrackAll(txs)
+
+	resubmit, all := env.tracker.recheck(true)
+	if len(resubmit) != len(txsB) {
+		t.Fatalf("Unexpected transactions to resubmit, got: %d, want: %d", len(resubmit), len(txsB))
+	}
+	if len(all) == 0 || len(all[address]) == 0 {
+		t.Fatalf("Unexpected transactions being tracked, got: %d, want: %d", 0, len(txs))
+	}
+	if len(all[address]) != len(txs) {
+		t.Fatalf("Unexpected transactions being tracked, got: %d, want: %d", len(all[address]), len(txs))
+	}
+}

core/txpool/subpool.go

@@ -86,6 +86,12 @@ type PendingFilter struct {
 	OnlyBlobTxs  bool // Return only blob transactions (block blob-space filling)
 }
 
+// TxMetadata denotes the metadata of a transaction.
+type TxMetadata struct {
+	Type uint8  // The type of the transaction
+	Size uint64 // The length of the 'rlp encoding' of a transaction
+}
+
 // SubPool represents a specialized transaction pool that lives on its own (e.g.
 // blob pool). Since independent of how many specialized pools we have, they do
 // need to be updated in lockstep and assemble into one coherent view for block
@@ -127,6 +133,10 @@ type SubPool interface {
 	// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
 	GetRLP(hash common.Hash) []byte
 
+	// GetMetadata returns the transaction type and transaction size with the
+	// given transaction hash.
+	GetMetadata(hash common.Hash) *TxMetadata
+
 	// GetBlobs returns a number of blobs are proofs for the given versioned hashes.
 	// This is a utility method for the engine API, enabling consensus clients to
 	// retrieve blobs from the pools directly instead of the network.

core/txpool/txpool.go

@@ -24,11 +24,13 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
+	"github.com/ethereum/go-ethereum/params"
 )
 
 // TxStatus is the current status of a transaction as seen by the pool.
@@ -53,11 +55,17 @@ var (
 // BlockChain defines the minimal set of methods needed to back a tx pool with
 // a chain. Exists to allow mocking the live chain out of tests.
 type BlockChain interface {
+	// Config retrieves the chain's fork configuration.
+	Config() *params.ChainConfig
+
 	// CurrentBlock returns the current head of the chain.
 	CurrentBlock() *types.Header
 
 	// SubscribeChainHeadEvent subscribes to new blocks being added to the chain.
 	SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
+
+	// StateAt returns a state database for a given root hash (generally the head).
+	StateAt(root common.Hash) (*state.StateDB, error)
 }
 
 // TxPool is an aggregator for various transaction specific pools, collectively
@@ -67,6 +75,11 @@ type BlockChain interface {
 // resource constraints.
 type TxPool struct {
 	subpools []SubPool // List of subpools for specialized transaction handling
+	chain    BlockChain
+	signer   types.Signer
+
+	stateLock sync.RWMutex   // The lock for protecting state instance
+	state     *state.StateDB // Current state at the blockchain head
 
 	reservations map[common.Address]SubPool // Map with the account to pool reservations
 	reserveLock  sync.Mutex                 // Lock protecting the account reservations
@@ -86,8 +99,21 @@ func New(gasTip uint64, chain BlockChain, subpools []SubPool) (*TxPool, error) {
 	// during initialization.
 	head := chain.CurrentBlock()
 
+	// Initialize the state with head block, or fallback to empty one in
+	// case the head state is not available (might occur when node is not
+	// fully synced).
+	statedb, err := chain.StateAt(head.Root)
+	if err != nil {
+		statedb, err = chain.StateAt(types.EmptyRootHash)
+	}
+	if err != nil {
+		return nil, err
+	}
 	pool := &TxPool{
 		subpools:     subpools,
+		chain:        chain,
+		signer:       types.LatestSigner(chain.Config()),
+		state:        statedb,
 		reservations: make(map[common.Address]SubPool),
 		quit:         make(chan chan error),
 		term:         make(chan struct{}),
@@ -101,7 +127,7 @@ func New(gasTip uint64, chain BlockChain, subpools []SubPool) (*TxPool, error) {
 			return nil, err
 		}
 	}
-	go pool.loop(head, chain)
+	go pool.loop(head)
 	return pool, nil
 }
 
@@ -179,14 +205,14 @@ func (p *TxPool) Close() error {
 // loop is the transaction pool's main event loop, waiting for and reacting to
 // outside blockchain events as well as for various reporting and transaction
 // eviction events.
-func (p *TxPool) loop(head *types.Header, chain BlockChain) {
+func (p *TxPool) loop(head *types.Header) {
 	// Close the termination marker when the pool stops
 	defer close(p.term)
 
 	// Subscribe to chain head events to trigger subpool resets
 	var (
 		newHeadCh  = make(chan core.ChainHeadEvent)
-		newHeadSub = chain.SubscribeChainHeadEvent(newHeadCh)
+		newHeadSub = p.chain.SubscribeChainHeadEvent(newHeadCh)
 	)
 	defer newHeadSub.Unsubscribe()
 
@@ -219,6 +245,14 @@ func (p *TxPool) loop(head *types.Header, chain BlockChain) {
 			// Try to inject a busy marker and start a reset if successful
 			select {
 			case resetBusy <- struct{}{}:
+				statedb, err := p.chain.StateAt(newHead.Root)
+				if err != nil {
+					log.Crit("Failed to reset txpool state", "err", err)
+				}
+				p.stateLock.Lock()
+				p.state = statedb
+				p.stateLock.Unlock()
+
 				// Busy marker injected, start a new subpool reset
 				go func(oldHead, newHead *types.Header) {
 					for _, subpool := range p.subpools {
@@ -320,6 +354,17 @@ func (p *TxPool) GetRLP(hash common.Hash) []byte {
 	return nil
 }
 
+// GetMetadata returns the transaction type and transaction size with the given
+// hash.
+func (p *TxPool) GetMetadata(hash common.Hash) *TxMetadata {
+	for _, subpool := range p.subpools {
+		if meta := subpool.GetMetadata(hash); meta != nil {
+			return meta
+		}
+	}
+	return nil
+}
+
 // GetBlobs returns a number of blobs are proofs for the given versioned hashes.
 // This is a utility method for the engine API, enabling consensus clients to
 // retrieve blobs from the pools directly instead of the network.
@@ -339,6 +384,20 @@ func (p *TxPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Pr
 // ValidateTxBasics checks whether a transaction is valid according to the consensus
 // rules, but does not check state-dependent validation such as sufficient balance.
 func (p *TxPool) ValidateTxBasics(tx *types.Transaction) error {
+	addr, err := types.Sender(p.signer, tx)
+	if err != nil {
+		return err
+	}
+	// Reject transactions with stale nonce. Gapped-nonce future transactions
+	// are considered valid and will be handled by the subpool according to its
+	// internal policy.
+	p.stateLock.RLock()
+	nonce := p.state.GetNonce(addr)
+	p.stateLock.RUnlock()
+
+	if nonce > tx.Nonce() {
+		return core.ErrNonceTooLow
+	}
 	for _, subpool := range p.subpools {
 		if subpool.Filter(tx) {
 			return subpool.ValidateTxBasics(tx)
@@ -418,9 +477,9 @@ func (p *TxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool)
 	return p.subs.Track(event.JoinSubscriptions(subs...))
 }
 
-// Nonce returns the next nonce of an account, with all transactions executable
+// PoolNonce returns the next nonce of an account, with all transactions executable
 // by the pool already applied on top.
-func (p *TxPool) Nonce(addr common.Address) uint64 {
+func (p *TxPool) PoolNonce(addr common.Address) uint64 {
 	// Since (for now) accounts are unique to subpools, only one pool will have
 	// (at max) a non-state nonce. To avoid stateful lookups, just return the
 	// highest nonce for now.
@@ -433,6 +492,15 @@ func (p *TxPool) Nonce(addr common.Address) uint64 {
 	return nonce
 }
 
+// Nonce returns the next nonce of an account at the current chain head. Unlike
+// PoolNonce, this function does not account for pending executable transactions.
+func (p *TxPool) Nonce(addr common.Address) uint64 {
+	p.stateLock.RLock()
+	defer p.stateLock.RUnlock()
+
+	return p.state.GetNonce(addr)
+}
+
 // Stats retrieves the current pool stats, namely the number of pending and the
 // number of queued (non-executable) transactions.
 func (p *TxPool) Stats() (int, int) {
@@ -508,6 +576,9 @@ func (p *TxPool) Sync() error {
 
 // Clear removes all tracked txs from the subpools.
 func (p *TxPool) Clear() {
+	// Invoke Sync to ensure that txs pending addition don't get added to the pool after
+	// the subpools are subsequently cleared
+	p.Sync()
 	for _, subpool := range p.subpools {
 		subpool.Clear()
 	}

eth/api_backend.go

@@ -28,13 +28,13 @@ import (
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
 	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/bloombits"
 	"github.com/ethereum/go-ethereum/core/filtermaps"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/txpool"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/eth/gasprice"
 	"github.com/ethereum/go-ethereum/eth/tracers"
 	"github.com/ethereum/go-ethereum/ethdb"
@@ -92,7 +92,13 @@ func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumb
 		}
 		return block, nil
 	}
-	return b.eth.blockchain.GetHeaderByNumber(uint64(number)), nil
+	var bn uint64
+	if number == rpc.EarliestBlockNumber {
+		bn = b.eth.blockchain.HistoryPruningCutoff()
+	} else {
+		bn = uint64(number)
+	}
+	return b.eth.blockchain.GetHeaderByNumber(bn), nil
 }
 
 func (b *EthAPIBackend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) {
@@ -144,11 +150,27 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe
 		}
 		return b.eth.blockchain.GetBlock(header.Hash(), header.Number.Uint64()), nil
 	}
-	return b.eth.blockchain.GetBlockByNumber(uint64(number)), nil
+	bn := uint64(number) // the resolved number
+	if number == rpc.EarliestBlockNumber {
+		bn = b.eth.blockchain.HistoryPruningCutoff()
+	}
+	block := b.eth.blockchain.GetBlockByNumber(bn)
+	if block == nil && bn < b.eth.blockchain.HistoryPruningCutoff() {
+		return nil, &ethconfig.PrunedHistoryError{}
+	}
+	return block, nil
 }
 
 func (b *EthAPIBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) {
-	return b.eth.blockchain.GetBlockByHash(hash), nil
+	number := b.eth.blockchain.GetBlockNumber(hash)
+	if number == nil {
+		return nil, nil
+	}
+	block := b.eth.blockchain.GetBlock(hash, *number)
+	if block == nil && *number < b.eth.blockchain.HistoryPruningCutoff() {
+		return nil, &ethconfig.PrunedHistoryError{}
+	}
+	return block, nil
 }
 
 // GetBody returns body of a block. It does not resolve special block numbers.
@@ -156,11 +178,15 @@ func (b *EthAPIBackend) GetBody(ctx context.Context, hash common.Hash, number rp
 	if number < 0 || hash == (common.Hash{}) {
 		return nil, errors.New("invalid arguments; expect hash and no special block numbers")
 	}
-	if body := b.eth.blockchain.GetBody(hash); body != nil {
+	body := b.eth.blockchain.GetBody(hash)
-		return body, nil
+	if body == nil {
+		if uint64(number) < b.eth.blockchain.HistoryPruningCutoff() {
+			return nil, &ethconfig.PrunedHistoryError{}
 		}
 		return nil, errors.New("block body not found")
 	}
+	return body, nil
+}
 
 func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) {
 	if blockNr, ok := blockNrOrHash.Number(); ok {
@@ -176,6 +202,9 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r
 		}
 		block := b.eth.blockchain.GetBlock(hash, header.Number.Uint64())
 		if block == nil {
+			if header.Number.Uint64() < b.eth.blockchain.HistoryPruningCutoff() {
+				return nil, &ethconfig.PrunedHistoryError{}
+			}
 			return nil, errors.New("header found, but block body is missing")
 		}
 		return block, nil
@@ -235,6 +264,10 @@ func (b *EthAPIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockN
 	return nil, nil, errors.New("invalid arguments; neither block nor hash specified")
 }
 
+func (b *EthAPIBackend) HistoryPruningCutoff() uint64 {
+	return b.eth.blockchain.HistoryPruningCutoff()
+}
+
 func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) {
 	return b.eth.blockchain.GetReceiptsByHash(hash), nil
 }
@@ -328,7 +361,7 @@ func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash)
 }
 
 func (b *EthAPIBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) {
-	return b.eth.txPool.Nonce(addr), nil
+	return b.eth.txPool.PoolNonce(addr), nil
 }
 
 func (b *EthAPIBackend) Stats() (runnable int, blocked int) {
@@ -403,17 +436,6 @@ func (b *EthAPIBackend) RPCTxFeeCap() float64 {
 	return b.eth.config.RPCTxFeeCap
 }
 
-func (b *EthAPIBackend) BloomStatus() (uint64, uint64) {
-	sections, _, _ := b.eth.bloomIndexer.Sections()
-	return params.BloomBitsBlocks, sections
-}
-
-func (b *EthAPIBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {
-	for i := 0; i < bloomFilterThreads; i++ {
-		go session.Multiplex(bloomRetrievalBatch, bloomRetrievalWait, b.eth.bloomRequests)
-	}
-}
-
 func (b *EthAPIBackend) NewMatcherBackend() filtermaps.MatcherBackend {
 	return b.eth.filterMaps.NewMatcherBackend()
 }

eth/backend.go

@@ -30,7 +30,6 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/core"
-	"github.com/ethereum/go-ethereum/core/bloombits"
 	"github.com/ethereum/go-ethereum/core/filtermaps"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state/pruner"
@@ -85,10 +84,6 @@ type Ethereum struct {
 	engine         consensus.Engine
 	accountManager *accounts.Manager
 
-	bloomRequests     chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests
-	bloomIndexer      *core.ChainIndexer             // Bloom indexer operating during block imports
-	closeBloomHandler chan struct{}
-
 	filterMaps      *filtermaps.FilterMaps
 	closeFilterMaps chan chan struct{}
 
@@ -181,11 +176,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		eventMux:        stack.EventMux(),
 		accountManager:  stack.AccountManager(),
 		engine:          engine,
-		closeBloomHandler: make(chan struct{}),
 		networkID:       networkID,
 		gasPrice:        config.Miner.GasPrice,
-		bloomRequests:     make(chan chan *bloombits.Retrieval),
-		bloomIndexer:      core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms),
 		p2pServer:       stack.Server(),
 		discmix:         enode.NewFairMix(0),
 		shutdownTracker: shutdowncheck.NewShutdownTracker(chainDb),
@@ -247,10 +239,19 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	if err != nil {
 		return nil, err
 	}
-	eth.bloomIndexer.Start(eth.blockchain)
+	fmConfig := filtermaps.Config{
-	fmConfig := filtermaps.Config{History: config.LogHistory, Disabled: config.LogNoHistory, ExportFileName: config.LogExportCheckpoints}
+		History:        config.LogHistory,
+		Disabled:       config.LogNoHistory,
+		ExportFileName: config.LogExportCheckpoints,
+		HashScheme:     scheme == rawdb.HashScheme,
+	}
 	chainView := eth.newChainView(eth.blockchain.CurrentBlock())
-	eth.filterMaps = filtermaps.NewFilterMaps(chainDb, chainView, 0, 0, filtermaps.DefaultParams, fmConfig)
+	historyCutoff := eth.blockchain.HistoryPruningCutoff()
+	var finalBlock uint64
+	if fb := eth.blockchain.CurrentFinalBlock(); fb != nil {
+		finalBlock = fb.Number.Uint64()
+	}
+	eth.filterMaps = filtermaps.NewFilterMaps(chainDb, chainView, historyCutoff, finalBlock, filtermaps.DefaultParams, fmConfig)
 	eth.closeFilterMaps = make(chan chan struct{})
 
 	if config.BlobPool.Datadir != "" {
@@ -379,7 +380,6 @@ func (s *Ethereum) Downloader() *downloader.Downloader { return s.handler.downlo
 func (s *Ethereum) Synced() bool                       { return s.handler.synced.Load() }
 func (s *Ethereum) SetSynced()                         { s.handler.enableSyncedFeatures() }
 func (s *Ethereum) ArchiveMode() bool                  { return s.config.NoPruning }
-func (s *Ethereum) BloomIndexer() *core.ChainIndexer   { return s.bloomIndexer }
 
 // Protocols returns all the currently configured
 // network protocols to start.
@@ -398,9 +398,6 @@ func (s *Ethereum) Start() error {
 		return err
 	}
 
-	// Start the bloom bits servicing goroutines
-	s.startBloomHandlers(params.BloomBitsBlocks)
-
 	// Regularly update shutdown marker
 	s.shutdownTracker.Start()
 
@@ -511,8 +508,6 @@ func (s *Ethereum) Stop() error {
 	s.handler.Stop()
 
 	// Then stop everything else.
-	s.bloomIndexer.Close()
-	close(s.closeBloomHandler)
 	ch := make(chan struct{})
 	s.closeFilterMaps <- ch
 	<-ch

eth/bloombits.go

@@ -1,74 +0,0 @@
-// Copyright 2017 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"time"
-
-	"github.com/ethereum/go-ethereum/common/bitutil"
-	"github.com/ethereum/go-ethereum/core/rawdb"
-)
-
-const (
-	// bloomServiceThreads is the number of goroutines used globally by an Ethereum
-	// instance to service bloombits lookups for all running filters.
-	bloomServiceThreads = 16
-
-	// bloomFilterThreads is the number of goroutines used locally per filter to
-	// multiplex requests onto the global servicing goroutines.
-	bloomFilterThreads = 3
-
-	// bloomRetrievalBatch is the maximum number of bloom bit retrievals to service
-	// in a single batch.
-	bloomRetrievalBatch = 16
-
-	// bloomRetrievalWait is the maximum time to wait for enough bloom bit requests
-	// to accumulate request an entire batch (avoiding hysteresis).
-	bloomRetrievalWait = time.Duration(0)
-)
-
-// startBloomHandlers starts a batch of goroutines to accept bloom bit database
-// retrievals from possibly a range of filters and serving the data to satisfy.
-func (eth *Ethereum) startBloomHandlers(sectionSize uint64) {
-	for i := 0; i < bloomServiceThreads; i++ {
-		go func() {
-			for {
-				select {
-				case <-eth.closeBloomHandler:
-					return
-
-				case request := <-eth.bloomRequests:
-					task := <-request
-					task.Bitsets = make([][]byte, len(task.Sections))
-					for i, section := range task.Sections {
-						head := rawdb.ReadCanonicalHash(eth.chainDb, (section+1)*sectionSize-1)
-						if compVector, err := rawdb.ReadBloomBits(eth.chainDb, task.Bit, section, head); err == nil {
-							if blob, err := bitutil.DecompressBytes(compVector, int(sectionSize/8)); err == nil {
-								task.Bitsets[i] = blob
-							} else {
-								task.Error = err
-							}
-						} else {
-							task.Error = err
-						}
-					}
-					request <- task
-				}
-			}
-		}()
-	}
-}

eth/ethconfig/historymode.go

@@ -90,3 +90,9 @@ var HistoryPrunePoints = map[common.Hash]*HistoryPrunePoint{
 		BlockHash:   common.HexToHash("0x229f6b18ca1552f1d5146deceb5387333f40dc6275aebee3f2c5c4ece07d02db"),
 	},
 }
+
+// PrunedHistoryError is returned when the requested history is pruned.
+type PrunedHistoryError struct{}
+
+func (e *PrunedHistoryError) Error() string  { return "pruned history unavailable" }
+func (e *PrunedHistoryError) ErrorCode() int { return 4444 }

eth/filters/api.go

@@ -29,6 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/internal/ethapi"
 	"github.com/ethereum/go-ethereum/rpc"
 )
@@ -354,9 +355,13 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type
 		if crit.ToBlock != nil {
 			end = crit.ToBlock.Int64()
 		}
+		// Block numbers below 0 are special cases.
 		if begin > 0 && end > 0 && begin > end {
 			return nil, errInvalidBlockRange
 		}
+		if begin > 0 && begin < int64(api.events.backend.HistoryPruningCutoff()) {
+			return nil, &ethconfig.PrunedHistoryError{}
+		}
 		// Construct the range filter
 		filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics)
 	}

eth/filters/filter.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/filtermaps"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
 )
@@ -86,6 +87,9 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) {
 		if header == nil {
 			return nil, errors.New("unknown block")
 		}
+		if header.Number.Uint64() < f.sys.backend.HistoryPruningCutoff() {
+			return nil, &ethconfig.PrunedHistoryError{}
+		}
 		return f.blockLogs(ctx, header)
 	}
 
@@ -114,12 +118,20 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) {
 				return 0, errors.New("safe header not found")
 			}
 			return hdr.Number.Uint64(), nil
+		case rpc.EarliestBlockNumber.Int64():
+			earliest := f.sys.backend.HistoryPruningCutoff()
+			hdr, _ := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(earliest))
+			if hdr == nil {
+				return 0, errors.New("earliest header not found")
 			}
+			return hdr.Number.Uint64(), nil
+		default:
 			if number < 0 {
 				return 0, errors.New("negative block number")
 			}
 			return uint64(number), nil
 		}
+	}
 
 	// range query need to resolve the special begin/end block number
 	begin, err := resolveSpecial(f.begin)
@@ -372,7 +384,7 @@ func (f *Filter) indexedLogs(ctx context.Context, mb filtermaps.MatcherBackend,
 // iteration and bloom matching.
 func (f *Filter) unindexedLogs(ctx context.Context, begin, end uint64) ([]*types.Log, error) {
 	start := time.Now()
-	log.Warn("Performing unindexed log search", "begin", begin, "end", end)
+	log.Debug("Performing unindexed log search", "begin", begin, "end", end)
 	var matches []*types.Log
 	for blockNumber := begin; blockNumber <= end; blockNumber++ {
 		select {
@@ -390,7 +402,7 @@ func (f *Filter) unindexedLogs(ctx context.Context, begin, end uint64) ([]*types
 		}
 		matches = append(matches, found...)
 	}
-	log.Trace("Performed unindexed log search", "begin", begin, "end", end, "matches", len(matches), "elapsed", common.PrettyDuration(time.Since(start)))
+	log.Debug("Performed unindexed log search", "begin", begin, "end", end, "matches", len(matches), "elapsed", common.PrettyDuration(time.Since(start)))
 	return matches, nil
 }
 

eth/filters/filter_system.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/filtermaps"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/eth/ethconfig"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
@@ -64,6 +65,7 @@ type Backend interface {
 
 	CurrentHeader() *types.Header
 	ChainConfig() *params.ChainConfig
+	HistoryPruningCutoff() uint64
 	SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
 	SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription
 	SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription
@@ -304,6 +306,14 @@ func (es *EventSystem) SubscribeLogs(crit ethereum.FilterQuery, logs chan []*typ
 		return nil, errPendingLogsUnsupported
 	}
 
+	if from == rpc.EarliestBlockNumber {
+		from = rpc.BlockNumber(es.backend.HistoryPruningCutoff())
+	}
+	// Queries beyond the pruning cutoff are not supported.
+	if uint64(from) < es.backend.HistoryPruningCutoff() {
+		return nil, &ethconfig.PrunedHistoryError{}
+	}
+
 	// only interested in new mined logs
 	if from == rpc.LatestBlockNumber && to == rpc.LatestBlockNumber {
 		return es.subscribeLogs(crit, logs), nil

eth/filters/filter_system_test.go

@@ -181,6 +181,10 @@ func (b *testBackend) setPending(block *types.Block, receipts types.Receipts) {
 	b.pendingReceipts = receipts
 }
 
+func (b *testBackend) HistoryPruningCutoff() uint64 {
+	return 0
+}
+
 func newTestFilterSystem(db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) {
 	backend := &testBackend{db: db}
 	sys := NewFilterSystem(backend, cfg)

eth/gasestimator/gasestimator.go

@@ -223,7 +223,9 @@ func run(ctx context.Context, call *core.Message, opts *Options) (*core.Executio
 		dirtyState = opts.State.Copy()
 	)
 	if opts.BlockOverrides != nil {
-		opts.BlockOverrides.Apply(&evmContext)
+		if err := opts.BlockOverrides.Apply(&evmContext); err != nil {
+			return nil, err
+		}
 	}
 	// Lower the basefee to 0 to avoid breaking EVM
 	// invariants (basefee < feecap).

eth/handler.go

@@ -71,6 +71,10 @@ type txPool interface {
 	// with given tx hash.
 	GetRLP(hash common.Hash) []byte
 
+	// GetMetadata returns the transaction type and transaction size with the
+	// given transaction hash.
+	GetMetadata(hash common.Hash) *txpool.TxMetadata
+
 	// Add should add the given transactions to the pool.
 	Add(txs []*types.Transaction, sync bool) []error
 
@@ -403,7 +407,7 @@ func (h *handler) unregisterPeer(id string) {
 	// Abort if the peer does not exist
 	peer := h.peers.peer(id)
 	if peer == nil {
-		logger.Error("Ethereum peer removal failed", "err", errPeerNotRegistered)
+		logger.Warn("Ethereum peer removal failed", "err", errPeerNotRegistered)
 		return
 	}
 	// Remove the `eth` peer if it exists

eth/handler_test.go

@@ -93,6 +93,22 @@ func (p *testTxPool) GetRLP(hash common.Hash) []byte {
 	return nil
 }
 
+// GetMetadata returns the transaction type and transaction size with the given
+// hash.
+func (p *testTxPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
+	p.lock.Lock()
+	defer p.lock.Unlock()
+
+	tx := p.pool[hash]
+	if tx != nil {
+		return &txpool.TxMetadata{
+			Type: tx.Type(),
+			Size: tx.Size(),
+		}
+	}
+	return nil
+}
+
 // Add appends a batch of transactions to the pool, and notifies any
 // listeners if the addition channel is non nil
 func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error {

eth/protocols/eth/broadcast.go

@@ -116,10 +116,10 @@ func (p *Peer) announceTransactions() {
 				size         common.StorageSize
 			)
 			for count = 0; count < len(queue) && size < maxTxPacketSize; count++ {
-				if tx := p.txpool.Get(queue[count]); tx != nil {
+				if meta := p.txpool.GetMetadata(queue[count]); meta != nil {
 					pending = append(pending, queue[count])
-					pendingTypes = append(pendingTypes, tx.Type())
+					pendingTypes = append(pendingTypes, meta.Type)
-					pendingSizes = append(pendingSizes, uint32(tx.Size()))
+					pendingSizes = append(pendingSizes, uint32(meta.Size))
 					size += common.HashLength
 				}
 			}

eth/protocols/eth/handler.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/txpool"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/ethereum/go-ethereum/p2p"
@@ -90,6 +91,10 @@ type TxPool interface {
 	// GetRLP retrieves the RLP-encoded transaction from the local txpool with
 	// the given hash.
 	GetRLP(hash common.Hash) []byte
+
+	// GetMetadata returns the transaction type and transaction size with the
+	// given transaction hash.
+	GetMetadata(hash common.Hash) *txpool.TxMetadata
 }
 
 // MakeProtocols constructs the P2P protocol definitions for `eth`.

eth/tracers/api.go

@@ -950,7 +950,9 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc
 	vmctx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil)
 	// Apply the customization rules if required.
 	if config != nil {
-		config.BlockOverrides.Apply(&vmctx)
+		if overrideErr := config.BlockOverrides.Apply(&vmctx); overrideErr != nil {
+			return nil, overrideErr
+		}
 		rules := api.backend.ChainConfig().Rules(vmctx.BlockNumber, vmctx.Random != nil, vmctx.Time)
 
 		precompiles = vm.ActivePrecompiledContracts(rules)

eth/tracers/logger/access_list_tracer.go

@@ -103,16 +103,10 @@ type AccessListTracer struct {
 // NewAccessListTracer creates a new tracer that can generate AccessLists.
 // An optional AccessList can be specified to occupy slots and addresses in
 // the resulting accesslist.
-func NewAccessListTracer(acl types.AccessList, from, to common.Address, precompiles []common.Address) *AccessListTracer {
+func NewAccessListTracer(acl types.AccessList, addressesToExclude map[common.Address]struct{}) *AccessListTracer {
-	excl := map[common.Address]struct{}{
-		from: {}, to: {},
-	}
-	for _, addr := range precompiles {
-		excl[addr] = struct{}{}
-	}
 	list := newAccessList()
 	for _, al := range acl {
-		if _, ok := excl[al.Address]; !ok {
+		if _, ok := addressesToExclude[al.Address]; !ok {
 			list.addAddress(al.Address)
 		}
 		for _, slot := range al.StorageKeys {
@@ -120,7 +114,7 @@ func NewAccessListTracer(acl types.AccessList, from, to common.Address, precompi
 		}
 	}
 	return &AccessListTracer{
-		excl: excl,
+		excl: addressesToExclude,
 		list: list,
 	}
 }

ethdb/database.go

@@ -17,7 +17,10 @@
 // Package ethdb defines the interfaces for an Ethereum data store.
 package ethdb
 
-import "io"
+import (
+	"errors"
+	"io"
+)
 
 // KeyValueReader wraps the Has and Get method of a backing data store.
 type KeyValueReader interface {
@@ -37,10 +40,14 @@ type KeyValueWriter interface {
 	Delete(key []byte) error
 }
 
+var ErrTooManyKeys = errors.New("too many keys in deleted range")
+
 // KeyValueRangeDeleter wraps the DeleteRange method of a backing data store.
 type KeyValueRangeDeleter interface {
 	// DeleteRange deletes all of the keys (and values) in the range [start,end)
 	// (inclusive on start, exclusive on end).
+	// Some implementations of DeleteRange may return ErrTooManyKeys after
+	// partially deleting entries in the given range.
 	DeleteRange(start, end []byte) error
 }
 

ethdb/leveldb/leveldb.go

@@ -207,8 +207,6 @@ func (db *Database) Delete(key []byte) error {
 	return db.db.Delete(key, nil)
 }
 
-var ErrTooManyKeys = errors.New("too many keys in deleted range")
-
 // DeleteRange deletes all of the keys (and values) in the range [start,end)
 // (inclusive on start, exclusive on end).
 // Note that this is a fallback implementation as leveldb does not natively
@@ -228,7 +226,7 @@ func (db *Database) DeleteRange(start, end []byte) error {
 			if err := batch.Write(); err != nil {
 				return err
 			}
-			return ErrTooManyKeys
+			return ethdb.ErrTooManyKeys
 		}
 		if err := batch.Delete(it.Key()); err != nil {
 			return err

internal/ethapi/api.go

@@ -549,21 +549,23 @@ func (api *BlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, block
 }
 
 // GetUncleCountByBlockNumber returns number of uncles in the block for the given block number
-func (api *BlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint {
+func (api *BlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) (*hexutil.Uint, error) {
-	if block, _ := api.b.BlockByNumber(ctx, blockNr); block != nil {
+	block, err := api.b.BlockByNumber(ctx, blockNr)
+	if block != nil {
 		n := hexutil.Uint(len(block.Uncles()))
-		return &n
+		return &n, nil
 	}
-	return nil
+	return nil, err
 }
 
 // GetUncleCountByBlockHash returns number of uncles in the block for the given block hash
-func (api *BlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint {
+func (api *BlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) (*hexutil.Uint, error) {
-	if block, _ := api.b.BlockByHash(ctx, blockHash); block != nil {
+	block, err := api.b.BlockByHash(ctx, blockHash)
+	if block != nil {
 		n := hexutil.Uint(len(block.Uncles()))
-		return &n
+		return &n, nil
 	}
-	return nil
+	return nil, err
 }
 
 // GetCode returns the code stored at the given address in the state for the given block number.
@@ -596,9 +598,7 @@ func (api *BlockChainAPI) GetStorageAt(ctx context.Context, address common.Addre
 func (api *BlockChainAPI) GetBlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]map[string]interface{}, error) {
 	block, err := api.b.BlockByNumberOrHash(ctx, blockNrOrHash)
 	if block == nil || err != nil {
-		// When the block doesn't exist, the RPC method should return JSON null
+		return nil, err
-		// as per specification.
-		return nil, nil
 	}
 	receipts, err := api.b.GetReceipts(ctx, block.Hash())
 	if err != nil {
@@ -660,7 +660,9 @@ func (context *ChainContext) Config() *params.ChainConfig {
 func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.StateDB, header *types.Header, overrides *override.StateOverride, blockOverrides *override.BlockOverrides, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) {
 	blockCtx := core.NewEVMBlockContext(header, NewChainContext(ctx, b), nil)
 	if blockOverrides != nil {
-		blockOverrides.Apply(&blockCtx)
+		if err := blockOverrides.Apply(&blockCtx); err != nil {
+			return nil, err
+		}
 	}
 	rules := b.ChainConfig().Rules(blockCtx.BlockNumber, blockCtx.Random != nil, blockCtx.Time)
 	precompiles := vm.ActivePrecompiledContracts(rules)
@@ -761,8 +763,7 @@ func (api *BlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockN
 	if err != nil {
 		return nil, err
 	}
-	// If the result contains a revert reason, try to unpack and return it.
+	if errors.Is(result.Err, vm.ErrExecutionReverted) {
-	if len(result.Revert()) > 0 {
 		return nil, newRevertError(result.Revert())
 	}
 	return result.Return(), result.Err
@@ -842,7 +843,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr
 	// Run the gas estimation and wrap any revertals into a custom return
 	estimate, revert, err := gasestimator.Estimate(ctx, call, opts, gasCap)
 	if err != nil {
-		if len(revert) > 0 {
+		if errors.Is(err, vm.ErrExecutionReverted) {
 			return 0, newRevertError(revert)
 		}
 		return 0, err
@@ -1116,12 +1117,13 @@ type accessListResult struct {
 
 // CreateAccessList creates an EIP-2930 type AccessList for the given transaction.
 // Reexec and BlockNrOrHash can be specified to create the accessList on top of a certain state.
-func (api *BlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) {
+// StateOverrides can be used to create the accessList while taking into account state changes from previous transactions.
+func (api *BlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash, stateOverrides *override.StateOverride) (*accessListResult, error) {
 	bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber)
 	if blockNrOrHash != nil {
 		bNrOrHash = *blockNrOrHash
 	}
-	acl, gasUsed, vmerr, err := AccessList(ctx, api.b, bNrOrHash, args)
+	acl, gasUsed, vmerr, err := AccessList(ctx, api.b, bNrOrHash, args, stateOverrides)
 	if err != nil {
 		return nil, err
 	}
@@ -1135,13 +1137,22 @@ func (api *BlockChainAPI) CreateAccessList(ctx context.Context, args Transaction
 // AccessList creates an access list for the given transaction.
 // If the accesslist creation fails an error is returned.
 // If the transaction itself fails, an vmErr is returned.
-func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrHash, args TransactionArgs) (acl types.AccessList, gasUsed uint64, vmErr error, err error) {
+func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrHash, args TransactionArgs, stateOverrides *override.StateOverride) (acl types.AccessList, gasUsed uint64, vmErr error, err error) {
 	// Retrieve the execution context
 	db, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash)
 	if db == nil || err != nil {
 		return nil, 0, nil, err
 	}
 
+	// Apply state overrides immediately after StateAndHeaderByNumberOrHash.
+	// If not applied here, there could be cases where user-specified overrides (e.g., nonce)
+	// may conflict with default values from the database, leading to inconsistencies.
+	if stateOverrides != nil {
+		if err := stateOverrides.Apply(db, nil); err != nil {
+			return nil, 0, nil, err
+		}
+	}
+
 	// Ensure any missing fields are filled, extract the recipient and input data
 	if err = args.setFeeDefaults(ctx, b, header); err != nil {
 		return nil, 0, nil, err
@@ -1165,10 +1176,33 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH
 	// Retrieve the precompiles since they don't need to be added to the access list
 	precompiles := vm.ActivePrecompiles(b.ChainConfig().Rules(header.Number, isPostMerge, header.Time))
 
+	// addressesToExclude contains sender, receiver, precompiles and valid authorizations
+	addressesToExclude := map[common.Address]struct{}{args.from(): {}, to: {}}
+	for _, addr := range precompiles {
+		addressesToExclude[addr] = struct{}{}
+	}
+
+	// Prevent redundant operations if args contain more authorizations than EVM may handle
+	maxAuthorizations := uint64(*args.Gas) / params.CallNewAccountGas
+	if uint64(len(args.AuthorizationList)) > maxAuthorizations {
+		return nil, 0, nil, errors.New("insufficient gas to process all authorizations")
+	}
+
+	for _, auth := range args.AuthorizationList {
+		// Duplicating stateTransition.validateAuthorization() logic
+		if (!auth.ChainID.IsZero() && auth.ChainID.CmpBig(b.ChainConfig().ChainID) != 0) || auth.Nonce+1 < auth.Nonce {
+			continue
+		}
+
+		if authority, err := auth.Authority(); err == nil {
+			addressesToExclude[authority] = struct{}{}
+		}
+	}
+
 	// Create an initial tracer
-	prevTracer := logger.NewAccessListTracer(nil, args.from(), to, precompiles)
+	prevTracer := logger.NewAccessListTracer(nil, addressesToExclude)
 	if args.AccessList != nil {
-		prevTracer = logger.NewAccessListTracer(*args.AccessList, args.from(), to, precompiles)
+		prevTracer = logger.NewAccessListTracer(*args.AccessList, addressesToExclude)
 	}
 	for {
 		if err := ctx.Err(); err != nil {
@@ -1185,7 +1219,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH
 		msg := args.ToMessage(header.BaseFee, true, true)
 
 		// Apply the transaction with the access list tracer
-		tracer := logger.NewAccessListTracer(accessList, args.from(), to, precompiles)
+		tracer := logger.NewAccessListTracer(accessList, addressesToExclude)
 		config := vm.Config{Tracer: tracer.Hooks(), NoBaseFee: true}
 		evm := b.GetEVM(ctx, statedb, header, &config, nil)
 
@@ -1224,37 +1258,41 @@ func NewTransactionAPI(b Backend, nonceLock *AddrLocker) *TransactionAPI {
 }
 
 // GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number.
-func (api *TransactionAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint {
+func (api *TransactionAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*hexutil.Uint, error) {
-	if block, _ := api.b.BlockByNumber(ctx, blockNr); block != nil {
+	block, err := api.b.BlockByNumber(ctx, blockNr)
+	if block != nil {
 		n := hexutil.Uint(len(block.Transactions()))
-		return &n
+		return &n, nil
 	}
-	return nil
+	return nil, err
 }
 
 // GetBlockTransactionCountByHash returns the number of transactions in the block with the given hash.
-func (api *TransactionAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint {
+func (api *TransactionAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) (*hexutil.Uint, error) {
-	if block, _ := api.b.BlockByHash(ctx, blockHash); block != nil {
+	block, err := api.b.BlockByHash(ctx, blockHash)
+	if block != nil {
 		n := hexutil.Uint(len(block.Transactions()))
-		return &n
+		return &n, nil
 	}
-	return nil
+	return nil, err
 }
 
 // GetTransactionByBlockNumberAndIndex returns the transaction for the given block number and index.
-func (api *TransactionAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) *RPCTransaction {
+func (api *TransactionAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (*RPCTransaction, error) {
-	if block, _ := api.b.BlockByNumber(ctx, blockNr); block != nil {
+	block, err := api.b.BlockByNumber(ctx, blockNr)
-		return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig())
+	if block != nil {
+		return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig()), nil
 	}
-	return nil
+	return nil, err
 }
 
 // GetTransactionByBlockHashAndIndex returns the transaction for the given block hash and index.
-func (api *TransactionAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) *RPCTransaction {
+func (api *TransactionAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) (*RPCTransaction, error) {
-	if block, _ := api.b.BlockByHash(ctx, blockHash); block != nil {
+	block, err := api.b.BlockByHash(ctx, blockHash)
-		return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig())
+	if block != nil {
+		return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig()), nil
 	}
-	return nil
+	return nil, err
 }
 
 // GetRawTransactionByBlockNumberAndIndex returns the bytes of the transaction for the given block number and index.

internal/ethapi/api_test.go

@@ -520,8 +520,12 @@ func (b testBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber)
 	if number == rpc.PendingBlockNumber {
 		return b.pending, nil
 	}
+	if number == rpc.EarliestBlockNumber {
+		number = 0
+	}
 	return b.chain.GetBlockByNumber(uint64(number)), nil
 }
+
 func (b testBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) {
 	return b.chain.GetBlockByHash(hash), nil
 }
@@ -618,6 +622,9 @@ func (b testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscripti
 func (b testBackend) NewMatcherBackend() filtermaps.MatcherBackend {
 	panic("implement me")
 }
+
+func (b testBackend) HistoryPruningCutoff() uint64 { return b.chain.HistoryPruningCutoff() }
+
 func TestEstimateGas(t *testing.T) {
 	t.Parallel()
 	// Initialize test accounts
@@ -1134,6 +1141,24 @@ func TestCall(t *testing.T) {
 			},
 			want: "0x0000000000000000000000000000000000000000000000000000000000000000",
 		},
+		{
+			name:        "unsupported block override beaconRoot",
+			blockNumber: rpc.LatestBlockNumber,
+			call:        TransactionArgs{},
+			blockOverrides: override.BlockOverrides{
+				BeaconRoot: &common.Hash{0, 1, 2},
+			},
+			expectErr: errors.New(`block override "beaconRoot" is not supported for this RPC method`),
+		},
+		{
+			name:        "unsupported block override withdrawals",
+			blockNumber: rpc.LatestBlockNumber,
+			call:        TransactionArgs{},
+			blockOverrides: override.BlockOverrides{
+				Withdrawals: &types.Withdrawals{},
+			},
+			expectErr: errors.New(`block override "withdrawals" is not supported for this RPC method`),
+		},
 	}
 	for _, tc := range testSuite {
 		result, err := api.Call(context.Background(), tc.call, &rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, &tc.overrides, &tc.blockOverrides)
@@ -3511,3 +3536,76 @@ func testRPCResponseWithFile(t *testing.T, testid int, result interface{}, rpc s
 func addressToHash(a common.Address) common.Hash {
 	return common.BytesToHash(a.Bytes())
 }
+
+func TestCreateAccessListWithStateOverrides(t *testing.T) {
+	// Initialize test backend
+	genesis := &core.Genesis{
+		Config: params.TestChainConfig,
+		Alloc: types.GenesisAlloc{
+			common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7"): {Balance: big.NewInt(1000000000000000000)},
+		},
+	}
+	backend := newTestBackend(t, 1, genesis, ethash.NewFaker(), nil)
+
+	// Create a new BlockChainAPI instance
+	api := NewBlockChainAPI(backend)
+
+	// Create test contract code - a simple storage contract
+	//
+	// SPDX-License-Identifier: MIT
+	// pragma solidity ^0.8.0;
+	//
+	// contract SimpleStorage {
+	//     uint256 private value;
+	//
+	//     function retrieve() public view returns (uint256) {
+	//         return value;
+	//     }
+	// }
+	var (
+		contractCode = hexutil.Bytes(common.Hex2Bytes("6080604052348015600f57600080fd5b506004361060285760003560e01c80632e64cec114602d575b600080fd5b60336047565b604051603e91906067565b60405180910390f35b60008054905090565b6000819050919050565b6061816050565b82525050565b6000602082019050607a6000830184605a565b9291505056"))
+		// Create state overrides with more complete state
+		contractAddr = common.HexToAddress("0x1234567890123456789012345678901234567890")
+		nonce        = hexutil.Uint64(1)
+		overrides    = &override.StateOverride{
+			contractAddr: override.OverrideAccount{
+				Code:    &contractCode,
+				Balance: (*hexutil.Big)(big.NewInt(1000000000000000000)),
+				Nonce:   &nonce,
+				State: map[common.Hash]common.Hash{
+					common.Hash{}: common.HexToHash("0x000000000000000000000000000000000000000000000000000000000000002a"),
+				},
+			},
+		}
+	)
+
+	// Create transaction arguments with gas and value
+	var (
+		from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7")
+		data = hexutil.Bytes(common.Hex2Bytes("2e64cec1")) // retrieve()
+		gas  = hexutil.Uint64(100000)
+		args = TransactionArgs{
+			From:  &from,
+			To:    &contractAddr,
+			Data:  &data,
+			Gas:   &gas,
+			Value: new(hexutil.Big),
+		}
+	)
+	// Call CreateAccessList
+	result, err := api.CreateAccessList(context.Background(), args, nil, overrides)
+	if err != nil {
+		t.Fatalf("Failed to create access list: %v", err)
+	}
+	if err != nil || result == nil {
+		t.Fatalf("Failed to create access list: %v", err)
+	}
+	require.NotNil(t, result.Accesslist)
+
+	// Verify access list contains the contract address and storage slot
+	expected := &types.AccessList{{
+		Address:     contractAddr,
+		StorageKeys: []common.Hash{{}},
+	}}
+	require.Equal(t, expected, result.Accesslist)
+}

internal/ethapi/backend.go

@@ -85,6 +85,7 @@ type Backend interface {
 
 	ChainConfig() *params.ChainConfig
 	Engine() consensus.Engine
+	HistoryPruningCutoff() uint64
 
 	// This is copied from filters.Backend
 	// eth/filters needs to be initialized from this backend type, so methods needed by

internal/ethapi/override/override.go

@@ -17,6 +17,7 @@
 package override
 
 import (
+	"errors"
 	"fmt"
 	"math/big"
 
@@ -128,12 +129,20 @@ type BlockOverrides struct {
 	PrevRandao    *common.Hash
 	BaseFeePerGas *hexutil.Big
 	BlobBaseFee   *hexutil.Big
+	BeaconRoot    *common.Hash
+	Withdrawals   *types.Withdrawals
 }
 
 // Apply overrides the given header fields into the given block context.
-func (o *BlockOverrides) Apply(blockCtx *vm.BlockContext) {
+func (o *BlockOverrides) Apply(blockCtx *vm.BlockContext) error {
 	if o == nil {
-		return
+		return nil
+	}
+	if o.BeaconRoot != nil {
+		return errors.New(`block override "beaconRoot" is not supported for this RPC method`)
+	}
+	if o.Withdrawals != nil {
+		return errors.New(`block override "withdrawals" is not supported for this RPC method`)
 	}
 	if o.Number != nil {
 		blockCtx.BlockNumber = o.Number.ToInt()
@@ -159,6 +168,7 @@ func (o *BlockOverrides) Apply(blockCtx *vm.BlockContext) {
 	if o.BlobBaseFee != nil {
 		blockCtx.BlobBaseFee = o.BlobBaseFee.ToInt()
 	}
+	return nil
 }
 
 // MakeHeader returns a new header object with the overridden

internal/ethapi/simulate.go

@@ -36,7 +36,6 @@ import (
 	"github.com/ethereum/go-ethereum/internal/ethapi/override"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/trie"
 )
 
 const (
@@ -95,6 +94,47 @@ type simOpts struct {
 	ReturnFullTransactions bool
 }
 
+// simChainHeadReader implements ChainHeaderReader which is needed as input for FinalizeAndAssemble.
+type simChainHeadReader struct {
+	context.Context
+	Backend
+}
+
+func (m *simChainHeadReader) Config() *params.ChainConfig {
+	return m.Backend.ChainConfig()
+}
+
+func (m *simChainHeadReader) CurrentHeader() *types.Header {
+	return m.Backend.CurrentHeader()
+}
+
+func (m *simChainHeadReader) GetHeader(hash common.Hash, number uint64) *types.Header {
+	header, err := m.Backend.HeaderByNumber(m.Context, rpc.BlockNumber(number))
+	if err != nil || header == nil {
+		return nil
+	}
+	if header.Hash() != hash {
+		return nil
+	}
+	return header
+}
+
+func (m *simChainHeadReader) GetHeaderByNumber(number uint64) *types.Header {
+	header, err := m.Backend.HeaderByNumber(m.Context, rpc.BlockNumber(number))
+	if err != nil {
+		return nil
+	}
+	return header
+}
+
+func (m *simChainHeadReader) GetHeaderByHash(hash common.Hash) *types.Header {
+	header, err := m.Backend.HeaderByHash(m.Context, hash)
+	if err != nil {
+		return nil
+	}
+	return header
+}
+
 // simulator is a stateful object that simulates a series of blocks.
 // it is not safe for concurrent use.
 type simulator struct {
@@ -209,6 +249,9 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header,
 	if sim.chainConfig.IsPrague(header.Number, header.Time) || sim.chainConfig.IsVerkle(header.Number, header.Time) {
 		core.ProcessParentBlockHash(header.ParentHash, evm)
 	}
+	if header.ParentBeaconRoot != nil {
+		core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, evm)
+	}
 	var allLogs []*types.Log
 	for i, call := range block.Calls {
 		if err := ctx.Err(); err != nil {
@@ -258,6 +301,10 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header,
 		}
 		callResults[i] = callRes
 	}
+	header.GasUsed = gasUsed
+	if sim.chainConfig.IsCancun(header.Number, header.Time) {
+		header.BlobGasUsed = &blobGasUsed
+	}
 	var requests [][]byte
 	// Process EIP-7685 requests
 	if sim.chainConfig.IsPrague(header.Number, header.Time) {
@@ -271,20 +318,16 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header,
 		// EIP-7251
 		core.ProcessConsolidationQueue(&requests, evm)
 	}
-	header.Root = sim.state.IntermediateRoot(true)
-	header.GasUsed = gasUsed
-	if sim.chainConfig.IsCancun(header.Number, header.Time) {
-		header.BlobGasUsed = &blobGasUsed
-	}
-	var withdrawals types.Withdrawals
-	if sim.chainConfig.IsShanghai(header.Number, header.Time) {
-		withdrawals = make([]*types.Withdrawal, 0)
-	}
 	if requests != nil {
 		reqHash := types.CalcRequestsHash(requests)
 		header.RequestsHash = &reqHash
 	}
-	b := types.NewBlock(header, &types.Body{Transactions: txes, Withdrawals: withdrawals}, receipts, trie.NewStackTrie(nil))
+	blockBody := &types.Body{Transactions: txes, Withdrawals: *block.BlockOverrides.Withdrawals}
+	chainHeadReader := &simChainHeadReader{ctx, sim.b}
+	b, err := sim.b.Engine().FinalizeAndAssemble(chainHeadReader, header, sim.state, blockBody, receipts)
+	if err != nil {
+		return nil, nil, err
+	}
 	repairLogs(callResults, b.Hash())
 	return b, callResults, nil
 }
@@ -346,6 +389,9 @@ func (sim *simulator) sanitizeChain(blocks []simBlock) ([]simBlock, error) {
 			n := new(big.Int).Add(prevNumber, big.NewInt(1))
 			block.BlockOverrides.Number = (*hexutil.Big)(n)
 		}
+		if block.BlockOverrides.Withdrawals == nil {
+			block.BlockOverrides.Withdrawals = &types.Withdrawals{}
+		}
 		diff := new(big.Int).Sub(block.BlockOverrides.Number.ToInt(), prevNumber)
 		if diff.Cmp(common.Big0) <= 0 {
 			return nil, &invalidBlockNumberError{fmt.Sprintf("block numbers must be in order: %d <= %d", block.BlockOverrides.Number.ToInt().Uint64(), prevNumber)}
@@ -360,7 +406,13 @@ func (sim *simulator) sanitizeChain(blocks []simBlock) ([]simBlock, error) {
 			for i := uint64(0); i < gap.Uint64(); i++ {
 				n := new(big.Int).Add(prevNumber, big.NewInt(int64(i+1)))
 				t := prevTimestamp + timestampIncrement
-				b := simBlock{BlockOverrides: &override.BlockOverrides{Number: (*hexutil.Big)(n), Time: (*hexutil.Uint64)(&t)}}
+				b := simBlock{
+					BlockOverrides: &override.BlockOverrides{
+						Number:      (*hexutil.Big)(n),
+						Time:        (*hexutil.Uint64)(&t),
+						Withdrawals: &types.Withdrawals{},
+					},
+				}
 				prevTimestamp = t
 				res = append(res, b)
 			}
@@ -405,6 +457,9 @@ func (sim *simulator) makeHeaders(blocks []simBlock) ([]*types.Header, error) {
 		var parentBeaconRoot *common.Hash
 		if sim.chainConfig.IsCancun(overrides.Number.ToInt(), (uint64)(*overrides.Time)) {
 			parentBeaconRoot = &common.Hash{}
+			if overrides.BeaconRoot != nil {
+				parentBeaconRoot = overrides.BeaconRoot
+			}
 		}
 		header = overrides.MakeHeader(&types.Header{
 			UncleHash:        types.EmptyUncleHash,

internal/ethapi/transaction_args_test.go

@@ -402,3 +402,5 @@ func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent)
 func (b *backendMock) Engine() consensus.Engine { return nil }
 
 func (b *backendMock) NewMatcherBackend() filtermaps.MatcherBackend { return nil }
+
+func (b *backendMock) HistoryPruningCutoff() uint64 { return 0 }

p2p/nat/nat.go

@@ -140,7 +140,7 @@ type ExtIP net.IP
 
 func (n ExtIP) ExternalIP() (net.IP, error)  { return net.IP(n), nil }
 func (n ExtIP) String() string               { return fmt.Sprintf("ExtIP(%v)", net.IP(n)) }
-func (n ExtIP) MarshalText() ([]byte, error) { return []byte(fmt.Sprintf("extip:%v", net.IP(n))), nil }
+func (n ExtIP) MarshalText() ([]byte, error) { return fmt.Appendf(nil, "extip:%v", net.IP(n)), nil }
 
 // These do nothing.
 

p2p/nat/natpmp.go

@@ -71,7 +71,7 @@ func (n *pmp) DeleteMapping(protocol string, extport, intport int) (err error) {
 }
 
 func (n *pmp) MarshalText() ([]byte, error) {
-	return []byte(fmt.Sprintf("natpmp:%v", n.gw)), nil
+	return fmt.Appendf(nil, "natpmp:%v", n.gw), nil
 }
 
 func discoverPMP() Interface {

p2p/nat/natupnp.go

@@ -82,7 +82,7 @@ func (n *upnp) ExternalIP() (addr net.IP, err error) {
 func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) (uint16, error) {
 	ip, err := n.internalAddress()
 	if err != nil {
-		return 0, nil // TODO: Shouldn't we return the error?
+		return 0, err
 	}
 	protocol = strings.ToUpper(protocol)
 	lifetimeS := uint32(lifetime / time.Second)
@@ -94,14 +94,15 @@ func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, li
 	if err == nil {
 		return uint16(extport), nil
 	}
-
+	// Try addAnyPortMapping if mapping specified port didn't work.
-	return uint16(extport), n.withRateLimit(func() error {
+	err = n.withRateLimit(func() error {
 		p, err := n.addAnyPortMapping(protocol, extport, intport, ip, desc, lifetimeS)
 		if err == nil {
 			extport = int(p)
 		}
 		return err
 	})
+	return uint16(extport), err
 }
 
 func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.IP, desc string, lifetimeS uint32) (uint16, error) {

params/network_params.go

@@ -20,14 +20,6 @@ package params
 // aren't necessarily consensus related.
 
 const (
-	// BloomBitsBlocks is the number of blocks a single bloom bit section vector
-	// contains on the server side.
-	BloomBitsBlocks uint64 = 4096
-
-	// BloomConfirms is the number of confirmation blocks before a bloom section is
-	// considered probably final and its rotated bits are calculated.
-	BloomConfirms = 256
-
 	// FullImmutabilityThreshold is the number of blocks after which a chain segment is
 	// considered immutable (i.e. soft finality). It is used by the downloader as a
 	// hard limit against deep ancestors, by the blockchain against deep reorgs, by

rpc/types.go

@@ -63,11 +63,11 @@ type jsonWriter interface {
 type BlockNumber int64
 
 const (
+	EarliestBlockNumber  = BlockNumber(-5)
 	SafeBlockNumber      = BlockNumber(-4)
 	FinalizedBlockNumber = BlockNumber(-3)
 	LatestBlockNumber    = BlockNumber(-2)
 	PendingBlockNumber   = BlockNumber(-1)
-	EarliestBlockNumber  = BlockNumber(0)
 )
 
 // UnmarshalJSON parses the given JSON fragment into a BlockNumber. It supports:

signer/core/apitypes/signed_data_internal_test.go

@@ -282,7 +282,7 @@ func TestTypedDataArrayValidate(t *testing.T) {
 			messageHash, tErr := td.HashStruct(td.PrimaryType, td.Message)
 			assert.NoError(t, tErr, "failed to hash message: %v", tErr)
 
-			digest := crypto.Keccak256Hash([]byte(fmt.Sprintf("%s%s%s", "\x19\x01", string(domainSeparator), string(messageHash))))
+			digest := crypto.Keccak256Hash(fmt.Appendf(nil, "%s%s%s", "\x19\x01", string(domainSeparator), string(messageHash)))
 			assert.Equal(t, tc.Digest, digest.String(), "digest doesn't not match")
 
 			assert.NoError(t, td.validate(), "validation failed", tErr)

signer/core/signed_data_test.go

@@ -369,7 +369,7 @@ func sign(typedData apitypes.TypedData) ([]byte, []byte, error) {
 	if err != nil {
 		return nil, nil, err
 	}
-	rawData := []byte(fmt.Sprintf("\x19\x01%s%s", string(domainSeparator), string(typedDataHash)))
+	rawData := fmt.Appendf(nil, "\x19\x01%s%s", string(domainSeparator), string(typedDataHash))
 	sighash := crypto.Keccak256(rawData)
 	return typedDataHash, sighash, nil
 }

trie/committer.go

@@ -57,32 +57,26 @@ func (c *committer) commit(path []byte, n node, parallel bool) node {
 	// Commit children, then parent, and remove the dirty flag.
 	switch cn := n.(type) {
 	case *shortNode:
-		// Commit child
-		collapsed := cn.copy()
-
 		// If the child is fullNode, recursively commit,
 		// otherwise it can only be hashNode or valueNode.
 		if _, ok := cn.Val.(*fullNode); ok {
-			collapsed.Val = c.commit(append(path, cn.Key...), cn.Val, false)
+			cn.Val = c.commit(append(path, cn.Key...), cn.Val, false)
 		}
 		// The key needs to be copied, since we're adding it to the
 		// modified nodeset.
-		collapsed.Key = hexToCompact(cn.Key)
+		cn.Key = hexToCompact(cn.Key)
-		hashedNode := c.store(path, collapsed)
+		hashedNode := c.store(path, cn)
 		if hn, ok := hashedNode.(hashNode); ok {
 			return hn
 		}
-		return collapsed
+		return cn
 	case *fullNode:
-		hashedKids := c.commitChildren(path, cn, parallel)
+		c.commitChildren(path, cn, parallel)
-		collapsed := cn.copy()
+		hashedNode := c.store(path, cn)
-		collapsed.Children = hashedKids
-
-		hashedNode := c.store(path, collapsed)
 		if hn, ok := hashedNode.(hashNode); ok {
 			return hn
 		}
-		return collapsed
+		return cn
 	case hashNode:
 		return cn
 	default:
@@ -92,11 +86,10 @@ func (c *committer) commit(path []byte, n node, parallel bool) node {
 }
 
 // commitChildren commits the children of the given fullnode
-func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) [17]node {
+func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) {
 	var (
 		wg      sync.WaitGroup
 		nodesMu sync.Mutex
-		children [17]node
 	)
 	for i := 0; i < 16; i++ {
 		child := n.Children[i]
@@ -106,22 +99,21 @@ func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) [17]
 		// If it's the hashed child, save the hash value directly.
 		// Note: it's impossible that the child in range [0, 15]
 		// is a valueNode.
-		if hn, ok := child.(hashNode); ok {
+		if _, ok := child.(hashNode); ok {
-			children[i] = hn
 			continue
 		}
 		// Commit the child recursively and store the "hashed" value.
 		// Note the returned node can be some embedded nodes, so it's
 		// possible the type is not hashNode.
 		if !parallel {
-			children[i] = c.commit(append(path, byte(i)), child, false)
+			n.Children[i] = c.commit(append(path, byte(i)), child, false)
 		} else {
 			wg.Add(1)
 			go func(index int) {
 				p := append(path, byte(index))
 				childSet := trienode.NewNodeSet(c.nodes.Owner)
 				childCommitter := newCommitter(childSet, c.tracer, c.collectLeaf)
-				children[index] = childCommitter.commit(p, child, false)
+				n.Children[index] = childCommitter.commit(p, child, false)
 				nodesMu.Lock()
 				c.nodes.MergeSet(childSet)
 				nodesMu.Unlock()
@@ -132,11 +124,6 @@ func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) [17]
 	if parallel {
 		wg.Wait()
 	}
-	// For the 17th child, it's possible the type is valuenode.
-	if n.Children[16] != nil {
-		children[16] = n.Children[16]
-	}
-	return children
 }
 
 // store hashes the node n and adds it to the modified nodeset. If leaf collection

trie/hasher.go

@@ -53,62 +53,56 @@ func returnHasherToPool(h *hasher) {
 	hasherPool.Put(h)
 }
 
-// hash collapses a node down into a hash node, also returning a copy of the
+// hash collapses a node down into a hash node.
-// original node initialized with the computed hash to replace the original one.
+func (h *hasher) hash(n node, force bool) node {
-func (h *hasher) hash(n node, force bool) (hashed node, cached node) {
 	// Return the cached hash if it's available
 	if hash, _ := n.cache(); hash != nil {
-		return hash, n
+		return hash
 	}
 	// Trie not processed yet, walk the children
 	switch n := n.(type) {
 	case *shortNode:
-		collapsed, cached := h.hashShortNodeChildren(n)
+		collapsed := h.hashShortNodeChildren(n)
 		hashed := h.shortnodeToHash(collapsed, force)
-		// We need to retain the possibly _not_ hashed node, in case it was too
-		// small to be hashed
 		if hn, ok := hashed.(hashNode); ok {
-			cached.flags.hash = hn
+			n.flags.hash = hn
 		} else {
-			cached.flags.hash = nil
+			n.flags.hash = nil
 		}
-		return hashed, cached
+		return hashed
 	case *fullNode:
-		collapsed, cached := h.hashFullNodeChildren(n)
+		collapsed := h.hashFullNodeChildren(n)
-		hashed = h.fullnodeToHash(collapsed, force)
+		hashed := h.fullnodeToHash(collapsed, force)
 		if hn, ok := hashed.(hashNode); ok {
-			cached.flags.hash = hn
+			n.flags.hash = hn
 		} else {
-			cached.flags.hash = nil
+			n.flags.hash = nil
 		}
-		return hashed, cached
+		return hashed
 	default:
 		// Value and hash nodes don't have children, so they're left as were
-		return n, n
+		return n
 	}
 }
 
-// hashShortNodeChildren collapses the short node. The returned collapsed node
+// hashShortNodeChildren returns a copy of the supplied shortNode, with its child
-// holds a live reference to the Key, and must not be modified.
+// being replaced by either the hash or an embedded node if the child is small.
-func (h *hasher) hashShortNodeChildren(n *shortNode) (collapsed, cached *shortNode) {
+func (h *hasher) hashShortNodeChildren(n *shortNode) *shortNode {
-	// Hash the short node's child, caching the newly hashed subtree
+	var collapsed shortNode
-	collapsed, cached = n.copy(), n.copy()
-	// Previously, we did copy this one. We don't seem to need to actually
-	// do that, since we don't overwrite/reuse keys
-	// cached.Key = common.CopyBytes(n.Key)
 	collapsed.Key = hexToCompact(n.Key)
-	// Unless the child is a valuenode or hashnode, hash it
 	switch n.Val.(type) {
 	case *fullNode, *shortNode:
-		collapsed.Val, cached.Val = h.hash(n.Val, false)
+		collapsed.Val = h.hash(n.Val, false)
+	default:
+		collapsed.Val = n.Val
 	}
-	return collapsed, cached
+	return &collapsed
 }
 
-func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached *fullNode) {
+// hashFullNodeChildren returns a copy of the supplied fullNode, with its child
-	// Hash the full node's children, caching the newly hashed subtrees
+// being replaced by either the hash or an embedded node if the child is small.
-	cached = n.copy()
+func (h *hasher) hashFullNodeChildren(n *fullNode) *fullNode {
-	collapsed = n.copy()
+	var children [17]node
 	if h.parallel {
 		var wg sync.WaitGroup
 		wg.Add(16)
@@ -116,9 +110,9 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached
 			go func(i int) {
 				hasher := newHasher(false)
 				if child := n.Children[i]; child != nil {
-					collapsed.Children[i], cached.Children[i] = hasher.hash(child, false)
+					children[i] = hasher.hash(child, false)
 				} else {
-					collapsed.Children[i] = nilValueNode
+					children[i] = nilValueNode
 				}
 				returnHasherToPool(hasher)
 				wg.Done()
@@ -128,19 +122,21 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached
 	} else {
 		for i := 0; i < 16; i++ {
 			if child := n.Children[i]; child != nil {
-				collapsed.Children[i], cached.Children[i] = h.hash(child, false)
+				children[i] = h.hash(child, false)
 			} else {
-				collapsed.Children[i] = nilValueNode
+				children[i] = nilValueNode
 			}
 		}
 	}
-	return collapsed, cached
+	if n.Children[16] != nil {
+		children[16] = n.Children[16]
+	}
+	return &fullNode{flags: nodeFlag{}, Children: children}
 }
 
-// shortnodeToHash creates a hashNode from a shortNode. The supplied shortnode
+// shortNodeToHash computes the hash of the given shortNode. The shortNode must
-// should have hex-type Key, which will be converted (without modification)
+// first be collapsed, with its key converted to compact form. If the RLP-encoded
-// into compact form for RLP encoding.
+// node data is smaller than 32 bytes, the node itself is returned.
-// If the rlp data is smaller than 32 bytes, `nil` is returned.
 func (h *hasher) shortnodeToHash(n *shortNode, force bool) node {
 	n.encode(h.encbuf)
 	enc := h.encodedBytes()
@@ -151,8 +147,8 @@ func (h *hasher) shortnodeToHash(n *shortNode, force bool) node {
 	return h.hashData(enc)
 }
 
-// fullnodeToHash is used to create a hashNode from a fullNode, (which
+// fullnodeToHash computes the hash of the given fullNode. If the RLP-encoded
-// may contain nil values)
+// node data is smaller than 32 bytes, the node itself is returned.
 func (h *hasher) fullnodeToHash(n *fullNode, force bool) node {
 	n.encode(h.encbuf)
 	enc := h.encodedBytes()
@@ -203,10 +199,10 @@ func (h *hasher) hashDataTo(dst, data []byte) {
 func (h *hasher) proofHash(original node) (collapsed, hashed node) {
 	switch n := original.(type) {
 	case *shortNode:
-		sn, _ := h.hashShortNodeChildren(n)
+		sn := h.hashShortNodeChildren(n)
 		return sn, h.shortnodeToHash(sn, false)
 	case *fullNode:
-		fn, _ := h.hashFullNodeChildren(n)
+		fn := h.hashFullNodeChildren(n)
 		return fn, h.fullnodeToHash(fn, false)
 	default:
 		// Value and hash nodes don't have children, so they're left as were

trie/node.go

@@ -79,15 +79,19 @@ func (n *fullNode) EncodeRLP(w io.Writer) error {
 	return eb.Flush()
 }
 
-func (n *fullNode) copy() *fullNode   { copy := *n; return &copy }
-func (n *shortNode) copy() *shortNode { copy := *n; return &copy }
-
 // nodeFlag contains caching-related metadata about a node.
 type nodeFlag struct {
 	hash  hashNode // cached hash of the node (may be nil)
 	dirty bool     // whether the node has changes that must be written to the database
 }
 
+func (n nodeFlag) copy() nodeFlag {
+	return nodeFlag{
+		hash:  common.CopyBytes(n.hash),
+		dirty: n.dirty,
+	}
+}
+
 func (n *fullNode) cache() (hashNode, bool)  { return n.flags.hash, n.flags.dirty }
 func (n *shortNode) cache() (hashNode, bool) { return n.flags.hash, n.flags.dirty }
 func (n hashNode) cache() (hashNode, bool)   { return nil, true }
@@ -228,7 +232,9 @@ func decodeRef(buf []byte) (node, []byte, error) {
 			err := fmt.Errorf("oversized embedded node (size is %d bytes, want size < %d)", size, hashLen)
 			return nil, buf, err
 		}
-		n, err := decodeNode(nil, buf)
+		// The buffer content has already been copied or is safe to use;
+		// no additional copy is required.
+		n, err := decodeNodeUnsafe(nil, buf)
 		return n, rest, err
 	case kind == rlp.String && len(val) == 0:
 		// empty node

trie/trie.go

@@ -29,11 +29,11 @@ import (
 	"github.com/ethereum/go-ethereum/triedb/database"
 )
 
-// Trie is a Merkle Patricia Trie. Use New to create a trie that sits on
+// Trie represents a Merkle Patricia Trie. Use New to create a trie that operates
-// top of a database. Whenever trie performs a commit operation, the generated
+// on top of a node database. During a commit operation, the trie collects all
-// nodes will be gathered and returned in a set. Once the trie is committed,
+// modified nodes into a set for return. After committing, the trie becomes
-// it's not usable anymore. Callers have to re-create the trie with new root
+// unusable, and callers must recreate it with the new root based on the updated
-// based on the updated trie database.
+// trie database.
 //
 // Trie is not safe for concurrent use.
 type Trie struct {
@@ -67,13 +67,13 @@ func (t *Trie) newFlag() nodeFlag {
 // Copy returns a copy of Trie.
 func (t *Trie) Copy() *Trie {
 	return &Trie{
-		root:        t.root,
+		root:        copyNode(t.root),
 		owner:       t.owner,
 		committed:   t.committed,
+		unhashed:    t.unhashed,
+		uncommitted: t.uncommitted,
 		reader:      t.reader,
 		tracer:      t.tracer.copy(),
-		uncommitted: t.uncommitted,
-		unhashed:    t.unhashed,
 	}
 }
 
@@ -169,14 +169,12 @@ func (t *Trie) get(origNode node, key []byte, pos int) (value []byte, newnode no
 		}
 		value, newnode, didResolve, err = t.get(n.Val, key, pos+len(n.Key))
 		if err == nil && didResolve {
-			n = n.copy()
 			n.Val = newnode
 		}
 		return value, n, didResolve, err
 	case *fullNode:
 		value, newnode, didResolve, err = t.get(n.Children[key[pos]], key, pos+1)
 		if err == nil && didResolve {
-			n = n.copy()
 			n.Children[key[pos]] = newnode
 		}
 		return value, n, didResolve, err
@@ -257,7 +255,6 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod
 		}
 		item, newnode, resolved, err = t.getNode(n.Val, path, pos+len(n.Key))
 		if err == nil && resolved > 0 {
-			n = n.copy()
 			n.Val = newnode
 		}
 		return item, n, resolved, err
@@ -265,7 +262,6 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod
 	case *fullNode:
 		item, newnode, resolved, err = t.getNode(n.Children[path[pos]], path, pos+1)
 		if err == nil && resolved > 0 {
-			n = n.copy()
 			n.Children[path[pos]] = newnode
 		}
 		return item, n, resolved, err
@@ -375,7 +371,6 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error
 		if !dirty || err != nil {
 			return false, n, err
 		}
-		n = n.copy()
 		n.flags = t.newFlag()
 		n.Children[key[0]] = nn
 		return true, n, nil
@@ -483,7 +478,6 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) {
 		if !dirty || err != nil {
 			return false, n, err
 		}
-		n = n.copy()
 		n.flags = t.newFlag()
 		n.Children[key[0]] = nn
 
@@ -576,6 +570,36 @@ func concat(s1 []byte, s2 ...byte) []byte {
 	return r
 }
 
+// copyNode deep-copies the supplied node along with its children recursively.
+func copyNode(n node) node {
+	switch n := (n).(type) {
+	case nil:
+		return nil
+	case valueNode:
+		return valueNode(common.CopyBytes(n))
+
+	case *shortNode:
+		return &shortNode{
+			flags: n.flags.copy(),
+			Key:   common.CopyBytes(n.Key),
+			Val:   copyNode(n.Val),
+		}
+	case *fullNode:
+		var children [17]node
+		for i, cn := range n.Children {
+			children[i] = copyNode(cn)
+		}
+		return &fullNode{
+			flags:    n.flags.copy(),
+			Children: children,
+		}
+	case hashNode:
+		return n
+	default:
+		panic(fmt.Sprintf("%T: unknown node type", n))
+	}
+}
+
 func (t *Trie) resolve(n node, prefix []byte) (node, error) {
 	if n, ok := n.(hashNode); ok {
 		return t.resolveAndTrack(n, prefix)
@@ -593,15 +617,16 @@ func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) {
 		return nil, err
 	}
 	t.tracer.onRead(prefix, blob)
-	return mustDecodeNode(n, blob), nil
+
+	// The returned node blob won't be changed afterward. No need to
+	// deep-copy the slice.
+	return decodeNodeUnsafe(n, blob)
 }
 
 // Hash returns the root hash of the trie. It does not write to the
 // database and can be used even if the trie doesn't have one.
 func (t *Trie) Hash() common.Hash {
-	hash, cached := t.hashRoot()
+	return common.BytesToHash(t.hashRoot().(hashNode))
-	t.root = cached
-	return common.BytesToHash(hash.(hashNode))
 }
 
 // Commit collects all dirty nodes in the trie and replaces them with the
@@ -652,9 +677,9 @@ func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) {
 }
 
 // hashRoot calculates the root hash of the given trie
-func (t *Trie) hashRoot() (node, node) {
+func (t *Trie) hashRoot() node {
 	if t.root == nil {
-		return hashNode(types.EmptyRootHash.Bytes()), nil
+		return hashNode(types.EmptyRootHash.Bytes())
 	}
 	// If the number of changes is below 100, we let one thread handle it
 	h := newHasher(t.unhashed >= 100)
@@ -662,8 +687,7 @@ func (t *Trie) hashRoot() (node, node) {
 		returnHasherToPool(h)
 		t.unhashed = 0
 	}()
-	hashed, cached := h.hash(t.root, true)
+	return h.hash(t.root, true)
-	return hashed, cached
 }
 
 // Witness returns a set containing all trie nodes that have been accessed.

trie/trie_test.go

@@ -1330,3 +1330,171 @@ func printSet(set *trienode.NodeSet) string {
 	}
 	return out.String()
 }
+
+func TestTrieCopy(t *testing.T) {
+	testTrieCopy(t, []kv{
+		{k: []byte("do"), v: []byte("verb")},
+		{k: []byte("ether"), v: []byte("wookiedoo")},
+		{k: []byte("horse"), v: []byte("stallion")},
+		{k: []byte("shaman"), v: []byte("horse")},
+		{k: []byte("doge"), v: []byte("coin")},
+		{k: []byte("dog"), v: []byte("puppy")},
+	})
+
+	var entries []kv
+	for i := 0; i < 256; i++ {
+		entries = append(entries, kv{k: testrand.Bytes(32), v: testrand.Bytes(32)})
+	}
+	testTrieCopy(t, entries)
+}
+
+func testTrieCopy(t *testing.T, entries []kv) {
+	tr := NewEmpty(nil)
+	for _, entry := range entries {
+		tr.Update(entry.k, entry.v)
+	}
+	trCpy := tr.Copy()
+
+	if tr.Hash() != trCpy.Hash() {
+		t.Errorf("Hash mismatch: old %v, copy %v", tr.Hash(), trCpy.Hash())
+	}
+
+	// Check iterator
+	it, _ := tr.NodeIterator(nil)
+	itCpy, _ := trCpy.NodeIterator(nil)
+
+	for it.Next(false) {
+		hasNext := itCpy.Next(false)
+		if !hasNext {
+			t.Fatal("Iterator is not matched")
+		}
+		if !bytes.Equal(it.Path(), itCpy.Path()) {
+			t.Fatal("Iterator is not matched")
+		}
+		if it.Leaf() != itCpy.Leaf() {
+			t.Fatal("Iterator is not matched")
+		}
+		if it.Leaf() && !bytes.Equal(it.LeafBlob(), itCpy.LeafBlob()) {
+			t.Fatal("Iterator is not matched")
+		}
+	}
+
+	// Check commit
+	root, nodes := tr.Commit(false)
+	rootCpy, nodesCpy := trCpy.Commit(false)
+	if root != rootCpy {
+		t.Fatal("root mismatch")
+	}
+	if len(nodes.Nodes) != len(nodesCpy.Nodes) {
+		t.Fatal("commit node mismatch")
+	}
+	for p, n := range nodes.Nodes {
+		nn, exists := nodesCpy.Nodes[p]
+		if !exists {
+			t.Fatalf("node not exists: %v", p)
+		}
+		if !reflect.DeepEqual(n, nn) {
+			t.Fatalf("node mismatch: %v", p)
+		}
+	}
+}
+
+func TestTrieCopyOldTrie(t *testing.T) {
+	testTrieCopyOldTrie(t, []kv{
+		{k: []byte("do"), v: []byte("verb")},
+		{k: []byte("ether"), v: []byte("wookiedoo")},
+		{k: []byte("horse"), v: []byte("stallion")},
+		{k: []byte("shaman"), v: []byte("horse")},
+		{k: []byte("doge"), v: []byte("coin")},
+		{k: []byte("dog"), v: []byte("puppy")},
+	})
+
+	var entries []kv
+	for i := 0; i < 256; i++ {
+		entries = append(entries, kv{k: testrand.Bytes(32), v: testrand.Bytes(32)})
+	}
+	testTrieCopyOldTrie(t, entries)
+}
+
+func testTrieCopyOldTrie(t *testing.T, entries []kv) {
+	tr := NewEmpty(nil)
+	for _, entry := range entries {
+		tr.Update(entry.k, entry.v)
+	}
+	hash := tr.Hash()
+
+	trCpy := tr.Copy()
+	for _, val := range entries {
+		if rand.Intn(2) == 0 {
+			trCpy.Delete(val.k)
+		} else {
+			trCpy.Update(val.k, testrand.Bytes(32))
+		}
+	}
+	for i := 0; i < 10; i++ {
+		trCpy.Update(testrand.Bytes(32), testrand.Bytes(32))
+	}
+	trCpy.Hash()
+	trCpy.Commit(false)
+
+	// Traverse the original tree, the changes made on the copy one shouldn't
+	// affect the old one
+	for _, entry := range entries {
+		d, _ := tr.Get(entry.k)
+		if !bytes.Equal(d, entry.v) {
+			t.Errorf("Unexpected data, key: %v, want: %v, got: %v", entry.k, entry.v, d)
+		}
+	}
+	if tr.Hash() != hash {
+		t.Errorf("Hash mismatch: old %v, new %v", hash, tr.Hash())
+	}
+}
+
+func TestTrieCopyNewTrie(t *testing.T) {
+	testTrieCopyNewTrie(t, []kv{
+		{k: []byte("do"), v: []byte("verb")},
+		{k: []byte("ether"), v: []byte("wookiedoo")},
+		{k: []byte("horse"), v: []byte("stallion")},
+		{k: []byte("shaman"), v: []byte("horse")},
+		{k: []byte("doge"), v: []byte("coin")},
+		{k: []byte("dog"), v: []byte("puppy")},
+	})
+
+	var entries []kv
+	for i := 0; i < 256; i++ {
+		entries = append(entries, kv{k: testrand.Bytes(32), v: testrand.Bytes(32)})
+	}
+	testTrieCopyNewTrie(t, entries)
+}
+
+func testTrieCopyNewTrie(t *testing.T, entries []kv) {
+	tr := NewEmpty(nil)
+	for _, entry := range entries {
+		tr.Update(entry.k, entry.v)
+	}
+	trCpy := tr.Copy()
+	hash := trCpy.Hash()
+
+	for _, val := range entries {
+		if rand.Intn(2) == 0 {
+			tr.Delete(val.k)
+		} else {
+			tr.Update(val.k, testrand.Bytes(32))
+		}
+	}
+	for i := 0; i < 10; i++ {
+		tr.Update(testrand.Bytes(32), testrand.Bytes(32))
+	}
+
+	// Traverse the original tree, the changes made on the copy one shouldn't
+	// affect the old one
+	for _, entry := range entries {
+		d, _ := trCpy.Get(entry.k)
+		if !bytes.Equal(d, entry.v) {
+			t.Errorf("Unexpected data, key: %v, want: %v, got: %v", entry.k, entry.v, d)
+		}
+	}
+	if trCpy.Hash() != hash {
+		t.Errorf("Hash mismatch: old %v, new %v", hash, tr.Hash())
+	}
+}

triedb/database/database.go

@@ -27,6 +27,8 @@ type NodeReader interface {
 	// node path and the corresponding node hash. No error will be returned
 	// if the node is not found.
 	//
+	// The returned node content won't be changed after the call.
+	//
 	// Don't modify the returned byte slice since it's not deep-copied and
 	// still be referenced by database.
 	Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error)

triedb/pathdb/iterator_test.go

@@ -371,27 +371,27 @@ func TestAccountIteratorTraversalValues(t *testing.T) {
 		h = make(map[common.Hash][]byte)
 	)
 	for i := byte(2); i < 0xff; i++ {
-		a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i))
+		a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i)
 		if i > 20 && i%2 == 0 {
-			b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i))
+			b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i)
 		}
 		if i%4 == 0 {
-			c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i))
+			c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i)
 		}
 		if i%7 == 0 {
-			d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i))
+			d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i)
 		}
 		if i%8 == 0 {
-			e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i))
+			e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i)
 		}
 		if i > 50 || i < 85 {
-			f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i))
+			f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i)
 		}
 		if i%64 == 0 {
-			g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i))
+			g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i)
 		}
 		if i%128 == 0 {
-			h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i))
+			h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i)
 		}
 	}
 	// Assemble a stack of snapshots from the account layers
@@ -480,27 +480,27 @@ func TestStorageIteratorTraversalValues(t *testing.T) {
 		h = make(map[common.Hash][]byte)
 	)
 	for i := byte(2); i < 0xff; i++ {
-		a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i))
+		a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i)
 		if i > 20 && i%2 == 0 {
-			b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i))
+			b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i)
 		}
 		if i%4 == 0 {
-			c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i))
+			c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i)
 		}
 		if i%7 == 0 {
-			d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i))
+			d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i)
 		}
 		if i%8 == 0 {
-			e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i))
+			e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i)
 		}
 		if i > 50 || i < 85 {
-			f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i))
+			f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i)
 		}
 		if i%64 == 0 {
-			g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i))
+			g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i)
 		}
 		if i%128 == 0 {
-			h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i))
+			h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i)
 		}
 	}
 	// Assemble a stack of snapshots from the account layers

triedb/pathdb/nodes.go

@@ -37,7 +37,8 @@ import (
 // the combined trie node set from several aggregated state transitions.
 type nodeSet struct {
 	size         uint64                                    // aggregated size of the trie node
-	nodes map[common.Hash]map[string]*trienode.Node // node set, mapped by owner and path
+	accountNodes map[string]*trienode.Node                 // account trie nodes, mapped by path
+	storageNodes map[common.Hash]map[string]*trienode.Node // storage trie nodes, mapped by owner and path
 }
 
 // newNodeSet constructs the set with the provided dirty trie nodes.
@@ -46,7 +47,17 @@ func newNodeSet(nodes map[common.Hash]map[string]*trienode.Node) *nodeSet {
 	if nodes == nil {
 		nodes = make(map[common.Hash]map[string]*trienode.Node)
 	}
-	s := &nodeSet{nodes: nodes}
+	s := &nodeSet{
+		accountNodes: make(map[string]*trienode.Node),
+		storageNodes: make(map[common.Hash]map[string]*trienode.Node),
+	}
+	for owner, subset := range nodes {
+		if owner == (common.Hash{}) {
+			s.accountNodes = subset
+		} else {
+			s.storageNodes[owner] = subset
+		}
+	}
 	s.computeSize()
 	return s
 }
@@ -54,13 +65,12 @@ func newNodeSet(nodes map[common.Hash]map[string]*trienode.Node) *nodeSet {
 // computeSize calculates the database size of the held trie nodes.
 func (s *nodeSet) computeSize() {
 	var size uint64
-	for owner, subset := range s.nodes {
+	for path, n := range s.accountNodes {
-		var prefix int
+		size += uint64(len(n.Blob) + len(path))
-		if owner != (common.Hash{}) {
-			prefix = common.HashLength // owner (32 bytes) for storage trie nodes
 	}
+	for _, subset := range s.storageNodes {
 		for path, n := range subset {
-			size += uint64(prefix + len(n.Blob) + len(path))
+			size += uint64(common.HashLength + len(n.Blob) + len(path))
 		}
 	}
 	s.size = size
@@ -79,15 +89,18 @@ func (s *nodeSet) updateSize(delta int64) {
 
 // node retrieves the trie node with node path and its trie identifier.
 func (s *nodeSet) node(owner common.Hash, path []byte) (*trienode.Node, bool) {
-	subset, ok := s.nodes[owner]
+	// Account trie node
+	if owner == (common.Hash{}) {
+		n, ok := s.accountNodes[string(path)]
+		return n, ok
+	}
+	// Storage trie node
+	subset, ok := s.storageNodes[owner]
 	if !ok {
 		return nil, false
 	}
 	n, ok := subset[string(path)]
-	if !ok {
+	return n, ok
-		return nil, false
-	}
-	return n, true
 }
 
 // merge integrates the provided dirty nodes into the set. The provided nodeset
@@ -97,15 +110,24 @@ func (s *nodeSet) merge(set *nodeSet) {
 		delta     int64   // size difference resulting from node merging
 		overwrite counter // counter of nodes being overwritten
 	)
-	for owner, subset := range set.nodes {
+
-		var prefix int
+	// Merge account nodes
-		if owner != (common.Hash{}) {
+	for path, n := range set.accountNodes {
-			prefix = common.HashLength
+		if orig, exist := s.accountNodes[path]; !exist {
+			delta += int64(len(n.Blob) + len(path))
+		} else {
+			delta += int64(len(n.Blob) - len(orig.Blob))
+			overwrite.add(len(orig.Blob) + len(path))
 		}
-		current, exist := s.nodes[owner]
+		s.accountNodes[path] = n
+	}
+
+	// Merge storage nodes
+	for owner, subset := range set.storageNodes {
+		current, exist := s.storageNodes[owner]
 		if !exist {
 			for path, n := range subset {
-				delta += int64(prefix + len(n.Blob) + len(path))
+				delta += int64(common.HashLength + len(n.Blob) + len(path))
 			}
 			// Perform a shallow copy of the map for the subset instead of claiming it
 			// directly from the provided nodeset to avoid potential concurrent map
@@ -113,19 +135,19 @@ func (s *nodeSet) merge(set *nodeSet) {
 			// accessible even after merging. Therefore, ownership of the nodes map
 			// should still belong to the original layer, and any modifications to it
 			// should be prevented.
-			s.nodes[owner] = maps.Clone(subset)
+			s.storageNodes[owner] = maps.Clone(subset)
 			continue
 		}
 		for path, n := range subset {
 			if orig, exist := current[path]; !exist {
-				delta += int64(prefix + len(n.Blob) + len(path))
+				delta += int64(common.HashLength + len(n.Blob) + len(path))
 			} else {
 				delta += int64(len(n.Blob) - len(orig.Blob))
-				overwrite.add(prefix + len(orig.Blob) + len(path))
+				overwrite.add(common.HashLength + len(orig.Blob) + len(path))
 			}
 			current[path] = n
 		}
-		s.nodes[owner] = current
+		s.storageNodes[owner] = current
 	}
 	overwrite.report(gcTrieNodeMeter, gcTrieNodeBytesMeter)
 	s.updateSize(delta)
@@ -136,36 +158,40 @@ func (s *nodeSet) merge(set *nodeSet) {
 func (s *nodeSet) revertTo(db ethdb.KeyValueReader, nodes map[common.Hash]map[string]*trienode.Node) {
 	var delta int64
 	for owner, subset := range nodes {
-		current, ok := s.nodes[owner]
+		if owner == (common.Hash{}) {
+			// Account trie nodes
+			for path, n := range subset {
+				orig, ok := s.accountNodes[path]
+				if !ok {
+					blob := rawdb.ReadAccountTrieNode(db, []byte(path))
+					if bytes.Equal(blob, n.Blob) {
+						continue
+					}
+					panic(fmt.Sprintf("non-existent account node (%v) blob: %v", path, crypto.Keccak256Hash(n.Blob).Hex()))
+				}
+				s.accountNodes[path] = n
+				delta += int64(len(n.Blob)) - int64(len(orig.Blob))
+			}
+		} else {
+			// Storage trie nodes
+			current, ok := s.storageNodes[owner]
 			if !ok {
 				panic(fmt.Sprintf("non-existent subset (%x)", owner))
 			}
 			for path, n := range subset {
 				orig, ok := current[path]
 				if !ok {
-				// There is a special case in merkle tree that one child is removed
+					blob := rawdb.ReadStorageTrieNode(db, owner, []byte(path))
-				// from a fullNode which only has two children, and then a new child
-				// with different position is immediately inserted into the fullNode.
-				// In this case, the clean child of the fullNode will also be marked
-				// as dirty because of node collapse and expansion. In case of database
-				// rollback, don't panic if this "clean" node occurs which is not
-				// present in buffer.
-				var blob []byte
-				if owner == (common.Hash{}) {
-					blob = rawdb.ReadAccountTrieNode(db, []byte(path))
-				} else {
-					blob = rawdb.ReadStorageTrieNode(db, owner, []byte(path))
-				}
-				// Ignore the clean node in the case described above.
 					if bytes.Equal(blob, n.Blob) {
 						continue
 					}
-				panic(fmt.Sprintf("non-existent node (%x %v) blob: %v", owner, path, crypto.Keccak256Hash(n.Blob).Hex()))
+					panic(fmt.Sprintf("non-existent storage node (%x %v) blob: %v", owner, path, crypto.Keccak256Hash(n.Blob).Hex()))
 				}
 				current[path] = n
 				delta += int64(len(n.Blob)) - int64(len(orig.Blob))
 			}
 		}
+	}
 	s.updateSize(delta)
 }
 
@@ -184,8 +210,21 @@ type journalNodes struct {
 
 // encode serializes the content of trie nodes into the provided writer.
 func (s *nodeSet) encode(w io.Writer) error {
-	nodes := make([]journalNodes, 0, len(s.nodes))
+	nodes := make([]journalNodes, 0, len(s.storageNodes)+1)
-	for owner, subset := range s.nodes {
+
+	// Encode account nodes
+	if len(s.accountNodes) > 0 {
+		entry := journalNodes{Owner: common.Hash{}}
+		for path, node := range s.accountNodes {
+			entry.Nodes = append(entry.Nodes, journalNode{
+				Path: []byte(path),
+				Blob: node.Blob,
+			})
+		}
+		nodes = append(nodes, entry)
+	}
+	// Encode storage nodes
+	for owner, subset := range s.storageNodes {
 		entry := journalNodes{Owner: owner}
 		for path, node := range subset {
 			entry.Nodes = append(entry.Nodes, journalNode{
@@ -204,8 +243,21 @@ func (s *nodeSet) decode(r *rlp.Stream) error {
 	if err := r.Decode(&encoded); err != nil {
 		return fmt.Errorf("load nodes: %v", err)
 	}
-	nodes := make(map[common.Hash]map[string]*trienode.Node)
+	s.accountNodes = make(map[string]*trienode.Node)
+	s.storageNodes = make(map[common.Hash]map[string]*trienode.Node)
+
 	for _, entry := range encoded {
+		if entry.Owner == (common.Hash{}) {
+			// Account nodes
+			for _, n := range entry.Nodes {
+				if len(n.Blob) > 0 {
+					s.accountNodes[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob)
+				} else {
+					s.accountNodes[string(n.Path)] = trienode.NewDeleted()
+				}
+			}
+		} else {
+			// Storage nodes
 			subset := make(map[string]*trienode.Node)
 			for _, n := range entry.Nodes {
 				if len(n.Blob) > 0 {
@@ -214,33 +266,38 @@ func (s *nodeSet) decode(r *rlp.Stream) error {
 					subset[string(n.Path)] = trienode.NewDeleted()
 				}
 			}
-		nodes[entry.Owner] = subset
+			s.storageNodes[entry.Owner] = subset
+		}
 	}
-	s.nodes = nodes
 	s.computeSize()
 	return nil
 }
 
 // write flushes nodes into the provided database batch as a whole.
 func (s *nodeSet) write(batch ethdb.Batch, clean *fastcache.Cache) int {
-	return writeNodes(batch, s.nodes, clean)
+	nodes := make(map[common.Hash]map[string]*trienode.Node)
+	if len(s.accountNodes) > 0 {
+		nodes[common.Hash{}] = s.accountNodes
+	}
+	for owner, subset := range s.storageNodes {
+		nodes[owner] = subset
+	}
+	return writeNodes(batch, nodes, clean)
 }
 
 // reset clears all cached trie node data.
 func (s *nodeSet) reset() {
-	s.nodes = make(map[common.Hash]map[string]*trienode.Node)
+	s.accountNodes = make(map[string]*trienode.Node)
+	s.storageNodes = make(map[common.Hash]map[string]*trienode.Node)
 	s.size = 0
 }
 
 // dbsize returns the approximate size of db write.
 func (s *nodeSet) dbsize() int {
 	var m int
-	for owner, nodes := range s.nodes {
+	m += len(s.accountNodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix
-		if owner == (common.Hash{}) {
+	for _, nodes := range s.storageNodes {
-			m += len(nodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix
-		} else {
 		m += len(nodes) * (len(rawdb.TrieNodeStoragePrefix)) // database key prefix
 	}
-	}
 	return m + int(s.size)
 }

version/version.go

@@ -19,6 +19,6 @@ package version
 const (
 	Major = 1          // Major version component of the current release
 	Minor = 15         // Minor version component of the current release
-	Patch = 6          // Patch version component of the current release
+	Patch = 8          // Patch version component of the current release
 	Meta  = "unstable" // Version metadata to append to the version string
 )