Merge 4c4fd5e64e into 12eabbd76d

cmd/geth/main.go

@@ -83,6 +83,7 @@ var (
 		utils.ExitWhenSyncedFlag,
 		utils.GCModeFlag,
 		utils.SnapshotFlag,
+		utils.TxSenderIndexFlag,
 		utils.TxLookupLimitFlag, // deprecated
 		utils.TransactionHistoryFlag,
 		utils.ChainHistoryFlag,

cmd/utils/flags.go

@@ -233,6 +233,12 @@ var (
 		Value:    true,
 		Category: flags.EthCategory,
 	}
+	TxSenderIndexFlag = &cli.BoolFlag{
+		Name:     "tx.index-sender",
+		Usage:    "Enable transaction indexing by sender and nonce",
+		Category: flags.EthCategory,
+		Value:    ethconfig.Defaults.TxIndexSender,
+	}
 	LightKDFFlag = &cli.BoolFlag{
 		Name:     "lightkdf",
 		Usage:    "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
@@ -1829,6 +1835,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.IsSet(StateSchemeFlag.Name) {
 		cfg.StateScheme = ctx.String(StateSchemeFlag.Name)
 	}
+	if ctx.IsSet(TxSenderIndexFlag.Name) {
+		cfg.TxIndexSender = ctx.Bool(TxSenderIndexFlag.Name)
+	}
 	// Parse transaction history flag, if user is still using legacy config
 	// file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'.
 	if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit {

core/blockchain.go

@@ -214,6 +214,9 @@ type BlockChainConfig struct {
 	// If the value is -1, indexing is disabled.
 	TxLookupLimit int64
 
+	// Enable sender+nonce transaction indexing
+	TxIndexSender bool
+
 	// StateSizeTracking indicates whether the state size tracking is enabled.
 	StateSizeTracking bool
 
@@ -1288,6 +1291,12 @@ func (bc *BlockChain) writeHeadBlock(block *types.Block) {
 	rawdb.WriteHeadFastBlockHash(batch, block.Hash())
 	rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64())
 	rawdb.WriteTxLookupEntriesByBlock(batch, block)
+
+	if bc.cfg.TxIndexSender {
+		signer := types.MakeSigner(bc.chainConfig, block.Number(), block.Time())
+		rawdb.WriteTxSenderNonceEntryByBlock(batch, block, signer)
+	}
+
 	rawdb.WriteHeadBlockHash(batch, block.Hash())
 
 	// Flush the whole batch into the disk, exit the node if failed

core/rawdb/accessors_indexes.go

@@ -134,6 +134,33 @@ func DeleteAllTxLookupEntries(db ethdb.KeyValueStore, condition func(common.Hash
 	}
 }
 
+// WriteTxSenderNonceIndex stores the mapping of sender+nonce to tx hash.
+func WriteTxSenderNonceEntry(db ethdb.KeyValueWriter, sender common.Address, nonce uint64, hash common.Hash) {
+	if err := db.Put(txSenderNonceKey(sender, nonce), hash.Bytes()); err != nil {
+		log.Crit("Failed to store sender nonce index", "err", err)
+	}
+}
+
+// WriteTxSenderNonceIndexByBlock stores a sender+nonce to transaction hash mapping
+// for every transaction in a block, enabling sender and nonce based transaction lookups.
+func WriteTxSenderNonceEntryByBlock(db ethdb.KeyValueWriter, block *types.Block, signer types.Signer) {
+	for _, tx := range block.Transactions() {
+		if sender, err := types.Sender(signer, tx); err == nil {
+			WriteTxSenderNonceEntry(db, sender, tx.Nonce(), tx.Hash())
+		}
+	}
+}
+
+// ReadTxSenderNonceIndex retrieves the hash for a specific sender and nonce.
+func ReadTxSenderNonceEntry(db ethdb.KeyValueReader, sender common.Address, nonce uint64) *common.Hash {
+	data, _ := db.Get(txSenderNonceKey(sender, nonce))
+	if len(data) == 0 {
+		return nil
+	}
+	hash := common.BytesToHash(data)
+	return &hash
+}
+
 // findTxInBlockBody traverses the given RLP-encoded block body, searching for
 // the transaction specified by its hash.
 func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Transaction, uint64, error) {

core/rawdb/accessors_indexes_test.go

@@ -297,3 +297,61 @@ func TestExtractReceiptFields(t *testing.T) {
 		}
 	}
 }
+
+func TestSenderNonceIndex(t *testing.T) {
+	db := NewMemoryDatabase()
+
+	addr1 := common.HexToAddress("0x1111111111111111111111111111111111111111")
+	addr2 := common.HexToAddress("0x2222222222222222222222222222222222222222")
+
+	txHash1 := common.HexToHash("0xaaaa") // Sender: addr1, Nonce: 1
+	txHash2 := common.HexToHash("0xbbbb") // Sender: addr1, Nonce: 2
+	txHash3 := common.HexToHash("0xcccc") // Sender: addr2, Nonce: 1
+
+	// Initially index is empty
+	if hash := ReadTxSenderNonceEntry(db, addr1, 1); hash != nil {
+		t.Fatalf("index should be empty, got %x", hash)
+	}
+
+	WriteTxSenderNonceEntry(db, addr1, 1, txHash1)
+	WriteTxSenderNonceEntry(db, addr1, 2, txHash2)
+	WriteTxSenderNonceEntry(db, addr2, 1, txHash3)
+
+	// Verify indices
+	tests := []struct {
+		name   string
+		sender common.Address
+		nonce  uint64
+		expect *common.Hash
+	}{
+		{"addr1-nonce1", addr1, 1, &txHash1},
+		{"addr1-nonce2", addr1, 2, &txHash2},
+		{"addr2-nonce1", addr2, 1, &txHash3},
+		{"addr2-nonce2 (missing)", addr2, 2, nil},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got := ReadTxSenderNonceEntry(db, tc.sender, tc.nonce)
+			if tc.expect == nil {
+				if got != nil {
+					t.Errorf("expected nil, got %x", got)
+				}
+			} else {
+				if got == nil {
+					t.Errorf("expected %x, got nil", *tc.expect)
+				} else if *got != *tc.expect {
+					t.Errorf("expected %x, got %x", *tc.expect, *got)
+				}
+			}
+		})
+	}
+
+	// Verify Overwrite
+	newHash := common.HexToHash("0xdddd")
+	WriteTxSenderNonceEntry(db, addr1, 1, newHash)
+
+	if got := ReadTxSenderNonceEntry(db, addr1, 1); *got != newHash {
+		t.Fatalf("failed to overwrite index: expected %x, got %x", newHash, got)
+	}
+}

core/rawdb/schema.go

@@ -115,6 +115,7 @@ var (
 	accessListPrefix    = []byte("j") // accessListPrefix + num (uint64 big endian) + hash -> block access list
 
 	txLookupPrefix        = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata
+	txSenderNoncePrefix   = []byte("x") // txSenderNoncePrefix + sender + nonce -> transaction hash
 	bloomBitsPrefix       = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits
 	SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value
 	SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value
@@ -225,6 +226,15 @@ func txLookupKey(hash common.Hash) []byte {
 	return append(txLookupPrefix, hash.Bytes()...)
 }
 
+// txSenderNonceKey = txSenderNoncePrefix + sender + nonce -> transaction hash
+func txSenderNonceKey(sender common.Address, nonce uint64) []byte {
+	buf := make([]byte, len(txSenderNoncePrefix)+common.AddressLength+8)
+	n := copy(buf, txSenderNoncePrefix)
+	n += copy(buf[n:], sender.Bytes())
+	binary.BigEndian.PutUint64(buf[n:], nonce)
+	return buf
+}
+
 // accountSnapshotKey = SnapshotAccountPrefix + hash
 func accountSnapshotKey(hash common.Hash) []byte {
 	return append(SnapshotAccountPrefix, hash.Bytes()...)

core/txpool/blobpool/blobpool.go

@@ -1608,6 +1608,25 @@ func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
 	}
 }
 
+// GetTxBySenderAndNonce returns a transaction of a sender and its corresponding nonce.
+func (p *BlobPool) GetTxBySenderAndNonce(sender common.Address, nonce uint64) *types.Transaction {
+	p.lock.RLock()
+	defer p.lock.RUnlock()
+
+	txs, ok := p.index[sender]
+	if !ok {
+		return nil
+	}
+	next := p.state.GetNonce(sender)
+	offset := int(nonce - next)
+
+	if offset < 0 || offset >= len(txs) {
+		return nil
+	}
+
+	return p.Get(txs[offset].hash)
+}
+
 // GetBlobs returns a number of blobs and proofs for the given versioned hashes.
 // Blobpool must place responses in the order given in the request, using null
 // for any missing blobs.

core/txpool/legacypool/legacypool.go

@@ -1036,6 +1036,25 @@ func (pool *LegacyPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
 	}
 }
 
+// GetTxBySenderAndNonce returns a transaction of a sender and it's corresponding nonce.
+func (pool *LegacyPool) GetTxBySenderAndNonce(sender common.Address, nonce uint64) *types.Transaction {
+	// Check if the transaction is in the pending pool
+	if txList := pool.pending[sender]; txList != nil {
+		if tx := txList.txs.items[nonce]; tx != nil {
+			return tx
+		}
+	}
+
+	// Check if the transaction is in the queue pool
+	if txList, ok := pool.queue.get(sender); ok {
+		if tx := txList.txs.items[nonce]; tx != nil {
+			return tx
+		}
+	}
+
+	return nil
+}
+
 // Has returns an indicator whether txpool has a transaction cached with the
 // given hash.
 func (pool *LegacyPool) Has(hash common.Hash) bool {

core/txpool/subpool.go

@@ -138,6 +138,9 @@ type SubPool interface {
 	// given transaction hash.
 	GetMetadata(hash common.Hash) *TxMetadata
 
+	// GetTxBySenderAndNonce returns a transaction of a sender and it's corresponding nonce.
+	GetTxBySenderAndNonce(sender common.Address, nonce uint64) *types.Transaction
+
 	// ValidateTxBasics checks whether a transaction is valid according to the consensus
 	// rules, but does not check state-dependent validation such as sufficient balance.
 	// This check is meant as a static check which can be performed without holding the

core/txpool/txpool.go

@@ -308,6 +308,16 @@ func (p *TxPool) GetMetadata(hash common.Hash) *TxMetadata {
 	return nil
 }
 
+// GetTxBySenderAndNonce returns a transaction of a sender and it's corresponding nonce.
+func (p *TxPool) GetTxBySenderAndNonce(sender common.Address, nonce uint64) *types.Transaction {
+	for _, subpool := range p.subpools {
+		if tx := subpool.GetTxBySenderAndNonce(sender, nonce); tx != nil {
+			return tx
+		}
+	}
+	return nil
+}
+
 // Add enqueues a batch of transactions into the pool if they are valid. Due
 // to the large transaction churn, add may postpone fully integrating the tx
 // to a later point to batch multiple ones together.

eth/api_backend.go

@@ -216,6 +216,32 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r
 	return nil, errors.New("invalid arguments; neither block nor hash specified")
 }
 
+// GetTransactionBySenderAndNonce returns the hash of a transaction for the given sender and nonce.
+// It checks the pool, then enforces a nonce check against the current state, and finally checks the historical TxSenderNonce index.
+func (b *EthAPIBackend) GetTransactionBySenderAndNonce(ctx context.Context, sender common.Address, nonce uint64) (*common.Hash, error) {
+	if pool := b.eth.TxPool(); pool != nil {
+		if tx := pool.GetTxBySenderAndNonce(sender, nonce); tx != nil {
+			hash := tx.Hash()
+			return &hash, nil
+		}
+	}
+
+	state, _, err := b.StateAndHeaderByNumberOrHash(ctx, rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber))
+	if err != nil {
+		return nil, err
+	}
+	currentNonce := state.GetNonce(sender)
+
+	// If the requested nonce is greater than or equal to the current nonce,
+	// and we already confirmed it's NOT in the pool, then the transaction
+	// definitely does not exist.
+	if nonce >= currentNonce {
+		return nil, nil
+	}
+	hash := rawdb.ReadTxSenderNonceEntry(b.eth.ChainDb(), sender, nonce)
+	return hash, nil
+}
+
 func (b *EthAPIBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) {
 	return b.eth.miner.Pending()
 }

eth/backend.go

@@ -238,6 +238,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 			StateScheme:             scheme,
 			HistoryPolicy:           histPolicy,
 			TxLookupLimit:           int64(min(config.TransactionHistory, math.MaxInt64)),
+			TxIndexSender:           config.TxIndexSender,
 			VmConfig: vm.Config{
 				EnablePreimageRecording: config.EnablePreimageRecording,
 			},

eth/ethconfig/config.go

@@ -57,6 +57,7 @@ var Defaults = Config{
 	TxLookupLimit:           2350000,
 	TransactionHistory:      2350000,
 	LogHistory:              2350000,
+	TxIndexSender:           false,
 	StateHistory:            pathdb.Defaults.StateHistory,
 	TrienodeHistory:         pathdb.Defaults.TrienodeHistory,
 	NodeFullValueCheckpoint: pathdb.Defaults.FullValueCheckpoint,
@@ -109,6 +110,7 @@ type Config struct {
 	// Deprecated: use 'TransactionHistory' instead.
 	TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
 
+	TxIndexSender        bool   `toml:",omitempty"` // Enables transaction indexing by sender and nonce (increases disk usage).
 	TransactionHistory   uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved.
 	LogHistory           uint64 `toml:",omitempty"` // The maximum number of blocks from head where a log search index is maintained.
 	LogNoHistory         bool   `toml:",omitempty"` // No log search index is maintained.

eth/ethconfig/gen_config.go

@@ -26,6 +26,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
 		NoPruning               bool
 		NoPrefetch              bool
 		TxLookupLimit           uint64 `toml:",omitempty"`
+		TxIndexSender           bool   `toml:",omitempty"`
 		TransactionHistory      uint64 `toml:",omitempty"`
 		LogHistory              uint64 `toml:",omitempty"`
 		LogNoHistory            bool   `toml:",omitempty"`
@@ -80,6 +81,7 @@ func (c Config) MarshalTOML() (interface{}, error) {
 	enc.NoPruning = c.NoPruning
 	enc.NoPrefetch = c.NoPrefetch
 	enc.TxLookupLimit = c.TxLookupLimit
+	enc.TxIndexSender = c.TxIndexSender
 	enc.TransactionHistory = c.TransactionHistory
 	enc.LogHistory = c.LogHistory
 	enc.LogNoHistory = c.LogNoHistory
@@ -138,6 +140,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 		NoPruning               *bool
 		NoPrefetch              *bool
 		TxLookupLimit           *uint64 `toml:",omitempty"`
+		TxIndexSender           *bool   `toml:",omitempty"`
 		TransactionHistory      *uint64 `toml:",omitempty"`
 		LogHistory              *uint64 `toml:",omitempty"`
 		LogNoHistory            *bool   `toml:",omitempty"`
@@ -213,6 +216,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
 	if dec.TxLookupLimit != nil {
 		c.TxLookupLimit = *dec.TxLookupLimit
 	}
+	if dec.TxIndexSender != nil {
+		c.TxIndexSender = *dec.TxIndexSender
+	}
 	if dec.TransactionHistory != nil {
 		c.TransactionHistory = *dec.TransactionHistory
 	}

ethclient/ethclient.go

@@ -288,6 +288,16 @@ func (ec *Client) TransactionByHash(ctx context.Context, hash common.Hash) (tx *
 	return json.tx, json.BlockNumber == nil, nil
 }
 
+// TransactionHashBySenderAndNonce returns the transaction hash for the given sender and nonce.
+func (ec *Client) TransactionHashBySenderAndNonce(ctx context.Context, sender common.Address, nonce hexutil.Uint64) (*common.Hash, error) {
+	var hash *common.Hash
+	err := ec.c.CallContext(ctx, &hash, "eth_getTransactionBySenderAndNonce", sender, nonce)
+	if err != nil {
+		return nil, err
+	}
+	return hash, nil
+}
+
 // TransactionSender returns the sender address of the given transaction. The transaction
 // must be known to the remote node and included in the blockchain at the given block and
 // index. The sender is the one derived by the protocol at the time of inclusion.

internal/ethapi/api.go

@@ -1525,6 +1525,15 @@ func (api *TransactionAPI) GetTransactionByHash(ctx context.Context, hash common
 	return newRPCTransaction(tx, blockHash, blockNumber, header.Time, index, header.BaseFee, api.b.ChainConfig()), nil
 }
 
+// GetTransactionBySenderAndNonce returns the hash of a transaction for the given sender and nonce.
+func (api *TransactionAPI) GetTransactionBySenderAndNonce(ctx context.Context, sender common.Address, nonce hexutil.Uint64) (*common.Hash, error) {
+	hash, err := api.b.GetTransactionBySenderAndNonce(ctx, sender, uint64(nonce))
+	if err != nil {
+		return nil, err
+	}
+	return hash, nil
+}
+
 // GetRawTransactionByHash returns the bytes of the transaction for the given hash.
 func (api *TransactionAPI) GetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) {
 	// Retrieve a finalized transaction, or a pooled otherwise

internal/ethapi/api_test.go

@@ -464,6 +464,7 @@ func fakeBlockHash(txh common.Hash) common.Hash {
 func newTestBackend(t *testing.T, n int, gspec *core.Genesis, engine consensus.Engine, generator func(i int, b *core.BlockGen)) *testBackend {
 	options := core.DefaultConfig().WithArchive(true)
 	options.TxLookupLimit = 0 // index all txs
+	options.TxIndexSender = true
 
 	accman, acc := newTestAccountManager(t)
 	gspec.Alloc[acc.Address] = types.Account{Balance: big.NewInt(params.Ether)}
@@ -709,6 +710,14 @@ func (b testBackend) HistoryPruningCutoff() uint64 {
 	return bn
 }
 
+// GetTransactionHashBySenderAndNonce implements the Backend interface for testing.
+func (b *testBackend) GetTransactionBySenderAndNonce(ctx context.Context, sender common.Address, nonce uint64) (*common.Hash, error) {
+	// Todo(shadow): to implement for txn pending in mempool after implementing other pending
+	// mempool functionality like  GetPoolTransactions
+	hash := rawdb.ReadTxSenderNonceEntry(b.db, sender, nonce)
+	return hash, nil
+}
+
 func TestEstimateGas(t *testing.T) {
 	t.Parallel()
 	// Initialize test accounts
@@ -4259,3 +4268,40 @@ func TestGetStorageValues(t *testing.T) {
 		t.Fatal("expected error for exceeding slot limit")
 	}
 }
+
+func TestGetTransactionBySenderAndNonce(t *testing.T) {
+	t.Parallel()
+
+	key, _ := crypto.HexToECDSA("b71c73a37e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+	addr := crypto.PubkeyToAddress(key.PublicKey)
+	nonce := uint64(0)
+	tx := types.NewTransaction(nonce, common.Address{0xaa}, big.NewInt(1000), 21000, big.NewInt(1000000000), nil)
+	signedTx, _ := types.SignTx(tx, types.HomesteadSigner{}, key)
+
+	genesis := &core.Genesis{
+		Config: params.TestChainConfig,
+		Alloc: core.GenesisAlloc{
+			addr: {Balance: big.NewInt(1000000000000000000)},
+		},
+	}
+
+	backend := newTestBackend(t, 1, genesis, ethash.NewFaker(), func(i int, b *core.BlockGen) {
+		if i == 0 {
+			b.AddTx(signedTx)
+		}
+	})
+
+	api := NewTransactionAPI(backend, new(AddrLocker))
+	ctx := context.Background()
+
+	result, err := api.GetTransactionBySenderAndNonce(ctx, addr, hexutil.Uint64(nonce))
+	if err != nil {
+		t.Fatalf("GetTransactionBySenderAndNonce failed: %v", err)
+	}
+	if result == nil {
+		t.Fatalf("Expected transaction, got nil")
+	}
+	if *result != signedTx.Hash() {
+		t.Errorf("Hash mismatch: have %x, want %x", *result, signedTx.Hash())
+	}
+}

internal/ethapi/backend.go

@@ -87,6 +87,7 @@ type Backend interface {
 	TxPoolContent() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction)
 	TxPoolContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction)
 	SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription
+	GetTransactionBySenderAndNonce(ctx context.Context, sender common.Address, nonce uint64) (*common.Hash, error)
 
 	ChainConfig() *params.ChainConfig
 	Engine() consensus.Engine

internal/ethapi/transaction_args_test.go

@@ -390,6 +390,11 @@ func (b *backendMock) GetCanonicalTransaction(txHash common.Hash) (bool, *types.
 func (b *backendMock) TxIndexDone() bool                                        { return true }
 func (b *backendMock) GetPoolTransactions() (types.Transactions, error)         { return nil, nil }
 func (b *backendMock) GetPoolTransaction(txHash common.Hash) *types.Transaction { return nil }
+
+// GetTransactionHashBySenderAndNonce implements the Backend interface for testing.
+func (b *backendMock) GetTransactionBySenderAndNonce(ctx context.Context, sender common.Address, nonce uint64) (*common.Hash, error) {
+	return nil, nil
+}
 func (b *backendMock) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) {
 	return 0, nil
 }