diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 850e26d161..33cb6b2441 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -83,6 +83,7 @@ var ( utils.ExitWhenSyncedFlag, utils.GCModeFlag, utils.SnapshotFlag, + utils.TxSenderIndexFlag, utils.TxLookupLimitFlag, // deprecated utils.TransactionHistoryFlag, utils.ChainHistoryFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c41cf4ee40..c10c46830d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -233,6 +233,12 @@ var ( Value: true, Category: flags.EthCategory, } + TxSenderIndexFlag = &cli.BoolFlag{ + Name: "tx.index-sender", + Usage: "Enable transaction indexing by sender and nonce", + Category: flags.EthCategory, + Value: ethconfig.Defaults.TxIndexSender, + } LightKDFFlag = &cli.BoolFlag{ Name: "lightkdf", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", @@ -1829,6 +1835,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(StateSchemeFlag.Name) { cfg.StateScheme = ctx.String(StateSchemeFlag.Name) } + if ctx.IsSet(TxSenderIndexFlag.Name) { + cfg.TxIndexSender = ctx.Bool(TxSenderIndexFlag.Name) + } // Parse transaction history flag, if user is still using legacy config // file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'. if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit { diff --git a/core/blockchain.go b/core/blockchain.go index 23b4169372..7c26aebe13 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -214,6 +214,9 @@ type BlockChainConfig struct { // If the value is -1, indexing is disabled. TxLookupLimit int64 + // Enable sender+nonce transaction indexing + TxIndexSender bool + // StateSizeTracking indicates whether the state size tracking is enabled. StateSizeTracking bool @@ -1288,6 +1291,12 @@ func (bc *BlockChain) writeHeadBlock(block *types.Block) { rawdb.WriteHeadFastBlockHash(batch, block.Hash()) rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64()) rawdb.WriteTxLookupEntriesByBlock(batch, block) + + if bc.cfg.TxIndexSender { + signer := types.MakeSigner(bc.chainConfig, block.Number(), block.Time()) + rawdb.WriteTxSenderNonceEntryByBlock(batch, block, signer) + } + rawdb.WriteHeadBlockHash(batch, block.Hash()) // Flush the whole batch into the disk, exit the node if failed diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 8c8c3ec9bb..b59fe354b9 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -134,6 +134,33 @@ func DeleteAllTxLookupEntries(db ethdb.KeyValueStore, condition func(common.Hash } } +// WriteTxSenderNonceIndex stores the mapping of sender+nonce to tx hash. +func WriteTxSenderNonceEntry(db ethdb.KeyValueWriter, sender common.Address, nonce uint64, hash common.Hash) { + if err := db.Put(txSenderNonceKey(sender, nonce), hash.Bytes()); err != nil { + log.Crit("Failed to store sender nonce index", "err", err) + } +} + +// WriteTxSenderNonceIndexByBlock stores a sender+nonce to transaction hash mapping +// for every transaction in a block, enabling sender and nonce based transaction lookups. +func WriteTxSenderNonceEntryByBlock(db ethdb.KeyValueWriter, block *types.Block, signer types.Signer) { + for _, tx := range block.Transactions() { + if sender, err := types.Sender(signer, tx); err == nil { + WriteTxSenderNonceEntry(db, sender, tx.Nonce(), tx.Hash()) + } + } +} + +// ReadTxSenderNonceIndex retrieves the hash for a specific sender and nonce. +func ReadTxSenderNonceEntry(db ethdb.KeyValueReader, sender common.Address, nonce uint64) *common.Hash { + data, _ := db.Get(txSenderNonceKey(sender, nonce)) + if len(data) == 0 { + return nil + } + hash := common.BytesToHash(data) + return &hash +} + // findTxInBlockBody traverses the given RLP-encoded block body, searching for // the transaction specified by its hash. func findTxInBlockBody(blockbody rlp.RawValue, target common.Hash) (*types.Transaction, uint64, error) { diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index a812fefeaa..926c92ba8f 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -297,3 +297,61 @@ func TestExtractReceiptFields(t *testing.T) { } } } + +func TestSenderNonceIndex(t *testing.T) { + db := NewMemoryDatabase() + + addr1 := common.HexToAddress("0x1111111111111111111111111111111111111111") + addr2 := common.HexToAddress("0x2222222222222222222222222222222222222222") + + txHash1 := common.HexToHash("0xaaaa") // Sender: addr1, Nonce: 1 + txHash2 := common.HexToHash("0xbbbb") // Sender: addr1, Nonce: 2 + txHash3 := common.HexToHash("0xcccc") // Sender: addr2, Nonce: 1 + + // Initially index is empty + if hash := ReadTxSenderNonceEntry(db, addr1, 1); hash != nil { + t.Fatalf("index should be empty, got %x", hash) + } + + WriteTxSenderNonceEntry(db, addr1, 1, txHash1) + WriteTxSenderNonceEntry(db, addr1, 2, txHash2) + WriteTxSenderNonceEntry(db, addr2, 1, txHash3) + + // Verify indices + tests := []struct { + name string + sender common.Address + nonce uint64 + expect *common.Hash + }{ + {"addr1-nonce1", addr1, 1, &txHash1}, + {"addr1-nonce2", addr1, 2, &txHash2}, + {"addr2-nonce1", addr2, 1, &txHash3}, + {"addr2-nonce2 (missing)", addr2, 2, nil}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := ReadTxSenderNonceEntry(db, tc.sender, tc.nonce) + if tc.expect == nil { + if got != nil { + t.Errorf("expected nil, got %x", got) + } + } else { + if got == nil { + t.Errorf("expected %x, got nil", *tc.expect) + } else if *got != *tc.expect { + t.Errorf("expected %x, got %x", *tc.expect, *got) + } + } + }) + } + + // Verify Overwrite + newHash := common.HexToHash("0xdddd") + WriteTxSenderNonceEntry(db, addr1, 1, newHash) + + if got := ReadTxSenderNonceEntry(db, addr1, 1); *got != newHash { + t.Fatalf("failed to overwrite index: expected %x, got %x", newHash, got) + } +} diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 54c76143b4..63fa939427 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -115,6 +115,7 @@ var ( accessListPrefix = []byte("j") // accessListPrefix + num (uint64 big endian) + hash -> block access list txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata + txSenderNoncePrefix = []byte("x") // txSenderNoncePrefix + sender + nonce -> transaction hash bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value @@ -225,6 +226,15 @@ func txLookupKey(hash common.Hash) []byte { return append(txLookupPrefix, hash.Bytes()...) } +// txSenderNonceKey = txSenderNoncePrefix + sender + nonce -> transaction hash +func txSenderNonceKey(sender common.Address, nonce uint64) []byte { + buf := make([]byte, len(txSenderNoncePrefix)+common.AddressLength+8) + n := copy(buf, txSenderNoncePrefix) + n += copy(buf[n:], sender.Bytes()) + binary.BigEndian.PutUint64(buf[n:], nonce) + return buf +} + // accountSnapshotKey = SnapshotAccountPrefix + hash func accountSnapshotKey(hash common.Hash) []byte { return append(SnapshotAccountPrefix, hash.Bytes()...) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 15f4430cc6..b690475d71 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1608,6 +1608,25 @@ func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { } } +// GetTxBySenderAndNonce returns a transaction of a sender and its corresponding nonce. +func (p *BlobPool) GetTxBySenderAndNonce(sender common.Address, nonce uint64) *types.Transaction { + p.lock.RLock() + defer p.lock.RUnlock() + + txs, ok := p.index[sender] + if !ok { + return nil + } + next := p.state.GetNonce(sender) + offset := int(nonce - next) + + if offset < 0 || offset >= len(txs) { + return nil + } + + return p.Get(txs[offset].hash) +} + // GetBlobs returns a number of blobs and proofs for the given versioned hashes. // Blobpool must place responses in the order given in the request, using null // for any missing blobs. diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 3d66803fd7..9e6f7d50bd 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1036,6 +1036,25 @@ func (pool *LegacyPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { } } +// GetTxBySenderAndNonce returns a transaction of a sender and it's corresponding nonce. +func (pool *LegacyPool) GetTxBySenderAndNonce(sender common.Address, nonce uint64) *types.Transaction { + // Check if the transaction is in the pending pool + if txList := pool.pending[sender]; txList != nil { + if tx := txList.txs.items[nonce]; tx != nil { + return tx + } + } + + // Check if the transaction is in the queue pool + if txList, ok := pool.queue.get(sender); ok { + if tx := txList.txs.items[nonce]; tx != nil { + return tx + } + } + + return nil +} + // Has returns an indicator whether txpool has a transaction cached with the // given hash. func (pool *LegacyPool) Has(hash common.Hash) bool { diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index 4cc1b193d6..95ba2c9039 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -138,6 +138,9 @@ type SubPool interface { // given transaction hash. GetMetadata(hash common.Hash) *TxMetadata + // GetTxBySenderAndNonce returns a transaction of a sender and it's corresponding nonce. + GetTxBySenderAndNonce(sender common.Address, nonce uint64) *types.Transaction + // ValidateTxBasics checks whether a transaction is valid according to the consensus // rules, but does not check state-dependent validation such as sufficient balance. // This check is meant as a static check which can be performed without holding the diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 9c78748422..a0bda404c5 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -308,6 +308,16 @@ func (p *TxPool) GetMetadata(hash common.Hash) *TxMetadata { return nil } +// GetTxBySenderAndNonce returns a transaction of a sender and it's corresponding nonce. +func (p *TxPool) GetTxBySenderAndNonce(sender common.Address, nonce uint64) *types.Transaction { + for _, subpool := range p.subpools { + if tx := subpool.GetTxBySenderAndNonce(sender, nonce); tx != nil { + return tx + } + } + return nil +} + // Add enqueues a batch of transactions into the pool if they are valid. Due // to the large transaction churn, add may postpone fully integrating the tx // to a later point to batch multiple ones together. diff --git a/eth/api_backend.go b/eth/api_backend.go index 5e3558d8eb..87160bab8a 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -216,6 +216,32 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r return nil, errors.New("invalid arguments; neither block nor hash specified") } +// GetTransactionBySenderAndNonce returns the hash of a transaction for the given sender and nonce. +// It checks the pool, then enforces a nonce check against the current state, and finally checks the historical TxSenderNonce index. +func (b *EthAPIBackend) GetTransactionBySenderAndNonce(ctx context.Context, sender common.Address, nonce uint64) (*common.Hash, error) { + if pool := b.eth.TxPool(); pool != nil { + if tx := pool.GetTxBySenderAndNonce(sender, nonce); tx != nil { + hash := tx.Hash() + return &hash, nil + } + } + + state, _, err := b.StateAndHeaderByNumberOrHash(ctx, rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber)) + if err != nil { + return nil, err + } + currentNonce := state.GetNonce(sender) + + // If the requested nonce is greater than or equal to the current nonce, + // and we already confirmed it's NOT in the pool, then the transaction + // definitely does not exist. + if nonce >= currentNonce { + return nil, nil + } + hash := rawdb.ReadTxSenderNonceEntry(b.eth.ChainDb(), sender, nonce) + return hash, nil +} + func (b *EthAPIBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) { return b.eth.miner.Pending() } diff --git a/eth/backend.go b/eth/backend.go index af8b04bda6..5c005ad2a2 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -238,6 +238,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { StateScheme: scheme, HistoryPolicy: histPolicy, TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), + TxIndexSender: config.TxIndexSender, VmConfig: vm.Config{ EnablePreimageRecording: config.EnablePreimageRecording, }, diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index b51b78e199..46d8d961f0 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -57,6 +57,7 @@ var Defaults = Config{ TxLookupLimit: 2350000, TransactionHistory: 2350000, LogHistory: 2350000, + TxIndexSender: false, StateHistory: pathdb.Defaults.StateHistory, TrienodeHistory: pathdb.Defaults.TrienodeHistory, NodeFullValueCheckpoint: pathdb.Defaults.FullValueCheckpoint, @@ -109,6 +110,7 @@ type Config struct { // Deprecated: use 'TransactionHistory' instead. TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + TxIndexSender bool `toml:",omitempty"` // Enables transaction indexing by sender and nonce (increases disk usage). TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. LogHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head where a log search index is maintained. LogNoHistory bool `toml:",omitempty"` // No log search index is maintained. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index c5e45348be..4736a4781c 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -26,6 +26,7 @@ func (c Config) MarshalTOML() (interface{}, error) { NoPruning bool NoPrefetch bool TxLookupLimit uint64 `toml:",omitempty"` + TxIndexSender bool `toml:",omitempty"` TransactionHistory uint64 `toml:",omitempty"` LogHistory uint64 `toml:",omitempty"` LogNoHistory bool `toml:",omitempty"` @@ -80,6 +81,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.NoPruning = c.NoPruning enc.NoPrefetch = c.NoPrefetch enc.TxLookupLimit = c.TxLookupLimit + enc.TxIndexSender = c.TxIndexSender enc.TransactionHistory = c.TransactionHistory enc.LogHistory = c.LogHistory enc.LogNoHistory = c.LogNoHistory @@ -138,6 +140,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { NoPruning *bool NoPrefetch *bool TxLookupLimit *uint64 `toml:",omitempty"` + TxIndexSender *bool `toml:",omitempty"` TransactionHistory *uint64 `toml:",omitempty"` LogHistory *uint64 `toml:",omitempty"` LogNoHistory *bool `toml:",omitempty"` @@ -213,6 +216,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TxLookupLimit != nil { c.TxLookupLimit = *dec.TxLookupLimit } + if dec.TxIndexSender != nil { + c.TxIndexSender = *dec.TxIndexSender + } if dec.TransactionHistory != nil { c.TransactionHistory = *dec.TransactionHistory } diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 1d8573f982..32e2a93428 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -288,6 +288,16 @@ func (ec *Client) TransactionByHash(ctx context.Context, hash common.Hash) (tx * return json.tx, json.BlockNumber == nil, nil } +// TransactionHashBySenderAndNonce returns the transaction hash for the given sender and nonce. +func (ec *Client) TransactionHashBySenderAndNonce(ctx context.Context, sender common.Address, nonce hexutil.Uint64) (*common.Hash, error) { + var hash *common.Hash + err := ec.c.CallContext(ctx, &hash, "eth_getTransactionBySenderAndNonce", sender, nonce) + if err != nil { + return nil, err + } + return hash, nil +} + // TransactionSender returns the sender address of the given transaction. The transaction // must be known to the remote node and included in the blockchain at the given block and // index. The sender is the one derived by the protocol at the time of inclusion. diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 6452fcf37c..4ba9a345f4 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1525,6 +1525,15 @@ func (api *TransactionAPI) GetTransactionByHash(ctx context.Context, hash common return newRPCTransaction(tx, blockHash, blockNumber, header.Time, index, header.BaseFee, api.b.ChainConfig()), nil } +// GetTransactionBySenderAndNonce returns the hash of a transaction for the given sender and nonce. +func (api *TransactionAPI) GetTransactionBySenderAndNonce(ctx context.Context, sender common.Address, nonce hexutil.Uint64) (*common.Hash, error) { + hash, err := api.b.GetTransactionBySenderAndNonce(ctx, sender, uint64(nonce)) + if err != nil { + return nil, err + } + return hash, nil +} + // GetRawTransactionByHash returns the bytes of the transaction for the given hash. func (api *TransactionAPI) GetRawTransactionByHash(ctx context.Context, hash common.Hash) (hexutil.Bytes, error) { // Retrieve a finalized transaction, or a pooled otherwise diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 561ce2c2d2..20d8fc3d4c 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -464,6 +464,7 @@ func fakeBlockHash(txh common.Hash) common.Hash { func newTestBackend(t *testing.T, n int, gspec *core.Genesis, engine consensus.Engine, generator func(i int, b *core.BlockGen)) *testBackend { options := core.DefaultConfig().WithArchive(true) options.TxLookupLimit = 0 // index all txs + options.TxIndexSender = true accman, acc := newTestAccountManager(t) gspec.Alloc[acc.Address] = types.Account{Balance: big.NewInt(params.Ether)} @@ -709,6 +710,14 @@ func (b testBackend) HistoryPruningCutoff() uint64 { return bn } +// GetTransactionHashBySenderAndNonce implements the Backend interface for testing. +func (b *testBackend) GetTransactionBySenderAndNonce(ctx context.Context, sender common.Address, nonce uint64) (*common.Hash, error) { + // Todo(shadow): to implement for txn pending in mempool after implementing other pending + // mempool functionality like GetPoolTransactions + hash := rawdb.ReadTxSenderNonceEntry(b.db, sender, nonce) + return hash, nil +} + func TestEstimateGas(t *testing.T) { t.Parallel() // Initialize test accounts @@ -4259,3 +4268,40 @@ func TestGetStorageValues(t *testing.T) { t.Fatal("expected error for exceeding slot limit") } } + +func TestGetTransactionBySenderAndNonce(t *testing.T) { + t.Parallel() + + key, _ := crypto.HexToECDSA("b71c73a37e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr := crypto.PubkeyToAddress(key.PublicKey) + nonce := uint64(0) + tx := types.NewTransaction(nonce, common.Address{0xaa}, big.NewInt(1000), 21000, big.NewInt(1000000000), nil) + signedTx, _ := types.SignTx(tx, types.HomesteadSigner{}, key) + + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{ + addr: {Balance: big.NewInt(1000000000000000000)}, + }, + } + + backend := newTestBackend(t, 1, genesis, ethash.NewFaker(), func(i int, b *core.BlockGen) { + if i == 0 { + b.AddTx(signedTx) + } + }) + + api := NewTransactionAPI(backend, new(AddrLocker)) + ctx := context.Background() + + result, err := api.GetTransactionBySenderAndNonce(ctx, addr, hexutil.Uint64(nonce)) + if err != nil { + t.Fatalf("GetTransactionBySenderAndNonce failed: %v", err) + } + if result == nil { + t.Fatalf("Expected transaction, got nil") + } + if *result != signedTx.Hash() { + t.Errorf("Hash mismatch: have %x, want %x", *result, signedTx.Hash()) + } +} diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index f23be85782..46835f1eda 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -87,6 +87,7 @@ type Backend interface { TxPoolContent() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) TxPoolContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription + GetTransactionBySenderAndNonce(ctx context.Context, sender common.Address, nonce uint64) (*common.Hash, error) ChainConfig() *params.ChainConfig Engine() consensus.Engine diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index ccb46a810d..21821e81e2 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -390,6 +390,11 @@ func (b *backendMock) GetCanonicalTransaction(txHash common.Hash) (bool, *types. func (b *backendMock) TxIndexDone() bool { return true } func (b *backendMock) GetPoolTransactions() (types.Transactions, error) { return nil, nil } func (b *backendMock) GetPoolTransaction(txHash common.Hash) *types.Transaction { return nil } + +// GetTransactionHashBySenderAndNonce implements the Backend interface for testing. +func (b *backendMock) GetTransactionBySenderAndNonce(ctx context.Context, sender common.Address, nonce uint64) (*common.Hash, error) { + return nil, nil +} func (b *backendMock) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { return 0, nil }