mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-07-25 06:06:44 +00:00
core: index more about txs
This commit is contained in:
parent
12d163c5bd
commit
019a6b4921
8 changed files with 260 additions and 87 deletions
|
|
@ -146,7 +146,11 @@ const (
|
||||||
// The following incompatible database changes were added:
|
// The following incompatible database changes were added:
|
||||||
// * Total difficulty has been removed from both the key-value store and the ancient store.
|
// * Total difficulty has been removed from both the key-value store and the ancient store.
|
||||||
// * The metadata structure of freezer is changed by adding 'flushOffset'
|
// * The metadata structure of freezer is changed by adding 'flushOffset'
|
||||||
BlockChainVersion uint64 = 9
|
//
|
||||||
|
// - Version 10
|
||||||
|
// The following incompatible database changes were added:
|
||||||
|
// * todo(omer)
|
||||||
|
BlockChainVersion uint64 = 10
|
||||||
)
|
)
|
||||||
|
|
||||||
// BlockChainConfig contains the configuration of the BlockChain object.
|
// BlockChainConfig contains the configuration of the BlockChain object.
|
||||||
|
|
@ -1127,7 +1131,7 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
|
||||||
if err := batch.Write(); err != nil {
|
if err := batch.Write(); err != nil {
|
||||||
log.Crit("Failed to write genesis block", "err", err)
|
log.Crit("Failed to write genesis block", "err", err)
|
||||||
}
|
}
|
||||||
bc.writeHeadBlock(genesis)
|
bc.writeHeadBlock(genesis, types.Receipts{})
|
||||||
|
|
||||||
// Last update all in-memory chain markers
|
// Last update all in-memory chain markers
|
||||||
bc.genesisBlock = genesis
|
bc.genesisBlock = genesis
|
||||||
|
|
@ -1185,13 +1189,15 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error {
|
||||||
// or if they are on a different side chain.
|
// or if they are on a different side chain.
|
||||||
//
|
//
|
||||||
// Note, this function assumes that the `mu` mutex is held!
|
// Note, this function assumes that the `mu` mutex is held!
|
||||||
func (bc *BlockChain) writeHeadBlock(block *types.Block) {
|
func (bc *BlockChain) writeHeadBlock(block *types.Block, receipts types.Receipts) {
|
||||||
// Add the block to the canonical chain number scheme and mark as the head
|
// Add the block to the canonical chain number scheme and mark as the head
|
||||||
batch := bc.db.NewBatch()
|
batch := bc.db.NewBatch()
|
||||||
rawdb.WriteHeadHeaderHash(batch, block.Hash())
|
rawdb.WriteHeadHeaderHash(batch, block.Hash())
|
||||||
rawdb.WriteHeadFastBlockHash(batch, block.Hash())
|
rawdb.WriteHeadFastBlockHash(batch, block.Hash())
|
||||||
rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64())
|
rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64())
|
||||||
rawdb.WriteTxLookupEntriesByBlock(batch, block)
|
if bc.txIndexer != nil {
|
||||||
|
bc.txIndexer.indexHead(batch, block, receipts)
|
||||||
|
}
|
||||||
rawdb.WriteHeadBlockHash(batch, block.Hash())
|
rawdb.WriteHeadBlockHash(batch, block.Hash())
|
||||||
|
|
||||||
// Flush the whole batch into the disk, exit the node if failed
|
// Flush the whole batch into the disk, exit the node if failed
|
||||||
|
|
@ -1536,7 +1542,7 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
bc.writeHeadBlock(block)
|
bc.writeHeadBlock(block, bc.GetReceiptsByHash(block.Hash()))
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1638,7 +1644,7 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types
|
||||||
}
|
}
|
||||||
|
|
||||||
// Set new head.
|
// Set new head.
|
||||||
bc.writeHeadBlock(block)
|
bc.writeHeadBlock(block, receipts)
|
||||||
|
|
||||||
bc.chainFeed.Send(ChainEvent{Header: block.Header()})
|
bc.chainFeed.Send(ChainEvent{Header: block.Header()})
|
||||||
if len(logs) > 0 {
|
if len(logs) > 0 {
|
||||||
|
|
@ -2430,7 +2436,7 @@ func (bc *BlockChain) reorg(oldHead *types.Header, newHead *types.Header) error
|
||||||
rebirthLogs = nil
|
rebirthLogs = nil
|
||||||
}
|
}
|
||||||
// Update the head block
|
// Update the head block
|
||||||
bc.writeHeadBlock(block)
|
bc.writeHeadBlock(block, bc.GetReceiptsByHash(block.Hash()))
|
||||||
}
|
}
|
||||||
if len(rebirthLogs) > 0 {
|
if len(rebirthLogs) > 0 {
|
||||||
bc.logsFeed.Send(rebirthLogs)
|
bc.logsFeed.Send(rebirthLogs)
|
||||||
|
|
@ -2505,7 +2511,7 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) {
|
||||||
return common.Hash{}, err
|
return common.Hash{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
bc.writeHeadBlock(head)
|
bc.writeHeadBlock(head, bc.GetReceiptsByHash(head.Hash()))
|
||||||
|
|
||||||
// Emit events
|
// Emit events
|
||||||
logs := bc.collectLogs(head, false)
|
logs := bc.collectLogs(head, false)
|
||||||
|
|
|
||||||
|
|
@ -34,6 +34,10 @@ import (
|
||||||
|
|
||||||
// DecodeTxLookupEntry decodes the supplied tx lookup data.
|
// DecodeTxLookupEntry decodes the supplied tx lookup data.
|
||||||
func DecodeTxLookupEntry(data []byte, db ethdb.Reader) *uint64 {
|
func DecodeTxLookupEntry(data []byte, db ethdb.Reader) *uint64 {
|
||||||
|
var index TxIndex
|
||||||
|
if err := rlp.DecodeBytes(data, &index); err == nil {
|
||||||
|
return &index.BlockNumber
|
||||||
|
}
|
||||||
// Database v6 tx lookup just stores the block number
|
// Database v6 tx lookup just stores the block number
|
||||||
if len(data) < common.HashLength {
|
if len(data) < common.HashLength {
|
||||||
number := new(big.Int).SetBytes(data).Uint64()
|
number := new(big.Int).SetBytes(data).Uint64()
|
||||||
|
|
@ -62,29 +66,53 @@ func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 {
|
||||||
return DecodeTxLookupEntry(data, db)
|
return DecodeTxLookupEntry(data, db)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ReadTxIndex retrieves the full index data if exists
|
||||||
|
func ReadTxIndex(db ethdb.Reader, hash common.Hash) *TxIndex {
|
||||||
|
data, _ := db.Get(txLookupKey(hash))
|
||||||
|
var index TxIndex
|
||||||
|
if err := rlp.DecodeBytes(data, &index); err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return &index
|
||||||
|
}
|
||||||
|
|
||||||
// writeTxLookupEntry stores a positional metadata for a transaction,
|
// writeTxLookupEntry stores a positional metadata for a transaction,
|
||||||
// enabling hash based transaction and receipt lookups.
|
// enabling hash based transaction and receipt lookups.
|
||||||
func writeTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash, numberBytes []byte) {
|
func writeTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash, bytes []byte) {
|
||||||
if err := db.Put(txLookupKey(hash), numberBytes); err != nil {
|
if err := db.Put(txLookupKey(hash), bytes); err != nil {
|
||||||
log.Crit("Failed to store transaction lookup entry", "err", err)
|
log.Crit("Failed to store transaction lookup entry", "err", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteTxLookupEntries is identical to WriteTxLookupEntry, but it works on
|
// TxIndex is the collection of data that is stored per transaction for
|
||||||
// a list of hashes
|
// speeding up hashed based lookups
|
||||||
func WriteTxLookupEntries(db ethdb.KeyValueWriter, number uint64, hashes []common.Hash) {
|
type TxIndex struct {
|
||||||
numberBytes := new(big.Int).SetUint64(number).Bytes()
|
Type uint8 // Transaction Type
|
||||||
for _, hash := range hashes {
|
Nonce uint64 // Transaction Nonce
|
||||||
writeTxLookupEntry(db, hash, numberBytes)
|
BlockNumber uint64
|
||||||
}
|
BlockHash common.Hash
|
||||||
|
BlockTime uint64
|
||||||
|
BaseFee *big.Int
|
||||||
|
TxIndex uint32
|
||||||
|
Sender common.Address
|
||||||
|
EffectiveGasPrice *big.Int
|
||||||
|
GasUsed uint64
|
||||||
|
LogIndex uint32
|
||||||
|
To *common.Address `rlp:"optional"`
|
||||||
|
BlobGas uint64 `rlp:"optional"`
|
||||||
|
BlobGasPrice *big.Int `rlp:"optional"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// WriteTxLookupEntriesByBlock stores a positional metadata for every transaction from
|
// WriteTxLookupEntries stores a positional metadata for a set of transactions
|
||||||
// a block, enabling hash based transaction and receipt lookups.
|
// enabling hash based transaction and receipt lookups.
|
||||||
func WriteTxLookupEntriesByBlock(db ethdb.KeyValueWriter, block *types.Block) {
|
func WriteTxLookupEntries(db ethdb.KeyValueWriter, hashes []common.Hash, indexes []TxIndex) {
|
||||||
numberBytes := block.Number().Bytes()
|
for index, hash := range hashes {
|
||||||
for _, tx := range block.Transactions() {
|
indexBytes, err := rlp.EncodeToBytes(indexes[index])
|
||||||
writeTxLookupEntry(db, tx.Hash(), numberBytes)
|
if err != nil {
|
||||||
|
log.Error("Failed to encode tx index", "error", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeTxLookupEntry(db, hash, indexBytes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -38,10 +38,26 @@ func TestLookupStorage(t *testing.T) {
|
||||||
name string
|
name string
|
||||||
writeTxLookupEntriesByBlock func(ethdb.KeyValueWriter, *types.Block)
|
writeTxLookupEntriesByBlock func(ethdb.KeyValueWriter, *types.Block)
|
||||||
}{
|
}{
|
||||||
|
{
|
||||||
|
"DatabaseV10",
|
||||||
|
func(db ethdb.KeyValueWriter, block *types.Block) {
|
||||||
|
var (
|
||||||
|
txHashes []common.Hash
|
||||||
|
indexes []TxIndex
|
||||||
|
)
|
||||||
|
for txIndex, tx := range block.Transactions() {
|
||||||
|
txHashes = append(txHashes, tx.Hash())
|
||||||
|
indexes = append(indexes, TxIndex{BlockNumber: block.NumberU64(), TxIndex: uint32(txIndex)})
|
||||||
|
}
|
||||||
|
WriteTxLookupEntries(db, txHashes, indexes)
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"DatabaseV6",
|
"DatabaseV6",
|
||||||
func(db ethdb.KeyValueWriter, block *types.Block) {
|
func(db ethdb.KeyValueWriter, block *types.Block) {
|
||||||
WriteTxLookupEntriesByBlock(db, block)
|
for _, tx := range block.Transactions() {
|
||||||
|
db.Put(txLookupKey(tx.Hash()), block.Number().Bytes())
|
||||||
|
}
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,6 @@
|
||||||
package rawdb
|
package rawdb
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/binary"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
|
|
@ -94,12 +93,8 @@ func PruneTransactionIndex(db ethdb.Database, pruneBlock uint64) {
|
||||||
if count%10000000 == 0 {
|
if count%10000000 == 0 {
|
||||||
log.Info("Pruning tx index", "count", count, "removed", removed)
|
log.Info("Pruning tx index", "count", count, "removed", removed)
|
||||||
}
|
}
|
||||||
if len(v) > 8 {
|
bn := DecodeTxLookupEntry(v, db)
|
||||||
log.Error("Skipping legacy tx index entry", "hash", txhash)
|
if *bn < pruneBlock {
|
||||||
return false
|
|
||||||
}
|
|
||||||
bn := decodeNumber(v)
|
|
||||||
if bn < pruneBlock {
|
|
||||||
removed++
|
removed++
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
|
@ -107,9 +102,3 @@ func PruneTransactionIndex(db ethdb.Database, pruneBlock uint64) {
|
||||||
})
|
})
|
||||||
WriteTxIndexTail(db, pruneBlock)
|
WriteTxIndexTail(db, pruneBlock)
|
||||||
}
|
}
|
||||||
|
|
||||||
func decodeNumber(b []byte) uint64 {
|
|
||||||
var numBuffer [8]byte
|
|
||||||
copy(numBuffer[8-len(b):], b)
|
|
||||||
return binary.BigEndian.Uint64(numBuffer[:])
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -76,7 +76,15 @@ func TestPruneTransactionIndex(t *testing.T) {
|
||||||
pruneBlock := lastBlock - 3
|
pruneBlock := lastBlock - 3
|
||||||
|
|
||||||
for i := uint64(0); i <= lastBlock; i++ {
|
for i := uint64(0); i <= lastBlock; i++ {
|
||||||
WriteTxLookupEntriesByBlock(chainDB, blocks[i])
|
var (
|
||||||
|
txHashes []common.Hash
|
||||||
|
indexes []TxIndex
|
||||||
|
)
|
||||||
|
for txIndex, tx := range blocks[i].Transactions() {
|
||||||
|
txHashes = append(txHashes, tx.Hash())
|
||||||
|
indexes = append(indexes, TxIndex{BlockNumber: blocks[i].NumberU64(), TxIndex: uint32(txIndex)})
|
||||||
|
}
|
||||||
|
WriteTxLookupEntries(chainDB, txHashes, indexes)
|
||||||
}
|
}
|
||||||
WriteTxIndexTail(chainDB, 0)
|
WriteTxIndexTail(chainDB, 0)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -18,16 +18,19 @@ package core
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"math/big"
|
||||||
"runtime"
|
"runtime"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
"github.com/ethereum/go-ethereum/common/prque"
|
"github.com/ethereum/go-ethereum/common/prque"
|
||||||
|
"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
|
||||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||||
"github.com/ethereum/go-ethereum/core/types"
|
"github.com/ethereum/go-ethereum/core/types"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/log"
|
"github.com/ethereum/go-ethereum/log"
|
||||||
|
"github.com/ethereum/go-ethereum/params"
|
||||||
"github.com/ethereum/go-ethereum/rlp"
|
"github.com/ethereum/go-ethereum/rlp"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -67,6 +70,7 @@ type txIndexer struct {
|
||||||
// be pruned and not available locally.
|
// be pruned and not available locally.
|
||||||
cutoff uint64
|
cutoff uint64
|
||||||
db ethdb.Database
|
db ethdb.Database
|
||||||
|
config *params.ChainConfig
|
||||||
term chan chan struct{}
|
term chan chan struct{}
|
||||||
closed chan struct{}
|
closed chan struct{}
|
||||||
}
|
}
|
||||||
|
|
@ -78,6 +82,7 @@ func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer {
|
||||||
limit: limit,
|
limit: limit,
|
||||||
cutoff: cutoff,
|
cutoff: cutoff,
|
||||||
db: chain.db,
|
db: chain.db,
|
||||||
|
config: chain.chainConfig,
|
||||||
term: make(chan chan struct{}),
|
term: make(chan chan struct{}),
|
||||||
closed: make(chan struct{}),
|
closed: make(chan struct{}),
|
||||||
}
|
}
|
||||||
|
|
@ -337,20 +342,62 @@ func (indexer *txIndexer) close() {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type blockTxHashes struct {
|
// generateTxIndex generates the data that will be stored alongside the transaction index
|
||||||
|
func (indexer *txIndexer) generateTxIndex(txs types.Transactions, header *types.Header, receipts types.Receipts) []rawdb.TxIndex {
|
||||||
|
var (
|
||||||
|
logIndex = 0
|
||||||
|
prevCumulativeGasUsed = uint64(0)
|
||||||
|
signer = types.MakeSigner(indexer.config, header.Number, header.Time)
|
||||||
|
indexes = make([]rawdb.TxIndex, len(txs))
|
||||||
|
)
|
||||||
|
|
||||||
|
var blobGasPrice *big.Int
|
||||||
|
if header.ExcessBlobGas != nil {
|
||||||
|
blobGasPrice = eip4844.CalcBlobFee(indexer.config, header)
|
||||||
|
}
|
||||||
|
|
||||||
|
for blockIndex, tx := range txs {
|
||||||
|
from, _ := signer.Sender(tx)
|
||||||
|
indexes[blockIndex] = rawdb.TxIndex{
|
||||||
|
Type: tx.Type(),
|
||||||
|
Nonce: tx.Nonce(),
|
||||||
|
To: tx.To(),
|
||||||
|
|
||||||
|
BlockNumber: header.Number.Uint64(),
|
||||||
|
BlockHash: header.Hash(),
|
||||||
|
BlockTime: header.Time,
|
||||||
|
BaseFee: header.BaseFee,
|
||||||
|
TxIndex: uint32(blockIndex),
|
||||||
|
Sender: from,
|
||||||
|
EffectiveGasPrice: tx.EffectiveGasPrice(header.BaseFee),
|
||||||
|
GasUsed: receipts[blockIndex].CumulativeGasUsed - prevCumulativeGasUsed,
|
||||||
|
LogIndex: uint32(logIndex),
|
||||||
|
BlobGas: tx.BlobGas(),
|
||||||
|
BlobGasPrice: blobGasPrice,
|
||||||
|
}
|
||||||
|
prevCumulativeGasUsed = receipts[blockIndex].CumulativeGasUsed
|
||||||
|
logIndex += len(receipts[blockIndex].Logs)
|
||||||
|
}
|
||||||
|
return indexes
|
||||||
|
}
|
||||||
|
|
||||||
|
type blockIndexingContext struct {
|
||||||
number uint64
|
number uint64
|
||||||
hashes []common.Hash
|
txHashes []common.Hash
|
||||||
|
indexes []rawdb.TxIndex
|
||||||
}
|
}
|
||||||
|
|
||||||
// iterate iterates over all transactions in the (canon) block
|
// iterate iterates over all transactions in the (canon) block
|
||||||
// number(s) given, and yields the hashes on a channel. If there is a signal
|
// number(s) given, and yields the hashes on a channel. If there is a signal
|
||||||
// received from interrupt channel, the iteration will be aborted and result
|
// received from interrupt channel, the iteration will be aborted and result
|
||||||
// channel will be closed.
|
// channel will be closed.
|
||||||
func (indexer *txIndexer) iterate(from uint64, to uint64, reverse bool, interrupt chan struct{}) chan *blockTxHashes {
|
func (indexer *txIndexer) iterate(from uint64, to uint64, reverse, indexing bool, interrupt chan struct{}) chan *blockIndexingContext {
|
||||||
// One thread sequentially reads data from db
|
// One thread sequentially reads data from db
|
||||||
type numberRlp struct {
|
type numberRlp struct {
|
||||||
number uint64
|
number uint64
|
||||||
rlp rlp.RawValue
|
headerRLP rlp.RawValue
|
||||||
|
bodyRLP rlp.RawValue
|
||||||
|
receiptsRLP rlp.RawValue
|
||||||
}
|
}
|
||||||
if to == from {
|
if to == from {
|
||||||
return nil
|
return nil
|
||||||
|
|
@ -361,7 +408,7 @@ func (indexer *txIndexer) iterate(from uint64, to uint64, reverse bool, interrup
|
||||||
}
|
}
|
||||||
var (
|
var (
|
||||||
rlpCh = make(chan *numberRlp, threads*2) // we send raw rlp over this channel
|
rlpCh = make(chan *numberRlp, threads*2) // we send raw rlp over this channel
|
||||||
hashesCh = make(chan *blockTxHashes, threads*2) // send hashes over hashesCh
|
contextsCh = make(chan *blockIndexingContext, threads*2) // send indexing context over contextsCh
|
||||||
)
|
)
|
||||||
// lookup runs in one instance
|
// lookup runs in one instance
|
||||||
lookup := func() {
|
lookup := func() {
|
||||||
|
|
@ -371,10 +418,20 @@ func (indexer *txIndexer) iterate(from uint64, to uint64, reverse bool, interrup
|
||||||
}
|
}
|
||||||
defer close(rlpCh)
|
defer close(rlpCh)
|
||||||
for n != end {
|
for n != end {
|
||||||
data := rawdb.ReadCanonicalBodyRLP(indexer.db, n)
|
blockHash := rawdb.ReadCanonicalHash(indexer.db, n)
|
||||||
|
rlps := &numberRlp{
|
||||||
|
number: n,
|
||||||
|
bodyRLP: rawdb.ReadBodyRLP(indexer.db, blockHash, n),
|
||||||
|
}
|
||||||
|
|
||||||
|
if indexing {
|
||||||
|
// We are indexing, read more to build the index
|
||||||
|
rlps.headerRLP = rawdb.ReadHeaderRLP(indexer.db, blockHash, n)
|
||||||
|
rlps.receiptsRLP = rawdb.ReadReceiptsRLP(indexer.db, blockHash, n)
|
||||||
|
}
|
||||||
// Feed the block to the aggregator, or abort on interrupt
|
// Feed the block to the aggregator, or abort on interrupt
|
||||||
select {
|
select {
|
||||||
case rlpCh <- &numberRlp{n, data}:
|
case rlpCh <- rlps:
|
||||||
case <-interrupt:
|
case <-interrupt:
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
@ -392,26 +449,47 @@ func (indexer *txIndexer) iterate(from uint64, to uint64, reverse bool, interrup
|
||||||
defer func() {
|
defer func() {
|
||||||
// Last processor closes the result channel
|
// Last processor closes the result channel
|
||||||
if nThreadsAlive.Add(-1) == 0 {
|
if nThreadsAlive.Add(-1) == 0 {
|
||||||
close(hashesCh)
|
close(contextsCh)
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
for data := range rlpCh {
|
for data := range rlpCh {
|
||||||
var body types.Body
|
var body types.Body
|
||||||
if err := rlp.DecodeBytes(data.rlp, &body); err != nil {
|
if err := rlp.DecodeBytes(data.bodyRLP, &body); err != nil {
|
||||||
log.Warn("Failed to decode block body", "block", data.number, "error", err)
|
log.Error("Failed to decode block body", "block", data.number, "error", err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var hashes []common.Hash
|
txHashes := make([]common.Hash, 0, len(body.Transactions))
|
||||||
for _, tx := range body.Transactions {
|
for _, tx := range body.Transactions {
|
||||||
hashes = append(hashes, tx.Hash())
|
txHashes = append(txHashes, tx.Hash())
|
||||||
}
|
}
|
||||||
result := &blockTxHashes{
|
|
||||||
hashes: hashes,
|
result := &blockIndexingContext{
|
||||||
number: data.number,
|
number: data.number,
|
||||||
|
txHashes: txHashes,
|
||||||
|
}
|
||||||
|
|
||||||
|
if indexing {
|
||||||
|
var header types.Header
|
||||||
|
if err := rlp.DecodeBytes(data.headerRLP, &header); err != nil {
|
||||||
|
log.Error("failed to decode header", "block", data.number, "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var storageReceipts []*types.ReceiptForStorage
|
||||||
|
if err := rlp.DecodeBytes(data.receiptsRLP, &storageReceipts); err != nil {
|
||||||
|
log.Error("failed to decode receipts", "block", data.number, "err", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
receipts := make([]*types.Receipt, len(storageReceipts))
|
||||||
|
for i, receipt := range storageReceipts {
|
||||||
|
receipts[i] = (*types.Receipt)(receipt)
|
||||||
|
}
|
||||||
|
result.indexes = indexer.generateTxIndex(body.Transactions, &header, receipts)
|
||||||
}
|
}
|
||||||
// Feed the block to the aggregator, or abort on interrupt
|
// Feed the block to the aggregator, or abort on interrupt
|
||||||
select {
|
select {
|
||||||
case hashesCh <- result:
|
case contextsCh <- result:
|
||||||
case <-interrupt:
|
case <-interrupt:
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
@ -421,7 +499,7 @@ func (indexer *txIndexer) iterate(from uint64, to uint64, reverse bool, interrup
|
||||||
for i := 0; i < int(threads); i++ {
|
for i := 0; i < int(threads); i++ {
|
||||||
go process()
|
go process()
|
||||||
}
|
}
|
||||||
return hashesCh
|
return contextsCh
|
||||||
}
|
}
|
||||||
|
|
||||||
// index creates txlookup indices of the specified block range.
|
// index creates txlookup indices of the specified block range.
|
||||||
|
|
@ -438,7 +516,7 @@ func (indexer *txIndexer) index(from uint64, to uint64, interrupt chan struct{},
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var (
|
var (
|
||||||
hashesCh = indexer.iterate(from, to, true, interrupt)
|
contextsCh = indexer.iterate(from, to, true, true, interrupt)
|
||||||
batch = indexer.db.NewBatch()
|
batch = indexer.db.NewBatch()
|
||||||
start = time.Now()
|
start = time.Now()
|
||||||
logged = start.Add(-7 * time.Second)
|
logged = start.Add(-7 * time.Second)
|
||||||
|
|
@ -447,10 +525,10 @@ func (indexer *txIndexer) index(from uint64, to uint64, interrupt chan struct{},
|
||||||
// in to be [to-1]. Therefore, setting lastNum to means that the
|
// in to be [to-1]. Therefore, setting lastNum to means that the
|
||||||
// queue gap-evaluation will work correctly
|
// queue gap-evaluation will work correctly
|
||||||
lastNum = to
|
lastNum = to
|
||||||
queue = prque.New[int64, *blockTxHashes](nil)
|
queue = prque.New[int64, *blockIndexingContext](nil)
|
||||||
blocks, txs = 0, 0 // for stats reporting
|
blocks, txs = 0, 0 // for stats reporting
|
||||||
)
|
)
|
||||||
for chanDelivery := range hashesCh {
|
for chanDelivery := range contextsCh {
|
||||||
// Push the delivery into the queue and process contiguous ranges.
|
// Push the delivery into the queue and process contiguous ranges.
|
||||||
// Since we iterate in reverse, so lower numbers have lower prio, and
|
// Since we iterate in reverse, so lower numbers have lower prio, and
|
||||||
// we can use the number directly as prio marker
|
// we can use the number directly as prio marker
|
||||||
|
|
@ -467,9 +545,9 @@ func (indexer *txIndexer) index(from uint64, to uint64, interrupt chan struct{},
|
||||||
// Next block available, pop it off and index it
|
// Next block available, pop it off and index it
|
||||||
delivery := queue.PopItem()
|
delivery := queue.PopItem()
|
||||||
lastNum = delivery.number
|
lastNum = delivery.number
|
||||||
rawdb.WriteTxLookupEntries(batch, delivery.number, delivery.hashes)
|
rawdb.WriteTxLookupEntries(batch, delivery.txHashes, delivery.indexes)
|
||||||
blocks++
|
blocks++
|
||||||
txs += len(delivery.hashes)
|
txs += len(delivery.txHashes)
|
||||||
// If enough data was accumulated in memory or we're at the last block, dump to disk
|
// If enough data was accumulated in memory or we're at the last block, dump to disk
|
||||||
if batch.ValueSize() > ethdb.IdealBatchSize {
|
if batch.ValueSize() > ethdb.IdealBatchSize {
|
||||||
rawdb.WriteTxIndexTail(batch, lastNum) // Also write the tail here
|
rawdb.WriteTxIndexTail(batch, lastNum) // Also write the tail here
|
||||||
|
|
@ -516,7 +594,7 @@ func (indexer *txIndexer) unindex(from uint64, to uint64, interrupt chan struct{
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var (
|
var (
|
||||||
hashesCh = indexer.iterate(from, to, false, interrupt)
|
contextsCh = indexer.iterate(from, to, false, false, interrupt)
|
||||||
batch = indexer.db.NewBatch()
|
batch = indexer.db.NewBatch()
|
||||||
start = time.Now()
|
start = time.Now()
|
||||||
logged = start.Add(-7 * time.Second)
|
logged = start.Add(-7 * time.Second)
|
||||||
|
|
@ -524,11 +602,11 @@ func (indexer *txIndexer) unindex(from uint64, to uint64, interrupt chan struct{
|
||||||
// we expect the first number to come in to be [from]. Therefore, setting
|
// we expect the first number to come in to be [from]. Therefore, setting
|
||||||
// nextNum to from means that the queue gap-evaluation will work correctly
|
// nextNum to from means that the queue gap-evaluation will work correctly
|
||||||
nextNum = from
|
nextNum = from
|
||||||
queue = prque.New[int64, *blockTxHashes](nil)
|
queue = prque.New[int64, *blockIndexingContext](nil)
|
||||||
blocks, txs = 0, 0 // for stats reporting
|
blocks, txs = 0, 0 // for stats reporting
|
||||||
)
|
)
|
||||||
// Otherwise spin up the concurrent iterator and unindexer
|
// Otherwise spin up the concurrent iterator and unindexer
|
||||||
for delivery := range hashesCh {
|
for delivery := range contextsCh {
|
||||||
// Push the delivery into the queue and process contiguous ranges.
|
// Push the delivery into the queue and process contiguous ranges.
|
||||||
queue.Push(delivery, -int64(delivery.number))
|
queue.Push(delivery, -int64(delivery.number))
|
||||||
for !queue.Empty() {
|
for !queue.Empty() {
|
||||||
|
|
@ -542,8 +620,8 @@ func (indexer *txIndexer) unindex(from uint64, to uint64, interrupt chan struct{
|
||||||
}
|
}
|
||||||
delivery := queue.PopItem()
|
delivery := queue.PopItem()
|
||||||
nextNum = delivery.number + 1
|
nextNum = delivery.number + 1
|
||||||
rawdb.DeleteTxLookupEntries(batch, delivery.hashes)
|
rawdb.DeleteTxLookupEntries(batch, delivery.txHashes)
|
||||||
txs += len(delivery.hashes)
|
txs += len(delivery.txHashes)
|
||||||
blocks++
|
blocks++
|
||||||
|
|
||||||
// If enough data was accumulated in memory or we're at the last block, dump to disk
|
// If enough data was accumulated in memory or we're at the last block, dump to disk
|
||||||
|
|
@ -583,3 +661,13 @@ func (indexer *txIndexer) unindex(from uint64, to uint64, interrupt chan struct{
|
||||||
logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
|
logger("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// indexHead indexes given head block synchronously
|
||||||
|
func (indexer *txIndexer) indexHead(db ethdb.KeyValueWriter, block *types.Block, receipts types.Receipts) {
|
||||||
|
indexes := indexer.generateTxIndex(block.Transactions(), block.Header(), receipts)
|
||||||
|
hashes := make([]common.Hash, len(block.Transactions()))
|
||||||
|
for i, tx := range block.Transactions() {
|
||||||
|
hashes[i] = tx.Hash()
|
||||||
|
}
|
||||||
|
rawdb.WriteTxLookupEntries(db, hashes, indexes)
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -27,10 +27,14 @@ import (
|
||||||
"github.com/ethereum/go-ethereum/crypto"
|
"github.com/ethereum/go-ethereum/crypto"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/params"
|
"github.com/ethereum/go-ethereum/params"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
)
|
)
|
||||||
|
|
||||||
func verifyIndexes(t *testing.T, db ethdb.Database, block *types.Block, exist bool) {
|
func verifyIndexes(t *testing.T, db ethdb.Database, block *types.Block, receipts types.Receipts, exist bool) {
|
||||||
for _, tx := range block.Transactions() {
|
signer := types.MakeSigner(params.TestChainConfig, block.Number(), block.Time())
|
||||||
|
|
||||||
|
for blockIndex, tx := range block.Transactions() {
|
||||||
lookup := rawdb.ReadTxLookupEntry(db, tx.Hash())
|
lookup := rawdb.ReadTxLookupEntry(db, tx.Hash())
|
||||||
if exist && lookup == nil {
|
if exist && lookup == nil {
|
||||||
t.Fatalf("missing %d %x", block.NumberU64(), tx.Hash().Hex())
|
t.Fatalf("missing %d %x", block.NumberU64(), tx.Hash().Hex())
|
||||||
|
|
@ -38,10 +42,35 @@ func verifyIndexes(t *testing.T, db ethdb.Database, block *types.Block, exist bo
|
||||||
if !exist && lookup != nil {
|
if !exist && lookup != nil {
|
||||||
t.Fatalf("unexpected %d %x", block.NumberU64(), tx.Hash().Hex())
|
t.Fatalf("unexpected %d %x", block.NumberU64(), tx.Hash().Hex())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if exist {
|
||||||
|
txIndex := rawdb.ReadTxIndex(db, tx.Hash())
|
||||||
|
require.NotNil(t, txIndex)
|
||||||
|
assert.Equal(t, tx.Type(), txIndex.Type)
|
||||||
|
assert.Equal(t, tx.Nonce(), txIndex.Nonce)
|
||||||
|
assert.Equal(t, block.NumberU64(), txIndex.BlockNumber)
|
||||||
|
assert.Equal(t, block.Hash(), txIndex.BlockHash)
|
||||||
|
assert.Equal(t, block.Time(), txIndex.BlockTime)
|
||||||
|
assert.Equal(t, block.BaseFee(), txIndex.BaseFee)
|
||||||
|
assert.Equal(t, uint32(blockIndex), txIndex.TxIndex)
|
||||||
|
|
||||||
|
sender, _ := signer.Sender(tx)
|
||||||
|
assert.Equal(t, txIndex.Sender, sender)
|
||||||
|
|
||||||
|
receipt := receipts[blockIndex]
|
||||||
|
assert.Equal(t, receipt.EffectiveGasPrice, txIndex.EffectiveGasPrice)
|
||||||
|
assert.Equal(t, receipt.GasUsed, txIndex.GasUsed)
|
||||||
|
if len(receipt.Logs) > 0 {
|
||||||
|
assert.Equal(t, receipt.Logs[0].Index, txIndex.LogIndex)
|
||||||
|
}
|
||||||
|
assert.Equal(t, tx.To(), txIndex.To)
|
||||||
|
assert.Equal(t, tx.BlobGas(), txIndex.BlobGas)
|
||||||
|
assert.Equal(t, receipt.BlobGasPrice, txIndex.BlobGasPrice)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func verify(t *testing.T, db ethdb.Database, blocks []*types.Block, expTail uint64) {
|
func verify(t *testing.T, db ethdb.Database, blocks []*types.Block, receipts []types.Receipts, expTail uint64) {
|
||||||
tail := rawdb.ReadTxIndexTail(db)
|
tail := rawdb.ReadTxIndexTail(db)
|
||||||
if tail == nil {
|
if tail == nil {
|
||||||
t.Fatal("Failed to write tx index tail")
|
t.Fatal("Failed to write tx index tail")
|
||||||
|
|
@ -50,11 +79,11 @@ func verify(t *testing.T, db ethdb.Database, blocks []*types.Block, expTail uint
|
||||||
if *tail != expTail {
|
if *tail != expTail {
|
||||||
t.Fatalf("Unexpected tx index tail, want %v, got %d", expTail, *tail)
|
t.Fatalf("Unexpected tx index tail, want %v, got %d", expTail, *tail)
|
||||||
}
|
}
|
||||||
for _, b := range blocks {
|
for i, b := range blocks {
|
||||||
if b.Number().Uint64() < *tail {
|
if b.Number().Uint64() < *tail {
|
||||||
verifyIndexes(t, db, b, false)
|
verifyIndexes(t, db, b, receipts[i], false)
|
||||||
} else {
|
} else {
|
||||||
verifyIndexes(t, db, b, true)
|
verifyIndexes(t, db, b, receipts[i], true)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -65,7 +94,7 @@ func verifyNoIndex(t *testing.T, db ethdb.Database, blocks []*types.Block) {
|
||||||
t.Fatalf("Unexpected tx index tail %d", *tail)
|
t.Fatalf("Unexpected tx index tail %d", *tail)
|
||||||
}
|
}
|
||||||
for _, b := range blocks {
|
for _, b := range blocks {
|
||||||
verifyIndexes(t, db, b, false)
|
verifyIndexes(t, db, b, nil, false)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -123,11 +152,12 @@ func TestTxIndexer(t *testing.T) {
|
||||||
indexer := &txIndexer{
|
indexer := &txIndexer{
|
||||||
limit: 0,
|
limit: 0,
|
||||||
db: db,
|
db: db,
|
||||||
|
config: gspec.Config,
|
||||||
}
|
}
|
||||||
for i, limit := range c.limits {
|
for i, limit := range c.limits {
|
||||||
indexer.limit = limit
|
indexer.limit = limit
|
||||||
indexer.run(chainHead, make(chan struct{}), make(chan struct{}))
|
indexer.run(chainHead, make(chan struct{}), make(chan struct{}))
|
||||||
verify(t, db, blocks, c.tails[i])
|
verify(t, db, blocks, receipts, c.tails[i])
|
||||||
}
|
}
|
||||||
db.Close()
|
db.Close()
|
||||||
}
|
}
|
||||||
|
|
@ -243,6 +273,7 @@ func TestTxIndexerRepair(t *testing.T) {
|
||||||
indexer := &txIndexer{
|
indexer := &txIndexer{
|
||||||
limit: c.limit,
|
limit: c.limit,
|
||||||
db: db,
|
db: db,
|
||||||
|
config: gspec.Config,
|
||||||
}
|
}
|
||||||
indexer.run(chainHead, make(chan struct{}), make(chan struct{}))
|
indexer.run(chainHead, make(chan struct{}), make(chan struct{}))
|
||||||
|
|
||||||
|
|
@ -252,7 +283,7 @@ func TestTxIndexerRepair(t *testing.T) {
|
||||||
if c.expTail == nil {
|
if c.expTail == nil {
|
||||||
verifyNoIndex(t, db, blocks)
|
verifyNoIndex(t, db, blocks)
|
||||||
} else {
|
} else {
|
||||||
verify(t, db, blocks, *c.expTail)
|
verify(t, db, blocks, receipts, *c.expTail)
|
||||||
}
|
}
|
||||||
db.Close()
|
db.Close()
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -404,6 +404,13 @@ func (tx *Transaction) EffectiveGasTipIntCmp(other *big.Int, baseFee *big.Int) i
|
||||||
return txTip.Cmp(other)
|
return txTip.Cmp(other)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// EffectiveGasPrice computes the gas price paid by the transaction, given
|
||||||
|
// the inclusion block baseFee
|
||||||
|
func (tx *Transaction) EffectiveGasPrice(baseFee *big.Int) *big.Int {
|
||||||
|
var dst big.Int
|
||||||
|
return tx.inner.effectiveGasPrice(&dst, baseFee)
|
||||||
|
}
|
||||||
|
|
||||||
// BlobGas returns the blob gas limit of the transaction for blob transactions, 0 otherwise.
|
// BlobGas returns the blob gas limit of the transaction for blob transactions, 0 otherwise.
|
||||||
func (tx *Transaction) BlobGas() uint64 {
|
func (tx *Transaction) BlobGas() uint64 {
|
||||||
if blobtx, ok := tx.inner.(*BlobTx); ok {
|
if blobtx, ok := tx.inner.(*BlobTx); ok {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue