forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-18 10:50:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

core/txpool/blobpool, eth: implement sparse blobpool (#34047)

Browse source 
This is the implementation of EIP-8070 Sparse Blobpool. It introduces
protocol version eth/72 which relays blob transaction cells instead of
full blobs.

The blobpool now store 'incomplete' transactions, where only some of
the cells are provided. The stored cell indexes are taken from the
custody bitmap, which is provided by the consensus layer in
forkchoiceUpdatedV4. This method will be called once Glamsterdam
activates, and the default custody is full custody, so for now there
is no change in the amount of stored cells for now.

The main entities added are the BlobBuffer and BlobFetcher, which work
together to track and fetch missing cells from connected peers. The
partial transactions become available for inclusion in blocks when
they are covered by enough peers that hold all cells.

This change also introduces engine_getBlobsV4, which allows for
cell-based responses (and partial blobs with only some of the cells).
We maintain backward compatibility with getBlobsV3 which expects full
blob responses by recovering the blob from available cells. Since this
process is resource-intensive, we proactively cache the conversion so
it is ready in time for getBlobsV3 calls. This mechanism will be
removed once support for getBlobsV4 is universal across all consensus
layer implementations.

devp2p tests for eth/72 are not part of this initial change. This is
to avoid breaking test success status for execution clients that do
not have eth/72 implemented yet. The tests will be added in a
subsequent change.

---------

Co-authored-by: Felix Lange <fjl@twurst.com>
This commit is contained in:
Bosul Mun 2026-07-15 14:41:04 +02:00 committed by GitHub
parent c3f2851872
commit d91b71fb36
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
47 changed files with 5130 additions and 569 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
beacon/engine/types.go
View file

@ -173,6 +173,11 @@ type BlobAndProofV2 struct {
CellProofs []hexutil.Bytes `json:"proofs"` // proofs MUST contain exactly CELLS_PER_EXT_BLOB cell proofs.
}
type BlobCellsAndProofsV1 struct {
BlobCells []*hexutil.Bytes `json:"blob_cells"`
Proofs []*hexutil.Bytes `json:"proofs"`
}
// BlobAndProofListV2 is a list of BlobAndProofV2 with a hand-rolled JSON marshaler
// that avoids the overhead of encoding/json for large blob payloads.
type BlobAndProofListV2 []*BlobAndProofV2

2
cmd/devp2p/internal/ethtest/conn.go
View file

@ -181,7 +181,7 @@ func (c *Conn) ReadEth() (any, error) {
case eth.TransactionsMsg:
msg = new(eth.TransactionsPacket)
case eth.NewPooledTransactionHashesMsg:
msg = new(eth.NewPooledTransactionHashesPacket)
msg = new(eth.NewPooledTransactionHashesPacket71)
case eth.GetPooledTransactionsMsg:
msg = new(eth.GetPooledTransactionsPacket)
case eth.PooledTransactionsMsg:

14
cmd/devp2p/internal/ethtest/suite.go
View file

@ -976,7 +976,7 @@ the transactions using a GetPooledTransactions request.`)
}
// Send announcement.
ann := eth.NewPooledTransactionHashesPacket{Types: txTypes, Sizes: sizes, Hashes: hashes}
ann := eth.NewPooledTransactionHashesPacket71{Types: txTypes, Sizes: sizes, Hashes: hashes}
err = conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann)
if err != nil {
t.Fatalf("failed to write to connection: %v", err)
@ -994,7 +994,7 @@ the transactions using a GetPooledTransactions request.`)
t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsRequest))
}
return
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket71:
continue
case *eth.TransactionsPacket:
continue
@ -1060,12 +1060,12 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
t2 = s.makeBlobTxs(2, 3, 0x2)
)
for _, test := range []struct {
ann eth.NewPooledTransactionHashesPacket
ann eth.NewPooledTransactionHashesPacket71
resp eth.PooledTransactionsResponse
}{
// Invalid tx size.
{
ann: eth.NewPooledTransactionHashesPacket{
ann: eth.NewPooledTransactionHashesPacket71{
Types: []byte{types.BlobTxType, types.BlobTxType},
Sizes: []uint32{uint32(t1[0].Size()), uint32(t1[1].Size() + 10)},
Hashes: []common.Hash{t1[0].Hash(), t1[1].Hash()},
@ -1074,7 +1074,7 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
},
// Wrong tx type.
{
ann: eth.NewPooledTransactionHashesPacket{
ann: eth.NewPooledTransactionHashesPacket71{
Types: []byte{types.DynamicFeeTxType, types.BlobTxType},
Sizes: []uint32{uint32(t2[0].Size()), uint32(t2[1].Size())},
Hashes: []common.Hash{t2[0].Hash(), t2[1].Hash()},
@ -1203,7 +1203,7 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types
return
}
ann := eth.NewPooledTransactionHashesPacket{
ann := eth.NewPooledTransactionHashesPacket71{
Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(badTx.Size())},
Hashes: []common.Hash{badTx.Hash()},
@ -1254,7 +1254,7 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types
return
}
ann := eth.NewPooledTransactionHashesPacket{
ann := eth.NewPooledTransactionHashesPacket71{
Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(tx.Size())},
Hashes: []common.Hash{tx.Hash()},

4
cmd/devp2p/internal/ethtest/transaction.go
View file

@ -74,7 +74,7 @@ func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error {
for _, tx := range txs {
got[tx.Hash()] = true
}
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket71:
for _, hash := range msg.Hashes {
got[hash] = true
}
@ -164,7 +164,7 @@ func (s *Suite) sendInvalidTxs(t *utesting.T, txs []*types.Transaction) error {
return fmt.Errorf("received bad tx: %s", tx.Hash())
}
}
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket71:
for _, hash := range msg.Hashes {
if _, ok := invalids[hash]; ok {
return fmt.Errorf("received bad tx: %s", hash)

1
cmd/geth/main.go
View file

@ -76,6 +76,7 @@ var (
utils.BlobPoolDataDirFlag,
utils.BlobPoolDataCapFlag,
utils.BlobPoolPriceBumpFlag,
utils.BlobPoolFetchProbabilityFlag,
utils.SyncModeFlag,
utils.SyncTargetFlag,
utils.ExitWhenSyncedFlag,

10
cmd/utils/flags.go
View file

@ -48,6 +48,7 @@ import (
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/fetcher"
"github.com/ethereum/go-ethereum/eth/filters"
"github.com/ethereum/go-ethereum/eth/gasprice"
"github.com/ethereum/go-ethereum/eth/syncer"
@ -502,6 +503,12 @@ var (
Value: ethconfig.Defaults.BlobPool.PriceBump,
Category: flags.BlobPoolCategory,
}
BlobPoolFetchProbabilityFlag = &cli.Uint64Flag{
Name: "blobpool.fetchprobability",
Usage: "Probability of fetching the full blob payload for sparse blobpool (min=15, max=100)",
Value: fetcher.DefaultFetchProbability,
Category: flags.BlobPoolCategory,
}
// Performance tuning settings
CacheFlag = &cli.IntFlag{
Name: "cache",
@ -1695,6 +1702,9 @@ func setBlobPool(ctx *cli.Context, cfg *blobpool.Config) {
if ctx.IsSet(BlobPoolPriceBumpFlag.Name) {
cfg.PriceBump = ctx.Uint64(BlobPoolPriceBumpFlag.Name)
}
if ctx.IsSet(BlobPoolFetchProbabilityFlag.Name) {
cfg.FetchProbability = ctx.Uint64(BlobPoolFetchProbabilityFlag.Name)
}
}
func setMiner(ctx *cli.Context, cfg *miner.Config) {

566
core/txpool/blobpool/blobpool.go
View file

@ -129,9 +129,12 @@ type blobTxMeta struct {
announced bool // Whether the tx has been announced to listeners
id uint64 // Storage ID in the pool's persistent store
storageSize uint32 // Byte size in the pool's persistent store
size uint64 // RLP-encoded size of transaction including the attached blob
id uint64 // Storage ID in the pool's persistent store
storageSize uint32 // Byte size in the pool's persistent store
size uint64 // RLP-encoded size of transaction including the attached blob
sizeWithoutBlob uint64 // RLP-encoded size of transaction without blob data (for ETH/72)
custody *types.CustodyBitmap
nonce uint64 // Needed to prioritize inclusion order within an account
costCap *uint256.Int // Needed to validate cumulative balance sufficiency
@ -149,70 +152,109 @@ type blobTxMeta struct {
evictionBlobFeeJumps float64 // Worse blob fee (converted to fee jumps) across all previous nonces
}
// blobTxForPool is the storage representation of a blob transaction in the
// BlobTxForPool is the storage representation of a blob transaction in the
// blobpool.
type blobTxForPool struct {
type BlobTxForPool struct {
Tx *types.Transaction // tx without sidecar
Version byte
Commitments []kzg4844.Commitment
Proofs []kzg4844.Proof
Blobs []kzg4844.Blob
CellSidecar *types.BlobTxCellSidecar
}
// Sidecar returns BlobTxSidecar of ptx.
func (ptx *blobTxForPool) Sidecar() *types.BlobTxSidecar {
return types.NewBlobTxSidecar(ptx.Version, ptx.Blobs, ptx.Commitments, ptx.Proofs)
// Sidecar returns BlobTxSidecar of pooled transaction. Since this function
// recovers the blob field in sidecar, it is expansive and needs to be
// avoided if possible. Returns error if recovery fails (e.g. insufficient cells).
func (ptx *BlobTxForPool) sidecar() (*types.BlobTxSidecar, error) {
sidecar := ptx.CellSidecar
blobs, err := kzg4844.RecoverBlobs(sidecar.Cells, sidecar.Custody.Indices())
if err != nil {
return nil, err
}
return types.NewBlobTxSidecar(sidecar.Version, blobs, sidecar.Commitments, sidecar.Proofs), nil
}
// TxSize returns the transaction size on the network without
// reconstructing the transaction.
func (ptx *blobTxForPool) TxSize() uint64 {
var blobs, commitments, proofs uint64
for i := range ptx.Blobs {
blobs += rlp.BytesSize(ptx.Blobs[i][:])
func (ptx *BlobTxForPool) txSize() uint64 {
sidecar := ptx.CellSidecar
var commitments, proofs uint64
for i := range sidecar.Commitments {
commitments += rlp.BytesSize(sidecar.Commitments[i][:])
}
for i := range ptx.Commitments {
commitments += rlp.BytesSize(ptx.Commitments[i][:])
}
for i := range ptx.Proofs {
proofs += rlp.BytesSize(ptx.Proofs[i][:])
for i := range sidecar.Proofs {
proofs += rlp.BytesSize(sidecar.Proofs[i][:])
}
var blob kzg4844.Blob
blobs := uint64(len(sidecar.Commitments)) * rlp.BytesSize(blob[:])
return ptx.Tx.Size() + rlp.ListSize(rlp.ListSize(blobs)+rlp.ListSize(commitments)+rlp.ListSize(proofs))
}
func (ptx *BlobTxForPool) txSizeWithoutBlob() uint64 {
sidecar := ptx.CellSidecar
var commitments, proofs uint64
for i := range sidecar.Commitments {
commitments += rlp.BytesSize(sidecar.Commitments[i][:])
}
for i := range sidecar.Proofs {
proofs += rlp.BytesSize(sidecar.Proofs[i][:])
}
return ptx.Tx.Size() + rlp.ListSize(rlp.ListSize(0)+rlp.ListSize(commitments)+rlp.ListSize(proofs))
}
// ToTx reconstructs a full Transaction with the sidecar attached.
func (ptx *blobTxForPool) ToTx() *types.Transaction {
return ptx.Tx.WithBlobTxSidecar(ptx.Sidecar())
func (ptx *BlobTxForPool) toTx() (*types.Transaction, error) {
sc, err := ptx.sidecar()
if err != nil {
return nil, err
}
return ptx.Tx.WithBlobTxSidecar(sc), nil
}
// newBlobTxForPool decomposes a blob transaction into blobTxForPool type.
func newBlobTxForPool(tx *types.Transaction) *blobTxForPool {
func newBlobTxForPool(tx *types.Transaction) (*BlobTxForPool, error) {
sc := tx.BlobTxSidecar()
if sc == nil {
panic("missing blob tx sidecar")
return nil, errors.New("missing blob tx sidecar")
}
return &blobTxForPool{
Tx: tx.WithoutBlobTxSidecar(),
cells, err := kzg4844.ComputeCells(sc.Blobs)
if err != nil {
return nil, err
}
sidecar := types.BlobTxCellSidecar{
Version: sc.Version,
Commitments: sc.Commitments,
Proofs: sc.Proofs,
Blobs: sc.Blobs,
Cells: cells,
Custody: types.CustodyBitmapAll,
}
return &BlobTxForPool{
Tx: tx.WithoutBlobTxSidecar(),
CellSidecar: &sidecar,
}, nil
}
// encodeForNetwork transforms stored blobTxForPool RLP into the standard
// network transaction encoding. This is used for getRLP.
// encodeForNetwork transforms stored BlobTxForPool RLP into the network
// transaction encoding for the given eth protocol version. Used for getRLP.
//
// Stored RLP: [type_byte || tx_fields, version, [comms], [proofs], [blobs]]
// V0: type_byte || rlp([tx_fields, [blobs], [comms], [proofs]])
// V1: type_byte || rlp([tx_fields, version, [blobs], [comms], [proofs]])
func encodeForNetwork(storedRLP []byte) ([]byte, error) {
// Stored RLP: [type_byte || tx_fields, [version, [cells], [comms], [proofs], custody]]
//
// eth/69, eth/70: [blobs] is recovered from stored cells via kzg.
//
// V0: type_byte || rlp([tx_fields, [blobs], [comms], [proofs]])
// V1: type_byte || rlp([tx_fields, version, [blobs], [comms], [proofs]])
//
// eth/72: [blobs] is replaced by an empty list (cells are fetched separately
//
// via GetCells).
// V0: type_byte || rlp([tx_fields, [], [comms], [proofs]])
// V1: type_byte || rlp([tx_fields, version, [], [comms], [proofs]])
func encodeForNetwork(storedRLP []byte, version uint) ([]byte, error) {
elems, err := rlp.SplitListValues(storedRLP)
if err != nil {
return nil, fmt.Errorf("invalid blobTxForPool RLP: %w", err)
return nil, fmt.Errorf("invalid BlobTxForPool RLP: %w", err)
}
if len(elems) < 5 {
return nil, fmt.Errorf("blobTxForPool has %d elements, need at least 5", len(elems))
if len(elems) < 2 {
return nil, fmt.Errorf("BlobTxForPool has %d elements, need at least 2", len(elems))
}
// 1. Extract tx byte and other tx fields
@ -226,19 +268,53 @@ func encodeForNetwork(storedRLP []byte) ([]byte, error) {
typeByte := txBytes[0]
txRLP := txBytes[1:]
// 2. Find the version of sidecar.
version, _, err := rlp.SplitUint64(elems[1])
if err != nil || version > 255 || version == 0 {
// 2. Split the nested CellSidecar list.
sidecarElems, err := rlp.SplitListValues(elems[1])
if err != nil {
return nil, fmt.Errorf("invalid CellSidecar RLP: %w", err)
}
if len(sidecarElems) < 5 {
return nil, fmt.Errorf("CellSidecar has %d elements, need at least 5", len(sidecarElems))
}
// 3. Find the version of sidecar.
sidecarVersion, _, err := rlp.SplitUint64(sidecarElems[0])
if err != nil || sidecarVersion > 255 || sidecarVersion == 0 {
return nil, fmt.Errorf("invalid version: %w", err)
}
// 3. Extract sidecar elements.
commitmentsRLP := elems[2]
proofsRLP := elems[3]
blobsRLP := elems[4]
// 4. Reconstruct into the network format.
outer := [][]byte{txRLP, elems[1], blobsRLP, commitmentsRLP, proofsRLP}
// 4. Extract sidecar elements.
commitmentsRLP := sidecarElems[2]
proofsRLP := sidecarElems[3]
// 5. Build the [blobs] field for the wire format.
var blobsField []byte
// todo - Didn't use eth.ETH72 due to circular import error in test
if version >= 72 {
// eth/72 omits the blob payload; peers fetch cells separately via GetCells.
blobsField = []byte{0xc0} // RLP-encoded empty list
} else {
// eth/69, eth/70 need actual blobs: recover them from stored cells.
var cells []kzg4844.Cell
if err := rlp.DecodeBytes(sidecarElems[1], &cells); err != nil {
return nil, fmt.Errorf("invalid cells RLP: %w", err)
}
var custody types.CustodyBitmap
if err := rlp.DecodeBytes(sidecarElems[4], &custody); err != nil {
return nil, fmt.Errorf("invalid custody RLP: %w", err)
}
blobs, err := kzg4844.RecoverBlobs(cells, custody.Indices())
if err != nil {
return nil, fmt.Errorf("failed to recover blobs: %w", err)
}
blobsField, err = rlp.EncodeToBytes(blobs)
if err != nil {
return nil, fmt.Errorf("failed to encode blobs: %w", err)
}
}
// 6. Reconstruct into the network format.
outer := [][]byte{txRLP, sidecarElems[0], blobsField, commitmentsRLP, proofsRLP}
body, err := rlp.MergeListValues(outer)
if err != nil {
return nil, err
@ -252,21 +328,23 @@ func encodeForNetwork(storedRLP []byte) ([]byte, error) {
// newBlobTxMeta retrieves the indexed metadata fields from a pooled blob
// transaction and assembles a helper struct to track in memory.
func newBlobTxMeta(id uint64, size uint64, storageSize uint32, ptx *blobTxForPool) *blobTxMeta {
func newBlobTxMeta(id uint64, storageSize uint32, ptx *BlobTxForPool) *blobTxMeta {
meta := &blobTxMeta{
hash: ptx.Tx.Hash(),
vhashes: ptx.Tx.BlobHashes(),
version: ptx.Version,
id: id,
storageSize: storageSize,
size: size,
nonce: ptx.Tx.Nonce(),
costCap: uint256.MustFromBig(ptx.Tx.Cost()),
execTipCap: uint256.MustFromBig(ptx.Tx.GasTipCap()),
execFeeCap: uint256.MustFromBig(ptx.Tx.GasFeeCap()),
blobFeeCap: uint256.MustFromBig(ptx.Tx.BlobGasFeeCap()),
execGas: ptx.Tx.Gas(),
blobGas: ptx.Tx.BlobGas(),
hash: ptx.Tx.Hash(),
vhashes: ptx.Tx.BlobHashes(),
version: ptx.CellSidecar.Version,
id: id,
storageSize: storageSize,
size: ptx.txSize(),
sizeWithoutBlob: ptx.txSizeWithoutBlob(),
nonce: ptx.Tx.Nonce(),
costCap: uint256.MustFromBig(ptx.Tx.Cost()),
execTipCap: uint256.MustFromBig(ptx.Tx.GasTipCap()),
execFeeCap: uint256.MustFromBig(ptx.Tx.GasFeeCap()),
blobFeeCap: uint256.MustFromBig(ptx.Tx.BlobGasFeeCap()),
execGas: ptx.Tx.Gas(),
blobGas: ptx.Tx.BlobGas(),
custody: &ptx.CellSidecar.Custody,
}
meta.basefeeJumps = dynamicFeeJumps(meta.execFeeCap)
meta.blobfeeJumps = dynamicBlobFeeJumps(meta.blobFeeCap)
@ -465,8 +543,10 @@ type BlobPool struct {
stored uint64 // Useful data size of all transactions on disk
limbo *limbo // Persistent data store for the non-finalized blobs
gapped map[common.Address][]*types.Transaction // Transactions that are currently gapped (nonce too high)
gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion
cQueue *conversionQueue
gapped map[common.Address][]*BlobTxForPool // Transactions that are currently gapped (nonce too high)
gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion
signer types.Signer // Transaction signer to use for sender recovery
chain BlockChain // Chain object to access the state through
@ -501,7 +581,7 @@ func New(config Config, chain BlockChain, hasPendingAuth func(common.Address) bo
lookup: newLookup(),
index: make(map[common.Address][]*blobTxMeta),
spent: make(map[common.Address]*uint256.Int),
gapped: make(map[common.Address][]*types.Transaction),
gapped: make(map[common.Address][]*BlobTxForPool),
gappedSource: make(map[common.Hash]common.Address),
}
}
@ -564,11 +644,14 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
)
index := func(id uint64, size uint32, blob []byte) {
err := p.parseTransaction(id, size, blob)
if err != nil {
toDelete = append(toDelete, id)
}
// Transactions in legacy format will be queued for cell computation.
// This entry will be swapped from the store after the conversion.
if errors.Is(err, errLegacyTx) {
convertTxs = append(convertTxs, id)
return
}
if err != nil {
toDelete = append(toDelete, id)
}
}
store, err := billy.Open(billy.Options{Path: queuedir, Repair: true}, slotter, index)
@ -577,45 +660,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
}
p.store = store
// Migrate legacy transactions (types.Transaction) to pooledBlobTx format.
if len(convertTxs) > 0 {
for _, id := range convertTxs {
var tx types.Transaction
data, err := p.store.Get(id)
if err != nil {
continue
}
err = rlp.DecodeBytes(data, &tx)
if err != nil {
continue
}
if tx.BlobTxSidecar() == nil {
continue
}
ptx := newBlobTxForPool(&tx)
blob, err := rlp.EncodeToBytes(ptx)
if err != nil {
continue
}
id, err := p.store.Put(blob)
if err != nil {
continue
}
meta := newBlobTxMeta(id, ptx.TxSize(), p.store.Size(id), ptx)
// If the newly inserted transaction fails to be tracked,
// it should also be removed with those in `toDelete`
sender, err := types.Sender(p.signer, ptx.Tx)
if err != nil {
toDelete = append(toDelete, id)
continue
}
if err := p.trackTransaction(meta, sender); err != nil {
toDelete = append(toDelete, id)
continue
}
}
}
p.cQueue = newConversionQueue()
if len(toDelete) > 0 {
log.Warn("Dropping invalidated blob transactions", "ids", toDelete)
@ -659,7 +704,8 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
// Pool initialized, attach the blob limbo to it to track blobs included
// recently but not yet finalized
p.limbo, err = newLimbo(p.chain.Config(), limbodir)
var convertLimbo []uint64
p.limbo, convertLimbo, err = newLimbo(p.chain.Config(), limbodir)
if err != nil {
p.Close()
return err
@ -678,12 +724,22 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
// Update the metrics and return the constructed pool
datacapGauge.Update(int64(p.config.Datacap))
p.updateStorageMetrics()
if len(convertTxs) > 0 {
p.cQueue.launchConversion(func() { p.convertLegacyTxs(convertTxs) })
}
if len(convertLimbo) > 0 {
p.cQueue.launchConversion(func() { p.convertLegacyLimbo(convertLimbo) })
}
return nil
}
// Close closes down the underlying persistent store.
func (p *BlobPool) Close() error {
var errs []error
if p.cQueue != nil {
p.cQueue.close()
}
if p.limbo != nil { // Close might be invoked due to error in constructor, before p,limbo is set
if err := p.limbo.Close(); err != nil {
errs = append(errs, err)
@ -704,25 +760,119 @@ func (p *BlobPool) Close() error {
// parseTransaction is a callback method on pool creation that gets called for
// each transaction on disk to create the in-memory metadata index.
// Return value `bool` is set to true when the entry has old Transaction type.
// Announced state is not initialized here, it needs to be initialized separately.
//
// If a legacy types.Transaction is found on disk, it is returned for migration
// in Init (the old ID will be deleted and a new pooledBlobTx written).
// If a pooledBlobTx is found, it is indexed directly and nil is returned.
func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error {
var ptx blobTxForPool
var ptx BlobTxForPool
if err := rlp.DecodeBytes(blob, &ptx); err != nil {
kind, content, _, splitErr := rlp.Split(blob)
// check whether it is legacy tx type
if splitErr == nil && kind == rlp.String && len(content) > 1 && content[0] == 3 {
return errLegacyTx
}
log.Error("Failed to decode blob pool entry", "id", id, "err", err)
return err
}
meta := newBlobTxMeta(id, ptx.TxSize(), size, &ptx)
meta := newBlobTxMeta(id, size, &ptx)
sender, err := types.Sender(p.signer, ptx.Tx)
if err != nil {
// This path is impossible unless the signature validity changes across
// restarts. For that ever improbable case, recover gracefully by ignoring
// this data entry.
log.Error("Failed to recover blob tx sender", "id", id, "hash", ptx.Tx.Hash(), "err", err)
return err
}
return p.trackTransaction(meta, sender)
}
func (p *BlobPool) convertLegacyTxs(ids []uint64) {
start := time.Now()
var converted, discarded int
for _, id := range ids {
p.lock.Lock()
data, err := p.store.Get(id)
if err != nil {
p.lock.Unlock()
continue
}
if derr := p.store.Delete(id); derr != nil {
log.Error("Failed to delete legacy blob tx", "id", id, "err", derr)
p.lock.Unlock()
continue
}
p.lock.Unlock()
var tx types.Transaction
if err := rlp.DecodeBytes(data, &tx); err != nil {
log.Error("Failed to decode legacy blob tx", "id", id, "err", err)
continue
}
ptx, err := newBlobTxForPool(&tx)
if err != nil {
log.Error("Failed to convert legacy blob tx", "hash", tx.Hash(), "err", err)
continue
}
if err := p.AddPooledTx(ptx); err != nil {
log.Debug("Discarded converted blob tx", "hash", tx.Hash(), "err", err)
discarded++
} else {
converted++
}
}
log.Info("Completed blob transaction conversion", "converted", converted, "discarded", discarded, "elapsed", common.PrettyDuration(time.Since(start)))
}
func (p *BlobPool) convertLegacyLimbo(ids []uint64) {
start := time.Now()
var converted, discarded int
for _, id := range ids {
p.lock.Lock()
data, err := p.limbo.store.Get(id)
if err != nil {
p.lock.Unlock()
continue
}
if derr := p.limbo.store.Delete(id); derr != nil {
log.Error("Failed to delete legacy blob tx", "id", id, "err", derr)
p.lock.Unlock()
continue
}
p.lock.Unlock()
var legacy struct {
TxHash common.Hash
Block uint64
Tx *types.Transaction
}
if err := rlp.DecodeBytes(data, &legacy); err != nil {
log.Error("Failed to decode legacy limbo entry", "id", id, "err", err)
continue
}
if legacy.Tx == nil || legacy.Tx.BlobTxSidecar() == nil {
continue
}
ptx, err := newBlobTxForPool(legacy.Tx)
if err != nil {
log.Error("Failed to convert legacy limbo entry", "hash", legacy.TxHash, "err", err)
continue
}
p.lock.Lock()
if err := p.limbo.setAndIndex(ptx, legacy.Block); err != nil {
log.Error("Failed to re-store converted limbo entry", "hash", legacy.TxHash, "err", err)
discarded++
} else {
converted++
}
p.lock.Unlock()
}
log.Info("Completed limbo blob conversion", "converted", converted, "discarded", discarded, "elapsed", common.PrettyDuration(time.Since(start)))
}
// trackTransaction registers a transaction's metadata in the pool's indices.
func (p *BlobPool) trackTransaction(meta *blobTxMeta, sender common.Address) error {
if p.lookup.exists(meta.hash) {
@ -1006,7 +1156,7 @@ func (p *BlobPool) offload(addr common.Address, nonce uint64, id uint64, inclusi
log.Error("Blobs missing for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err)
return
}
var ptx blobTxForPool
var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
log.Error("Blobs corrupted for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err)
return
@ -1094,7 +1244,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) {
log.Error("Blobs missing for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err)
continue
}
var ptx blobTxForPool
var ptx BlobTxForPool
if err = rlp.DecodeBytes(data, &ptx); err != nil {
log.Error("Blobs corrupted for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err)
continue
@ -1131,7 +1281,7 @@ func (p *BlobPool) vhashesByTx() map[common.Hash][]common.Hash {
// getByVhash reads and decodes the blob transaction which has the given
// versioned hash. Returns nil if unavailable.
func (p *BlobPool) getByVhash(vhash common.Hash) *blobTxForPool {
func (p *BlobPool) getByVhash(vhash common.Hash) *BlobTxForPool {
p.lock.RLock()
txID, exists := p.lookup.storeidOfBlob(vhash)
p.lock.RUnlock()
@ -1143,7 +1293,7 @@ func (p *BlobPool) getByVhash(vhash common.Hash) *blobTxForPool {
log.Error("Tracked blob transaction missing from store", "id", txID, "err", err)
return nil
}
var ptx blobTxForPool
var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
log.Error("Blobs corrupted for tracked transaction", "id", txID, "err", err)
return nil
@ -1300,7 +1450,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error {
// A reorged-out legacy blob transaction can therefore not be re-added, so drop
// it on the floor instead of putting it back.
head := p.head.Load()
if p.chain.Config().IsOsaka(head.Number, head.Time) && ptx.Version == types.BlobSidecarVersion0 {
if p.chain.Config().IsOsaka(head.Number, head.Time) && ptx.CellSidecar.Version == types.BlobSidecarVersion0 {
log.Debug("Dropping reorged legacy blob transaction", "hash", txhash)
return errors.New("legacy blob sidecar unsupported post-osaka")
}
@ -1314,7 +1464,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error {
log.Error("Failed to write transaction into storage", "hash", ptx.Tx.Hash(), "err", err)
return err
}
meta := newBlobTxMeta(id, ptx.TxSize(), p.store.Size(id), ptx)
meta := newBlobTxMeta(id, p.store.Size(id), ptx)
if _, ok := p.index[addr]; !ok {
if err := p.reserver.Hold(addr); err != nil {
log.Warn("Failed to reserve account for blob pool", "tx", ptx.Tx.Hash(), "from", addr, "err", err)
@ -1543,6 +1693,7 @@ func (p *BlobPool) Has(hash common.Hash) bool {
return poolHas || gapped
}
// getRLP returns the raw RLP-encoded pooledBlobTx data from the store.
func (p *BlobPool) getRLP(hash common.Hash) []byte {
// Track the amount of time waiting to retrieve a fully resolved blob tx from
// the pool and the amount of time actually spent on pulling the data from disk.
@ -1569,32 +1720,34 @@ func (p *BlobPool) getRLP(hash common.Hash) []byte {
}
// Get returns a transaction if it is contained in the pool, or nil otherwise.
// Note that this function always try to recover full blobs
func (p *BlobPool) Get(hash common.Hash) *types.Transaction {
data := p.getRLP(hash)
if len(data) == 0 {
return nil
}
var ptx blobTxForPool
var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
id, _ := p.lookup.storeidOfTx(hash)
log.Error("Blobs corrupted for traced transaction",
"hash", hash, "id", id, "err", err)
log.Error("Blobs corrupted for traced transaction", "hash", hash, "id", id, "err", err)
return nil
}
return ptx.ToTx()
tx, err := ptx.toTx()
if err != nil {
log.Error("Failed to recover transaction in blobpool", "hash", hash, "err", err)
return nil
}
return tx
}
// GetRLP returns a RLP-encoded transaction for network if it is contained in the pool.
// It converts the pool's internal type to the RLP format used by the eth protocol:
// e.g. type_byte || [..., version, [blobs], [comms], [proofs]]
func (p *BlobPool) GetRLP(hash common.Hash) []byte {
// GetRLP returns an RLP-encoded transaction if it is contained in the pool.
func (p *BlobPool) GetRLP(hash common.Hash, version uint) []byte {
data := p.getRLP(hash)
if len(data) == 0 {
// Not in this pool, do not log.
return nil
}
rlp, err := encodeForNetwork(data)
rlp, err := encodeForNetwork(data, version)
if err != nil {
log.Error("Failed to encode pooled tx into the network type", "hash", hash, "err", err)
return nil
@ -1611,13 +1764,14 @@ func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
p.lock.RLock()
defer p.lock.RUnlock()
size, ok := p.lookup.sizeOfTx(hash)
meta, ok := p.lookup.txIndex[hash]
if !ok {
return nil
}
return &txpool.TxMetadata{
Type: types.BlobTxType,
Size: size,
Type: types.BlobTxType,
Size: meta.size,
SizeWithoutBlob: meta.sizeWithoutBlob,
}
}
@ -1657,12 +1811,16 @@ func (p *BlobPool) getBlobs(vhashes []common.Hash, version byte) (_ []*kzg4844.B
}
// Decode the blob transaction
var ptx blobTxForPool
var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
log.Error("Blobs corrupted for traced transaction", "id", txID, "err", err)
continue
}
sidecar := ptx.Sidecar()
sidecar, err := ptx.sidecar()
if err != nil {
log.Error("Failed to recover sidecar in blobpool", "id", txID, "err", err)
continue
}
// Traverse the blobs in the transaction
for i, hash := range ptx.Tx.BlobHashes() {
list, ok := indices[hash]
@ -1699,6 +1857,90 @@ func (p *BlobPool) getBlobs(vhashes []common.Hash, version byte) (_ []*kzg4844.B
return blobs, commitments, proofs, nil
}
// GetBlobHashes returns the blob versioned hashes for a given transaction hash.
func (p *BlobPool) GetBlobHashes(txHash common.Hash) []common.Hash {
p.lock.RLock()
defer p.lock.RUnlock()
vhashes, ok := p.lookup.blobHashesOfTx(txHash)
if !ok {
return nil
}
return vhashes
}
// GetBlobCells returns cells for the given versioned blob hashes. Nil entries
// mean that the cell was not available.
func (p *BlobPool) GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error) {
var (
cells = make([][]*kzg4844.Cell, len(vhashes))
proofs = make([][]*kzg4844.Proof, len(vhashes))
vindex = make(map[common.Hash][]int) // Indices of versioned hashes in the request
filled = make(map[common.Hash]struct{})
)
for i, h := range vhashes {
vindex[h] = append(vindex[h], i)
}
requestedIndices := mask.Indices()
for _, vhash := range vhashes {
if _, ok := filled[vhash]; ok {
continue
}
p.lock.RLock()
txID, exists := p.lookup.storeidOfBlob(vhash)
p.lock.RUnlock()
if !exists {
continue
}
data, err := p.store.Get(txID)
if err != nil {
continue
}
var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
continue
}
tx := ptx.Tx
cellsPerBlob := ptx.CellSidecar.Custody.OneCount()
storedIndices := ptx.CellSidecar.Custody.Indices()
for blobIdx, hash := range tx.BlobHashes() {
indices, ok := vindex[hash]
if !ok {
continue
}
filled[hash] = struct{}{}
blobCells := make([]*kzg4844.Cell, len(requestedIndices))
blobProofs := make([]*kzg4844.Proof, len(requestedIndices))
for i, cellIdx := range requestedIndices {
pos := -1
for k, storedIdx := range storedIndices {
if storedIdx == cellIdx {
pos = k
break
}
}
if pos >= 0 {
cell := ptx.CellSidecar.Cells[blobIdx*cellsPerBlob+pos]
blobCells[i] = &cell
proofIdx := blobIdx*kzg4844.CellProofsPerBlob + int(cellIdx)
if proofIdx < len(ptx.CellSidecar.Proofs) {
proof := ptx.CellSidecar.Proofs[proofIdx]
blobProofs[i] = &proof
}
}
}
for _, idx := range indices {
cells[idx] = blobCells
proofs[idx] = blobProofs
}
}
}
return cells, proofs, nil
}
// availableBlobs returns whether the blobs are available in the subpool.
func (p *BlobPool) availableBlobs(vhashes []common.Hash) []bool {
available := make([]bool, len(vhashes))
@ -1718,14 +1960,23 @@ func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error {
if errs[i] = p.ValidateTxBasics(tx); errs[i] != nil {
continue
}
errs[i] = p.add(tx)
ptx, err := p.cQueue.convert(tx)
if err != nil {
errs[i] = err
continue
}
if err := txpool.ValidateCells(ptx.CellSidecar); err != nil {
errs[i] = err
continue
}
errs[i] = p.AddPooledTx(ptx)
}
return errs
}
// add inserts a new blob transaction into the pool if it passes validation (both
// consensus validity and pool restrictions).
func (p *BlobPool) add(tx *types.Transaction) (err error) {
func (p *BlobPool) AddPooledTx(ptx *BlobTxForPool) (err error) {
// The blob pool blocks on adding a transaction. This is because blob txs are
// only even pulled from the network, so this method will act as the overload
// protection for fetches.
@ -1738,13 +1989,15 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
addtimeHist.Update(time.Since(start).Nanoseconds())
}(time.Now())
return p.addLocked(tx, true)
return p.addLocked(ptx, true)
}
// addLocked inserts a new blob transaction into the pool if it passes validation (both
// consensus validity and pool restrictions). It must be called with the pool lock held.
// Only for internal use.
func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error) {
func (p *BlobPool) addLocked(ptx *BlobTxForPool, checkGapped bool) (err error) {
tx := ptx.Tx
// Ensure the transaction is valid from all perspectives
if err := p.validateTx(tx); err != nil {
log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err)
@ -1761,7 +2014,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
from, _ := types.Sender(p.signer, tx)
allowance := p.gappedAllowance(from)
if allowance >= 1 && len(p.gappedSource) < maxGapped {
p.gapped[from] = append(p.gapped[from], tx)
p.gapped[from] = append(p.gapped[from], ptx)
p.gappedSource[tx.Hash()] = from
gappedGauge.Update(int64(len(p.gappedSource)))
log.Trace("added tx to gapped blob queue", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
@ -1807,7 +2060,6 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
}
// Transaction permitted into the pool from a nonce and cost perspective,
// insert it into the database and update the indices
ptx := newBlobTxForPool(tx)
blob, err := rlp.EncodeToBytes(ptx)
if err != nil {
log.Error("Failed to encode transaction for storage", "hash", tx.Hash(), "err", err)
@ -1817,7 +2069,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
if err != nil {
return err
}
meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), ptx)
meta := newBlobTxMeta(id, p.store.Size(id), ptx)
var (
next = p.state.GetNonce(from)
@ -1928,13 +2180,13 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
// We have to add in nonce order, but we want to stable sort to cater for situations
// where transactions are replaced, keeping the original receive order for same nonce
sort.SliceStable(gtxs, func(i, j int) bool {
return gtxs[i].Nonce() < gtxs[j].Nonce()
return gtxs[i].Tx.Nonce() < gtxs[j].Tx.Nonce()
})
for len(gtxs) > 0 {
stateNonce := p.state.GetNonce(from)
firstgap := stateNonce + uint64(len(p.index[from]))
if gtxs[0].Nonce() > firstgap {
if gtxs[0].Tx.Nonce() > firstgap {
// Anything beyond the first gap is not addable yet
break
}
@ -1942,26 +2194,26 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
// Drop any buffered transactions that became stale in the meantime (included in chain or replaced)
// If we arrive to the transaction in the pending range (between the state Nonce and first gap, we
// try to add them now while removing from here.
tx := gtxs[0]
ptx := gtxs[0]
gtxs[0] = nil
gtxs = gtxs[1:]
delete(p.gappedSource, tx.Hash())
delete(p.gappedSource, ptx.Tx.Hash())
if tx.Nonce() < stateNonce {
if ptx.Tx.Nonce() < stateNonce {
// Stale, drop it. Eventually we could add to limbo here if hash matches.
log.Trace("Gapped blob transaction became stale", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from]))
log.Trace("Gapped blob transaction became stale", "hash", ptx.Tx.Hash(), "from", from, "nonce", ptx.Tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from]))
continue
}
if tx.Nonce() <= firstgap {
if ptx.Tx.Nonce() <= firstgap {
// If we hit the pending range, including the first gap, add it and continue to try to add more.
// We do not recurse here, but continue to loop instead.
// We are under lock, so we can add the transaction directly.
if err := p.addLocked(tx, false); err == nil {
if err := p.addLocked(ptx, false); err == nil {
gappedPromotedMeter.Mark(1)
log.Trace("Gapped blob transaction added to pool", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
log.Trace("Gapped blob transaction added to pool", "hash", ptx.Tx.Hash(), "from", from, "nonce", ptx.Tx.Nonce(), "qlen", len(p.gapped[from]))
} else {
log.Trace("Gapped blob transaction not accepted", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "err", err)
log.Trace("Gapped blob transaction not accepted", "hash", ptx.Tx.Hash(), "from", from, "nonce", ptx.Tx.Nonce(), "err", err)
}
}
}
@ -2087,6 +2339,10 @@ func (p *BlobPool) Pending(filter txpool.PendingFilter) (map[common.Address][]*t
break // execution gas limit is too high
}
}
// Skip transactions without enough cells to recover blobs
if !filter.PartialCells && tx.custody != nil && tx.custody.OneCount() < kzg4844.DataPerBlob {
break // not enough cells to build a full payload, discard rest of txs from the account
}
// Transaction was accepted according to the filter, append to the pending list
lazies = append(lazies, &txpool.LazyTransaction{
Pool: p,
@ -2229,10 +2485,10 @@ func (p *BlobPool) evictGapped() {
// and we overwrite the slice for this account after filtering.
keep := txs[:0]
for i, gtx := range txs {
if gtx.Time().Before(cutoff) || gtx.Nonce() < nonce {
if gtx.Tx.Time().Before(cutoff) || gtx.Tx.Nonce() < nonce {
// Evict old or stale transactions
// Should we add stale to limbo here if it would belong?
delete(p.gappedSource, gtx.Hash())
delete(p.gappedSource, gtx.Tx.Hash())
txs[i] = nil // Explicitly nil out evicted element
} else {
keep = append(keep, gtx)
@ -2355,7 +2611,7 @@ func (p *BlobPool) Clear() {
// Reset counters and the gapped buffer
p.stored = 0
p.gapped = make(map[common.Address][]*types.Transaction)
p.gapped = make(map[common.Address][]*BlobTxForPool)
p.gappedSource = make(map[common.Hash]common.Address)
var (
@ -2364,3 +2620,13 @@ func (p *BlobPool) Clear() {
)
p.evict = newPriceHeap(basefee, blobfee, p.index)
}
// GetCustody returns the custody bitmap for a given transaction hash.
func (p *BlobPool) GetCustody(hash common.Hash) *types.CustodyBitmap {
p.lock.RLock()
defer p.lock.RUnlock()
if meta := p.lookup.txIndex[hash]; meta != nil {
return &meta.custody
}
return nil
}

388
core/txpool/blobpool/blobpool_test.go
View file

@ -31,6 +31,7 @@ import (
"slices"
"sync"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
@ -233,7 +234,8 @@ func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64,
// encodeForPool encodes a blob transaction in the blobTxForPool storage format.
func encodeForPool(tx *types.Transaction) []byte {
blob, _ := rlp.EncodeToBytes(newBlobTxForPool(tx))
ptx, _ := newBlobTxForPool(tx)
blob, _ := rlp.EncodeToBytes(ptx)
return blob
}
@ -267,6 +269,15 @@ func makeMultiBlobTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCa
return types.MustSignNewTx(key, types.LatestSigner(params.MainnetChainConfig), blobtx)
}
// removeBlobs returns a copy of tx with the blob payload removed from the
// sidecar, keeping commitments, proofs and cells intact (simulating what
// ETH/72 peers send).
func removeBlobs(tx *types.Transaction) *types.Transaction {
sidecar := tx.BlobTxSidecar().Copy()
sidecar.Blobs = nil
return tx.WithBlobTxSidecar(sidecar)
}
// makeUnsignedTx is a utility method to construct a random blob transaction
// without signing it.
func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64) *types.BlobTx {
@ -471,7 +482,7 @@ func verifyBlobRetrievals(t *testing.T, pool *BlobPool) {
// - 8. Fully duplicate transactions (matching hash) must be dropped
// - 9. Duplicate nonces from the same account must be dropped
func TestOpenDrops(t *testing.T) {
//log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true)))
// log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true)))
// Create a temporary folder for the persistent backend
storage := t.TempDir()
@ -498,75 +509,76 @@ func TestOpenDrops(t *testing.T) {
S: new(uint256.Int),
})
blob, _ := rlp.EncodeToBytes(tx)
badsig, _ := store.Put(blob)
badsig := tx.Hash()
store.Put(blob)
// Insert a sequence of transactions with a nonce gap in between to verify
// that anything gapped will get evicted (case 3).
var (
gapper, _ = crypto.GenerateKey()
valids = make(map[uint64]struct{})
gapped = make(map[uint64]struct{})
valids = make(map[common.Hash]struct{})
gapped = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 3, 4, 6, 7} { // first gap at #2, another at #5
tx := makeTx(nonce, 1, 1, 1, gapper)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce < 2 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
gapped[id] = struct{}{}
gapped[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions with a gapped starting nonce to verify
// that the entire set will get dropped (case 3).
var (
dangler, _ = crypto.GenerateKey()
dangling = make(map[uint64]struct{})
dangling = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{1, 2, 3} { // first gap at #0, all set dangling
tx := makeTx(nonce, 1, 1, 1, dangler)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
dangling[id] = struct{}{}
store.Put(blob)
dangling[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with already passed nonces to verify
// that the entire set will get dropped (case 4).
var (
filler, _ = crypto.GenerateKey()
filled = make(map[uint64]struct{})
filled = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // account nonce at 3, all set filled
tx := makeTx(nonce, 1, 1, 1, filler)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
filled[id] = struct{}{}
store.Put(blob)
filled[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with partially passed nonces to verify
// that the included part of the set will get dropped (case 4).
var (
overlapper, _ = crypto.GenerateKey()
overlapped = make(map[uint64]struct{})
overlapped = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2, 3} { // account nonce at 2, half filled
tx := makeTx(nonce, 1, 1, 1, overlapper)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce >= 2 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
overlapped[id] = struct{}{}
overlapped[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions with an underpriced first to verify that
// the entire set will get dropped (case 5).
var (
underpayer, _ = crypto.GenerateKey()
underpaid = make(map[uint64]struct{})
underpaid = make(map[common.Hash]struct{})
)
for i := 0; i < 5; i++ { // make #0 underpriced
var tx *types.Transaction
@ -577,15 +589,15 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
id, _ := store.Put(blob)
underpaid[id] = struct{}{}
store.Put(blob)
underpaid[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with an underpriced in between to verify
// that it and anything newly gapped will get evicted (case 5).
var (
outpricer, _ = crypto.GenerateKey()
outpriced = make(map[uint64]struct{})
outpriced = make(map[common.Hash]struct{})
)
for i := 0; i < 5; i++ { // make #2 underpriced
var tx *types.Transaction
@ -596,18 +608,18 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if i < 2 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
outpriced[id] = struct{}{}
outpriced[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions fully overdrafted to verify that the
// entire set will get invalidated (case 6).
var (
exceeder, _ = crypto.GenerateKey()
exceeded = make(map[uint64]struct{})
exceeded = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // nonce 0 overdrafts the account
var tx *types.Transaction
@ -618,14 +630,14 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
id, _ := store.Put(blob)
exceeded[id] = struct{}{}
store.Put(blob)
exceeded[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions partially overdrafted to verify that part
// of the set will get invalidated (case 6).
var (
overdrafter, _ = crypto.GenerateKey()
overdrafted = make(map[uint64]struct{})
overdrafted = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // nonce 1 overdrafts the account
var tx *types.Transaction
@ -636,44 +648,46 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce < 1 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
overdrafted[id] = struct{}{}
overdrafted[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions overflowing the account cap to verify
// that part of the set will get invalidated (case 7).
var (
overcapper, _ = crypto.GenerateKey()
overcapped = make(map[uint64]struct{})
overcapped = make(map[common.Hash]struct{})
)
for nonce := uint64(0); nonce < maxTxsPerAccount+3; nonce++ {
blob := encodeForPool(makeTx(nonce, 1, 1, 1, overcapper))
tx := makeTx(nonce, 1, 1, 1, overcapper)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce < maxTxsPerAccount {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
overcapped[id] = struct{}{}
overcapped[tx.Hash()] = struct{}{}
}
}
// Insert a batch of duplicated transactions to verify that only one of each
// version will remain (case 8).
var (
duplicater, _ = crypto.GenerateKey()
duplicated = make(map[uint64]struct{})
duplicated = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} {
blob := encodeForPool(makeTx(nonce, 1, 1, 1, duplicater))
tx := makeTx(nonce, 1, 1, 1, duplicater)
blob := encodeForPool(tx)
for i := 0; i < int(nonce)+1; i++ {
id, _ := store.Put(blob)
store.Put(blob)
if i == 0 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
duplicated[id] = struct{}{}
duplicated[tx.Hash()] = struct{}{}
}
}
}
@ -681,17 +695,18 @@ func TestOpenDrops(t *testing.T) {
// remain (case 9).
var (
repeater, _ = crypto.GenerateKey()
repeated = make(map[uint64]struct{})
repeated = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} {
for i := 0; i < int(nonce)+1; i++ {
blob := encodeForPool(makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater))
tx := makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if i == 0 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
repeated[id] = struct{}{}
repeated[tx.Hash()] = struct{}{}
}
}
}
@ -728,39 +743,41 @@ func TestOpenDrops(t *testing.T) {
// Verify that the malformed (case 1), badly signed (case 2) and gapped (case
// 3) txs have been deleted from the pool
alive := make(map[uint64]struct{})
alive := make(map[common.Hash]struct{})
for _, txs := range pool.index {
for _, tx := range txs {
switch tx.id {
case malformed:
t.Errorf("malformed RLP transaction remained in storage")
case badsig:
t.Errorf("invalidly signed transaction remained in storage")
default:
if _, ok := dangling[tx.id]; ok {
if badsig == tx.hash {
t.Errorf("invalidly signed transaction remained in storage")
}
if _, ok := dangling[tx.hash]; ok {
t.Errorf("dangling transaction remained in storage: %d", tx.id)
} else if _, ok := filled[tx.id]; ok {
} else if _, ok := filled[tx.hash]; ok {
t.Errorf("filled transaction remained in storage: %d", tx.id)
} else if _, ok := overlapped[tx.id]; ok {
} else if _, ok := overlapped[tx.hash]; ok {
t.Errorf("overlapped transaction remained in storage: %d", tx.id)
} else if _, ok := gapped[tx.id]; ok {
} else if _, ok := gapped[tx.hash]; ok {
t.Errorf("gapped transaction remained in storage: %d", tx.id)
} else if _, ok := underpaid[tx.id]; ok {
} else if _, ok := underpaid[tx.hash]; ok {
t.Errorf("underpaid transaction remained in storage: %d", tx.id)
} else if _, ok := outpriced[tx.id]; ok {
} else if _, ok := outpriced[tx.hash]; ok {
t.Errorf("outpriced transaction remained in storage: %d", tx.id)
} else if _, ok := exceeded[tx.id]; ok {
} else if _, ok := exceeded[tx.hash]; ok {
t.Errorf("fully overdrafted transaction remained in storage: %d", tx.id)
} else if _, ok := overdrafted[tx.id]; ok {
} else if _, ok := overdrafted[tx.hash]; ok {
t.Errorf("partially overdrafted transaction remained in storage: %d", tx.id)
} else if _, ok := overcapped[tx.id]; ok {
} else if _, ok := overcapped[tx.hash]; ok {
t.Errorf("overcapped transaction remained in storage: %d", tx.id)
} else if _, ok := duplicated[tx.id]; ok {
t.Errorf("duplicated transaction remained in storage: %d", tx.id)
} else if _, ok := repeated[tx.id]; ok {
} else if _, ok := repeated[tx.hash]; ok {
t.Errorf("repeated nonce transaction remained in storage: %d", tx.id)
} else {
alive[tx.id] = struct{}{}
if _, ok := alive[tx.hash]; ok {
t.Errorf("duplicated transaction remained in storage: %d", tx.id)
}
alive[tx.hash] = struct{}{}
}
}
}
@ -769,14 +786,14 @@ func TestOpenDrops(t *testing.T) {
if len(alive) != len(valids) {
t.Errorf("valid transaction count mismatch: have %d, want %d", len(alive), len(valids))
}
for id := range alive {
if _, ok := valids[id]; !ok {
t.Errorf("extra transaction %d", id)
for hash := range alive {
if _, ok := valids[hash]; !ok {
t.Errorf("extra transaction %s", hash)
}
}
for id := range valids {
if _, ok := alive[id]; !ok {
t.Errorf("missing transaction %d", id)
for hash := range valids {
if _, ok := alive[hash]; !ok {
t.Errorf("missing transaction %s", hash)
}
}
// Verify all the calculated pool internals. Interestingly, this is **not**
@ -995,7 +1012,10 @@ func TestOpenCap(t *testing.T) {
keep = []common.Address{addr1, addr3}
drop = []common.Address{addr2}
size = 2 * (txAvgSize + blobSize + uint64(txBlobOverhead))
// After migration to pooledBlobTx, cells (128 x 2048 = 2*blobSize) replace blobs.
// The actual billy slot size for pooledBlobTx is 2*(blobSize+txBlobOverhead)+txAvgSize.
pooledSlotSize uint64 = 2*(blobSize+uint64(txBlobOverhead)) + txAvgSize
size = 2 * pooledSlotSize
)
store.Put(blob1)
store.Put(blob2)
@ -1004,7 +1024,7 @@ func TestOpenCap(t *testing.T) {
// Verify pool capping twice: first by reducing the data cap, then restarting
// with a high cap to ensure everything was persisted previously
for _, datacap := range []uint64{2 * (txAvgSize + blobSize + uint64(txBlobOverhead)), 1000 * (txAvgSize + blobSize + uint64(txBlobOverhead))} {
for _, datacap := range []uint64{size, 1000 * pooledSlotSize} {
// Create a blob pool out of the pre-seeded data, but cap it to 2 blob transaction
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
@ -1310,6 +1330,14 @@ func TestLegacyTxConversion(t *testing.T) {
}
defer pool.Close()
deadline := time.Now().Add(30 * time.Second)
for pool.Get(tx1.Hash()) == nil || pool.Get(tx2.Hash()) == nil {
if time.Now().After(deadline) {
t.Fatalf("legacy txs were not converted in time")
}
time.Sleep(10 * time.Millisecond)
}
// Both transactions should be retrievable.
for _, want := range []*types.Transaction{tx1, tx2} {
got := pool.Get(want.Hash())
@ -1326,7 +1354,7 @@ func TestLegacyTxConversion(t *testing.T) {
// Legacy formats should not exist on pool.store
pool.store.Iterate(func(id uint64, size uint32, blob []byte) {
var ptx blobTxForPool
var ptx BlobTxForPool
if err := rlp.DecodeBytes(blob, &ptx); err != nil {
t.Errorf("entry %d not in new blobTxForPool format: %v", id, err)
}
@ -1335,6 +1363,75 @@ func TestLegacyTxConversion(t *testing.T) {
verifyPoolInternals(t, pool)
}
func TestLegacyLimboConversion(t *testing.T) {
storage := t.TempDir()
os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700)
limbodir := filepath.Join(storage, limboedTransactionStore)
os.MkdirAll(limbodir, 0700)
key, _ := crypto.GenerateKey()
tx := makeMultiBlobTx(0, 1, 1000, 100, 2, 0, key)
store, err := billy.Open(billy.Options{Path: limbodir}, newSlotterEIP7594(testMaxBlobsPerBlock), nil)
if err != nil {
t.Fatalf("failed to open limbo billy: %v", err)
}
data, err := rlp.EncodeToBytes(&struct {
TxHash common.Hash
Block uint64
Tx *types.Transaction
}{TxHash: tx.Hash(), Block: 42, Tx: tx})
if err != nil {
t.Fatalf("failed to legacy-encode limbo entry: %v", err)
}
if _, err := store.Put(data); err != nil {
t.Fatalf("failed to put legacy limbo entry: %v", err)
}
store.Close()
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
statedb.Commit(0, true, false)
chain := &testBlockChain{
config: params.MainnetChainConfig,
basefee: uint256.NewInt(params.InitialBaseFee),
blobfee: uint256.NewInt(params.BlobTxMinBlobGasprice),
statedb: statedb,
}
pool := New(Config{Datadir: storage}, chain, nil)
if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil {
t.Fatalf("failed to create blob pool: %v", err)
}
defer pool.Close()
deadline := time.Now().Add(30 * time.Second)
for {
pool.lock.RLock()
_, ok := pool.limbo.index[tx.Hash()]
pool.lock.RUnlock()
if ok {
break
}
if time.Now().After(deadline) {
t.Fatalf("legacy limbo entry was not converted in time")
}
time.Sleep(10 * time.Millisecond)
}
pool.lock.Lock()
ptx, err := pool.limbo.pull(tx.Hash())
pool.lock.Unlock()
if err != nil {
t.Fatalf("failed to pull converted limbo entry: %v", err)
}
full, err := ptx.toTx()
if err != nil {
t.Fatalf("failed to reconstruct tx from converted limbo entry: %v", err)
}
if full.Hash() != tx.Hash() {
t.Fatalf("converted limbo tx hash mismatch: have %s, want %s", full.Hash(), tx.Hash())
}
}
// TestBlobCountLimit tests the blobpool enforced limits on the max blob count.
func TestBlobCountLimit(t *testing.T) {
var (
@ -1384,7 +1481,7 @@ func TestBlobCountLimit(t *testing.T) {
// Check that first succeeds second fails.
if errs[0] != nil {
t.Fatalf("expected tx with 7 blobs to succeed, got %v", errs[0])
t.Fatalf("expected tx with 7 blobs to succeed, got: %v", errs[0])
}
if !errors.Is(errs[1], txpool.ErrTxBlobLimitExceeded) {
t.Fatalf("expected tx with 8 blobs to fail, got: %v", errs[1])
@ -2103,23 +2200,40 @@ func TestGetBlobs(t *testing.T) {
pool.Close()
}
// TestEncodeForNetwork verifies that encodeForNetwork produces output identical
// to rlp.EncodeToBytes on the original transaction, for both V0 and V1 sidecars.
// TestEncodeForNetwork verifies that encodeForNetwork produces the correct wire
// encoding for each (sidecar version, eth protocol version) combination.
// - eth/69, eth/70: blobs recovered from cells, output matches the original tx
// - eth/72: blob payload omitted, output matches removeBlobs(tx)
func TestEncodeForNetwork(t *testing.T) {
t.Run("v1", func(t *testing.T) { testEncodeForNetwork(t) })
cases := []struct {
name string
ethVer uint
}{
{"eth70", 70},
{"eth72", 72},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
testEncodeForNetwork(t, tc.ethVer)
})
}
}
func testEncodeForNetwork(t *testing.T) {
func testEncodeForNetwork(t *testing.T, ethVer uint) {
key, _ := crypto.GenerateKey()
tx := makeMultiBlobTx(0, 1, 1, 1, 1, 0, key)
wantRLP, err := rlp.EncodeToBytes(tx)
wantTx := tx
if ethVer >= 72 {
wantTx = removeBlobs(tx)
}
wantRLP, err := rlp.EncodeToBytes(wantTx)
if err != nil {
t.Fatalf("failed to encode tx: %v", err)
}
storedRLP := encodeForPool(tx)
gotRLP, err := encodeForNetwork(storedRLP)
gotRLP, err := encodeForNetwork(storedRLP, ethVer)
if err != nil {
t.Fatalf("encodeForNetwork failed: %v", err)
}
@ -2190,7 +2304,8 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) {
b.Fatal(err)
}
statedb.AddBalance(addr, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
pool.add(tx)
pooledTx, _ := newBlobTxForPool(tx)
pool.AddPooledTx(pooledTx)
}
statedb.Commit(0, true, false)
defer pool.Close()
@ -2211,3 +2326,116 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) {
}
}
}
func TestGetCells(t *testing.T) {
storage := t.TempDir()
os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700)
store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotterEIP7594(params.BlobTxMaxBlobs), nil)
var (
key1, _ = crypto.GenerateKey()
addr1 = crypto.PubkeyToAddress(key1.PublicKey)
tx1 = makeMultiBlobTx(0, 1, 1000, 100, 3, 0, key1) // blobs [0, 3)
pooledTx1, _ = newBlobTxForPool(tx1)
blob1, _ = rlp.EncodeToBytes(pooledTx1)
)
store.Put(blob1)
store.Close()
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
statedb.Commit(0, true, false)
chain := &testBlockChain{
config: params.MainnetChainConfig,
basefee: uint256.NewInt(params.InitialBaseFee),
blobfee: uint256.NewInt(params.BlobTxMinBlobGasprice),
statedb: statedb,
}
pool := New(Config{Datadir: storage}, chain, nil)
if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil {
t.Fatalf("failed to create blob pool: %v", err)
}
defer pool.Close()
tests := []struct {
name string
hash common.Hash
mask types.CustodyBitmap
expectedLen int
shouldFail bool
}{
{
name: "Get cells with single index",
hash: tx1.Hash(),
mask: types.NewCustodyBitmap([]uint64{5}),
expectedLen: 3,
shouldFail: false,
},
{
name: "Get cells with all indices",
hash: tx1.Hash(),
mask: types.CustodyBitmapAll,
expectedLen: 384,
shouldFail: false,
},
{
name: "Get cells with no indices",
hash: tx1.Hash(),
mask: types.CustodyBitmap{},
expectedLen: 0,
shouldFail: false,
},
{
name: "Get cells for non-existent transaction",
hash: common.Hash{0x01, 0x02, 0x03},
mask: types.NewCustodyBitmap([]uint64{0, 1}),
expectedLen: 0,
shouldFail: true,
},
{
name: "Get cells with random indices",
hash: tx1.Hash(),
mask: types.NewRandomCustodyBitmap(8),
expectedLen: 24,
shouldFail: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
vhashes := pool.GetBlobHashes(tt.hash)
if tt.shouldFail {
if vhashes != nil {
t.Errorf("expected nil vhashes for non-existent tx")
}
return
}
if vhashes == nil {
t.Fatalf("expected vhashes, got nil")
}
blobCells, _, err := pool.GetBlobCells(vhashes, tt.mask)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Count total non-nil cells across all blobs
totalCells := 0
for _, bc := range blobCells {
for _, c := range bc {
if c != nil {
totalCells++
}
}
}
if totalCells != tt.expectedLen {
t.Errorf("expected %d cells, got %d", tt.expectedLen, totalCells)
}
})
}
}

318
core/txpool/blobpool/buffer.go Normal file
View file

@ -0,0 +1,318 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package blobpool
import (
"cmp"
"fmt"
"slices"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
)
var (
blobBufferTxFirstCounter = metrics.NewRegisteredCounter("blobpool/buffer/txfirst", nil)
blobBufferCellsFirstCounter = metrics.NewRegisteredCounter("blobpool/buffer/cellsfirst", nil)
blobBufferTotalTx = metrics.NewRegisteredGauge("blobpool/buffer/txcount", nil)
blobBufferTotalCells = metrics.NewRegisteredGauge("blobpool/buffer/cellcount", nil)
)
const (
bufferLifetime = 2 * time.Minute
)
// PeerDelivery holds cells delivered by a single peer, in blob-major order.
type PeerDelivery struct {
Cells []kzg4844.Cell
Indices []uint64
}
type txEntry struct {
tx *types.Transaction
// Technically it is not required to store peer information to drop properly.
// This is mainly for per peer size limit check.
peer string
added time.Time
}
type cellEntry struct {
deliveries map[string]*PeerDelivery
custody types.CustodyBitmap
added time.Time
}
type BlobBuffer struct {
mu sync.Mutex
txs map[common.Hash]*txEntry
cells map[common.Hash]*cellEntry
completed []*BlobTxForPool
completedCount atomic.Int32
cb BlobBufferFunctions
}
type BlobBufferFunctions struct {
ValidateTx func(*types.Transaction) error
AddToPool func(*BlobTxForPool) error
DropPeer func(string)
}
func NewBlobBuffer(cb BlobBufferFunctions) *BlobBuffer {
return &BlobBuffer{
txs: make(map[common.Hash]*txEntry),
cells: make(map[common.Hash]*cellEntry),
cb: cb,
}
}
// Flush adds all completed entries to the pool and returns the hashes
// and corresponding errors (nil on success) for each attempted insert.
func (b *BlobBuffer) Flush() ([]common.Hash, []error) {
// Read the count first and return early if there is nothing to do.
// Flush is called very frequently from the blob fetcher so this
// optimization is warranted.
if b.completedCount.Load() == 0 {
return nil, nil
}
b.mu.Lock()
defer b.mu.Unlock()
txs := make([]common.Hash, len(b.completed))
errs := make([]error, len(b.completed))
for i, ptx := range b.completed {
txs[i] = ptx.Tx.Hash()
errs[i] = b.cb.AddToPool(ptx)
}
b.completed = nil
b.completedCount.Store(0)
return txs, errs
}
// AddTx buffers a blob transaction (without blobs) from an ETH/72 peer.
// If cells are already buffered, verification and pool insertion are attempted.
func (b *BlobBuffer) AddTx(txs []*types.Transaction, peer string) []error {
b.mu.Lock()
defer b.mu.Unlock()
defer b.updateMetrics()()
// First remove any timed-out entries.
b.evict()
errs := make([]error, len(txs))
for i, tx := range txs {
hash := tx.Hash()
sidecar := tx.BlobTxSidecar()
if sidecar == nil {
errs[i] = fmt.Errorf("blob transaction without sidecar")
continue
}
// tx validation (basic w/o lock)
// error will be handled by tx fetcher
if err := b.cb.ValidateTx(tx); err != nil {
errs[i] = err
continue
}
if entry, ok := b.cells[hash]; ok {
b.storeCompleted(hash, tx, entry)
continue
}
blobBufferTxFirstCounter.Inc(1)
b.txs[hash] = &txEntry{tx: tx, peer: peer, added: time.Now()}
}
return errs
}
// AddCells buffers per-peer cell deliveries from the blob fetcher.
// If the transaction is already buffered, verification and pool insertion are attempted.
func (b *BlobBuffer) AddCells(hash common.Hash, deliveries map[string]*PeerDelivery, custody types.CustodyBitmap) {
b.mu.Lock()
defer b.mu.Unlock()
defer b.updateMetrics()()
// First remove any timed-out entries.
b.evict()
b.cells[hash] = &cellEntry{
deliveries: deliveries,
custody: custody,
added: time.Now(),
}
if txe, ok := b.txs[hash]; ok {
b.storeCompleted(hash, txe.tx, b.cells[hash])
}
blobBufferCellsFirstCounter.Inc(1)
}
// storeCompleted verifies cells per-peer, sorts them, and schedules them for
// addition into the pool. The actual addition happens in Flush().
func (b *BlobBuffer) storeCompleted(hash common.Hash, tx *types.Transaction, cells *cellEntry) {
sidecar := tx.BlobTxSidecar()
// Per-peer cell verification
if badPeers := b.verifyCells(cells, sidecar); len(badPeers) > 0 {
b.dropPeers(badPeers)
delete(b.cells, hash)
delete(b.txs, hash)
return
}
blobCount := len(tx.BlobHashes())
sorted, custody := sortCells(cells, blobCount)
cellSidecar := types.BlobTxCellSidecar{
Version: sidecar.Version,
Commitments: sidecar.Commitments,
Proofs: sidecar.Proofs,
Cells: sorted,
Custody: custody,
}
pooledTx := &BlobTxForPool{
Tx: tx.WithoutBlobTxSidecar(),
CellSidecar: &cellSidecar,
}
b.completed = append(b.completed, pooledTx)
b.completedCount.Add(1)
delete(b.cells, hash)
delete(b.txs, hash)
}
func (b *BlobBuffer) HasTx(hash common.Hash) bool {
b.mu.Lock()
defer b.mu.Unlock()
_, ok := b.txs[hash]
return ok
}
func (b *BlobBuffer) HasCells(hash common.Hash) bool {
b.mu.Lock()
defer b.mu.Unlock()
_, ok := b.cells[hash]
return ok
}
func (b *BlobBuffer) dropPeers(peers []string) {
if b.cb.DropPeer == nil {
return
}
for _, p := range peers {
b.cb.DropPeer(p)
}
}
func (b *BlobBuffer) evict() {
now := time.Now()
for hash, entry := range b.txs {
if now.Sub(entry.added) > bufferLifetime {
delete(b.txs, hash)
}
}
for hash, entry := range b.cells {
if now.Sub(entry.added) > bufferLifetime {
delete(b.cells, hash)
}
}
}
// updateMetrics updates the metrics gauges.
// This should be called at the start of any operation that changes the buffer
// content. The returned function is to be called at the end of the operation,
// usually with defer.
func (b *BlobBuffer) updateMetrics() func() {
preTxCount := len(b.txs)
preCellsCount := len(b.cells)
return func() {
if len(b.txs) != preTxCount {
blobBufferTotalTx.Update(int64(len(b.txs)))
}
if len(b.cells) != preCellsCount {
blobBufferTotalCells.Update(int64(len(b.cells)))
}
}
}
// verifyCells verifies each peer's cells against the sidecar by treating each
// per-peer delivery as a mini BlobTxCellSidecar and reusing txpool.ValidateCells.
// Returns the list of peers whose cells failed verification.
func (b *BlobBuffer) verifyCells(entry *cellEntry, sidecar *types.BlobTxSidecar) []string {
var badPeers []string
for peer, delivery := range entry.deliveries {
perPeer := &types.BlobTxCellSidecar{
Version: sidecar.Version,
Cells: delivery.Cells,
Commitments: sidecar.Commitments,
Proofs: sidecar.Proofs,
Custody: types.NewCustodyBitmap(delivery.Indices),
}
if err := txpool.ValidateCells(perPeer); err != nil {
log.Debug("Cell verification failed", "peer", peer, "err", err)
badPeers = append(badPeers, peer)
}
}
return badPeers
}
// sortCells merges all per-peer deliveries into a single flat cell array
// sorted by custody index.
//
// e.g.
// peer A: cells = [blob0_cell5, blob0_cell3, blob1_cell5, blob1_cell3]
// peer B: cells = [blob0_cell1, blob0_cell7, blob1_cell1, blob1_cell7]
// -> [blob0_cell1, blob0_cell3, blob0_cell5, blob0_cell7, blob1_cell1, blob1_cell3, blob1_cell5, blob1_cell7]
func sortCells(entry *cellEntry, blobCount int) ([]kzg4844.Cell, types.CustodyBitmap) {
// indices per delivery
var indices []uint64
// 1. compose per blob cells
blob := make([][]kzg4844.Cell, blobCount)
for _, d := range entry.deliveries {
n := len(d.Indices)
indices = append(indices, d.Indices...)
for b := range blobCount {
blob[b] = append(blob[b], d.Cells[b*n:(b+1)*n]...)
}
}
// 2. sort
perm := make([]int, len(indices))
for i := range perm {
perm[i] = i
}
// perm represents the position of cells in sorted array
slices.SortFunc(perm, func(a, b int) int {
return cmp.Compare(indices[a], indices[b])
})
// reorder cells
var res []kzg4844.Cell
for b := range blobCount {
for _, p := range perm {
res = append(res, blob[b][p])
}
}
custody := types.NewCustodyBitmap(indices)
return res, custody
}

206
core/txpool/blobpool/buffer_test.go Normal file
View file

@ -0,0 +1,206 @@
package blobpool
import (
"crypto/ecdsa"
"testing"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
// makeV1Tx creates a V1 blob transaction with cell proofs, then strips blobs
// (simulating what ETH/72 peers send).
func makeV1Tx(t *testing.T, nonce uint64, blobCount int, blobOffset int, key *ecdsa.PrivateKey) *types.Transaction {
t.Helper()
tx := makeMultiBlobTx(nonce, 1, 1, 1, blobCount, blobOffset, key)
return removeBlobs(tx)
}
// makePeerDelivery creates a PeerDelivery for given cell indices from a set of blobs.
func makePeerDelivery(t *testing.T, blobOffset, blobCount int, indices []uint64) *PeerDelivery {
t.Helper()
var allCells []kzg4844.Cell
for i := 0; i < blobCount; i++ {
cells, err := kzg4844.ComputeCells([]kzg4844.Blob{*testBlobs[blobOffset+i]})
if err != nil {
t.Fatal(err)
}
allCells = append(allCells, cells...)
}
var deliveryCells []kzg4844.Cell
for b := 0; b < blobCount; b++ {
for _, idx := range indices {
deliveryCells = append(deliveryCells, allCells[b*kzg4844.CellsPerBlob+int(idx)])
}
}
return &PeerDelivery{Cells: deliveryCells, Indices: indices}
}
func newTestBuffer(t *testing.T) *BlobBuffer {
t.Helper()
return NewBlobBuffer(BlobBufferFunctions{
ValidateTx: func(tx *types.Transaction) error { return nil },
AddToPool: func(ptx *BlobTxForPool) error { return nil },
DropPeer: func(peer string) {},
})
}
func TestSortCells(t *testing.T) {
blobCount := 2
blobOffset := 0
peerA := makePeerDelivery(t, blobOffset, blobCount, []uint64{5, 3})
peerB := makePeerDelivery(t, blobOffset, blobCount, []uint64{1, 7})
custody := types.NewCustodyBitmap([]uint64{1, 3, 5, 7})
entry := &cellEntry{
deliveries: map[string]*PeerDelivery{
"peerA": peerA,
"peerB": peerB,
},
custody: custody,
}
sorted, resultCustody := sortCells(entry, blobCount)
resultIndices := resultCustody.Indices()
if len(resultIndices) != 4 {
t.Fatalf("expected 4 indices, got %d", len(resultIndices))
}
for i, expected := range []uint64{1, 3, 5, 7} {
if resultIndices[i] != expected {
t.Errorf("index %d: expected %d, got %d", i, expected, resultIndices[i])
}
}
expected := makePeerDelivery(t, blobOffset, blobCount, []uint64{1, 3, 5, 7})
if len(sorted) != len(expected.Cells) {
t.Fatalf("sorted length %d != expected %d", len(sorted), len(expected.Cells))
}
for i := range sorted {
if sorted[i] != expected.Cells[i] {
t.Errorf("cell %d mismatch", i)
}
}
}
func TestAddTxThenCells(t *testing.T) {
key, _ := crypto.GenerateKey()
blobCount := 2
buf := newTestBuffer(t)
tx := makeV1Tx(t, 0, blobCount, 0, key)
hash := tx.Hash()
if err := buf.AddTx([]*types.Transaction{tx}, "peerA")[0]; err != nil {
t.Fatal(err)
}
if !buf.HasTx(hash) {
t.Fatal("tx should be buffered")
}
dataIndices := make([]uint64, kzg4844.DataPerBlob)
for i := range dataIndices {
dataIndices[i] = uint64(i)
}
delivery := makePeerDelivery(t, 0, blobCount, dataIndices)
custody := types.NewCustodyBitmap(dataIndices)
buf.AddCells(hash, map[string]*PeerDelivery{"peerB": delivery}, custody)
if buf.HasTx(hash) || buf.HasCells(hash) {
t.Fatal("buffer should be empty after add")
}
}
func TestAddCellsThenTx(t *testing.T) {
key, _ := crypto.GenerateKey()
blobCount := 2
buf := newTestBuffer(t)
tx := makeV1Tx(t, 0, blobCount, 0, key)
hash := tx.Hash()
dataIndices := make([]uint64, kzg4844.DataPerBlob)
for i := range dataIndices {
dataIndices[i] = uint64(i)
}
delivery := makePeerDelivery(t, 0, blobCount, dataIndices)
custody := types.NewCustodyBitmap(dataIndices)
buf.AddCells(hash, map[string]*PeerDelivery{"peerB": delivery}, custody)
if !buf.HasCells(hash) {
t.Fatal("cells should be buffered")
}
if err := buf.AddTx([]*types.Transaction{tx}, "peerA")[0]; err != nil {
t.Fatal(err)
}
if buf.HasTx(hash) || buf.HasCells(hash) {
t.Fatal("buffer should be empty after add")
}
}
func TestMultiPeerDelivery(t *testing.T) {
key, _ := crypto.GenerateKey()
blobCount := 2
buf := newTestBuffer(t)
tx := makeV1Tx(t, 0, blobCount, 0, key)
hash := tx.Hash()
buf.AddTx([]*types.Transaction{tx}, "peerA")
indicesA := []uint64{0, 2, 4, 6}
indicesB := []uint64{1, 3, 5, 7}
deliveryA := makePeerDelivery(t, 0, blobCount, indicesA)
deliveryB := makePeerDelivery(t, 0, blobCount, indicesB)
allIndices := append(indicesA, indicesB...)
custody := types.NewCustodyBitmap(allIndices)
buf.AddCells(hash, map[string]*PeerDelivery{
"peerB": deliveryA,
"peerC": deliveryB,
}, custody)
if buf.HasTx(hash) || buf.HasCells(hash) {
t.Fatal("buffer should be empty after add")
}
}
func TestBadCell(t *testing.T) {
key, _ := crypto.GenerateKey()
blobCount := 1
var dropped []string
buf := NewBlobBuffer(BlobBufferFunctions{
ValidateTx: func(tx *types.Transaction) error { return nil },
AddToPool: func(ptx *BlobTxForPool) error { return nil },
DropPeer: func(peer string) { dropped = append(dropped, peer) },
})
tx := makeV1Tx(t, 0, blobCount, 0, key)
hash := tx.Hash()
buf.AddTx([]*types.Transaction{tx}, "peerA")
goodDelivery := makePeerDelivery(t, 0, blobCount, []uint64{0, 1, 2, 3})
badDelivery := makePeerDelivery(t, 0, blobCount, []uint64{4, 5, 6, 7})
for i := range badDelivery.Cells {
for j := range badDelivery.Cells[i] {
badDelivery.Cells[i][j] ^= 0xFF
}
}
allIndices := []uint64{0, 1, 2, 3, 4, 5, 6, 7}
custody := types.NewCustodyBitmap(allIndices)
buf.AddCells(hash, map[string]*PeerDelivery{
"peerB": goodDelivery,
"peerC": badDelivery,
}, custody)
if len(dropped) != 1 || dropped[0] != "peerC" {
t.Fatalf("only peerC should have been dropped, got: %v", dropped)
}
if buf.HasTx(hash) || buf.HasCells(hash) {
t.Fatal("buffer should be empty after bad cell drop")
}
}

233
core/txpool/blobpool/cache.go
View file

@ -36,7 +36,7 @@ import (
)
const (
topKTimeout = 4 * time.Second
topKTimeout = 1 * time.Second
hasBlobsTimeout = 1 * time.Second
)
@ -46,12 +46,14 @@ var (
// missing on the lower level (in this case, the blobpool). The amount that
// we failed to predict) can be calculated with the telemetry span
// (blobs.filled - cache.hit).
cacheHitMeter = metrics.NewRegisteredMeter("blobpool/cache/hit", nil)
cacheMissMeter = metrics.NewRegisteredMeter("blobpool/cache/miss", nil)
cacheBlobsGauge = metrics.NewRegisteredGauge("blobpool/cache/blobs", nil)
cacheHitMeter = metrics.NewRegisteredMeter("blobpool/cache/hit", nil)
cacheMissMeter = metrics.NewRegisteredMeter("blobpool/cache/miss", nil)
cacheEntriesGauge = metrics.NewRegisteredGauge("blobpool/cache/entries", nil)
)
type cachedBlob struct {
cell []kzg4844.Cell
custody types.CustodyBitmap
blob *kzg4844.Blob
commitment kzg4844.Commitment
proofs []kzg4844.Proof
@ -74,11 +76,16 @@ type Cache struct {
mu sync.Mutex
entries map[common.Hash]*cachedBlob
// needCell is owned by the loop goroutine; it is only read and written
// there, so it needs no synchronization. It is flipped on via enableCellCh.
needCell bool
// channels into loop
quit chan struct{}
topkRequest chan struct{}
topkTimer mclock.Timer
hasBlobsCh chan []common.Hash // list of tx hashes that should be pinned
cellModeCh chan bool // signals the loop to switch cell mode on/offo
step func() // test hook fired after each loop iteration
@ -103,6 +110,7 @@ func newCache(p *BlobPool, clock mclock.Clock, step func()) *Cache {
step: step,
quit: make(chan struct{}),
topkRequest: make(chan struct{}, 1),
cellModeCh: make(chan bool, 1),
}
c.wg.Add(1)
@ -197,13 +205,10 @@ func (c *Cache) GetBlobs(ctx context.Context, vhashes []common.Hash, version byt
c.mu.Lock()
for vhash, idxs := range indices {
n := len(idxs)
cached := c.entries[vhash]
if cached == nil || cached.version != version {
cached, ok := c.entries[vhash]
if !ok || cached.version != version || cached.blob == nil {
cacheMiss += n
if cached == nil {
misses = append(misses, vhash)
}
misses = append(misses, vhash)
continue
}
cacheHits += n
@ -241,6 +246,82 @@ func (c *Cache) GetBlobs(ctx context.Context, vhashes []common.Hash, version byt
return blobs, commitments, proofs, nil
}
// GetCells returns cells for the given versioned blob hashes, filtered
// by the requested cell indices (mask). Each entry in the result
// corresponds to one vhash. Nil entries mean the cell was not available.
// If the cell is not available in cache, it falls back to the blobpool.
func (c *Cache) GetCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error) {
var (
cells = make([][]*kzg4844.Cell, len(vhashes))
proofs = make([][]*kzg4844.Proof, len(vhashes))
indices = make(map[common.Hash][]int)
misses []common.Hash
)
for i, h := range vhashes {
indices[h] = append(indices[h], i)
}
requested := mask.Indices()
c.mu.Lock()
for vhash, idxs := range indices {
cached, ok := c.entries[vhash]
if !ok {
misses = append(misses, vhash)
continue
}
stored := cached.custody.Indices()
blobCells := make([]*kzg4844.Cell, len(requested))
blobProofs := make([]*kzg4844.Proof, len(requested))
for i, cellIdx := range requested {
pos := -1
for k, s := range stored {
if s == cellIdx {
pos = k
break
}
}
if pos >= 0 && pos < len(cached.cell) {
cell := cached.cell[pos]
blobCells[i] = &cell
if int(cellIdx) < len(cached.proofs) {
pf := cached.proofs[cellIdx]
blobProofs[i] = &pf
}
}
}
for _, idx := range idxs {
cells[idx] = blobCells
proofs[idx] = blobProofs
}
}
c.mu.Unlock()
if len(misses) > 0 {
mc, mp, err := c.blobpool.GetBlobCells(misses, mask)
if err != nil {
return nil, nil, err
}
for j, vhash := range misses {
for _, idx := range indices[vhash] {
cells[idx] = mc[j]
proofs[idx] = mp[j]
}
}
}
return cells, proofs, nil
}
// EnableCell allows the cache to store only cells without recovering
// blobs. This means we can also cache cells that lack enough blobs to
// recover. It signals the loop to switch to cell mode and re-select
// transactions from this wider pool.
func (c *Cache) SetCellMode(on bool) {
select {
case c.cellModeCh <- on:
case <-c.quit:
}
}
func (c *Cache) loop() {
defer c.wg.Done()
@ -258,6 +339,14 @@ func (c *Cache) loop() {
c.update(want)
c.triggerTopKAfter(topKTimeout)
case on := <-c.cellModeCh:
// This runs when the CL signals (non-)support for cell proofs. Enable/disable
// cell mode and re-select immediately to force an update.
if c.needCell != on {
c.needCell = on
c.triggerTopK()
}
case <-c.quit:
c.cancelUpdate()
if c.topkTimer != nil {
@ -285,6 +374,8 @@ func (c *Cache) cancelUpdate() {
// are no longer wanted and loads the missing ones from the blobpool in the
// background.
func (c *Cache) update(want []common.Hash) {
cellMode := c.needCell
wantSet := make(map[common.Hash]struct{}, len(want))
for _, vh := range want {
wantSet[vh] = struct{}{}
@ -298,7 +389,13 @@ func (c *Cache) update(want []common.Hash) {
c.mu.Lock()
var missing []common.Hash
for vh := range wantSet {
if _, ok := c.entries[vh]; !ok {
e, ok := c.entries[vh]
if ok && ((cellMode && e.cell == nil) || (!cellMode && e.blob == nil)) {
delete(c.entries, vh)
cacheEntriesGauge.Dec(1)
ok = false
}
if !ok {
missing = append(missing, vh)
}
}
@ -307,7 +404,7 @@ func (c *Cache) update(want []common.Hash) {
continue
}
delete(c.entries, vh)
cacheBlobsGauge.Dec(1)
cacheEntriesGauge.Dec(1)
}
c.mu.Unlock()
@ -330,37 +427,92 @@ func (c *Cache) update(want []common.Hash) {
if ptx == nil {
continue
}
sidecar := ptx.Sidecar()
if sidecar == nil {
continue
if cellMode {
c.loadCells(ptx, wantSet)
} else {
c.loadBlobs(ptx, wantSet)
}
c.mu.Lock()
for i, v := range sidecar.BlobHashes() {
if _, ok := wantSet[v]; !ok {
continue
}
if _, exists := c.entries[v]; exists {
continue // recompute only new entries
}
cellProofs, err := sidecar.CellProofsAt(i)
if err != nil {
log.Error("Failed to get cell proofs", "txhash", ptx.Tx.Hash(), "err", err)
continue
}
c.entries[v] = &cachedBlob{
blob: &sidecar.Blobs[i],
commitment: sidecar.Commitments[i],
proofs: cellProofs,
version: sidecar.Version,
}
cacheBlobsGauge.Inc(1)
}
c.mu.Unlock()
}
}()
}
// loadCells loads the cells whose vhash is in wantSet from ptx.
func (c *Cache) loadCells(ptx *BlobTxForPool, wantSet map[common.Hash]struct{}) {
cs := ptx.CellSidecar
cellsPerBlob := cs.Custody.OneCount()
c.mu.Lock()
defer c.mu.Unlock()
for i, v := range ptx.Tx.BlobHashes() {
if _, ok := wantSet[v]; !ok {
continue
}
if _, exists := c.entries[v]; exists {
continue
}
cellStart := i * cellsPerBlob
if cellStart+cellsPerBlob > len(cs.Cells) {
continue
}
blobCells := make([]kzg4844.Cell, cellsPerBlob)
copy(blobCells, cs.Cells[cellStart:cellStart+cellsPerBlob])
var pf []kzg4844.Proof
if ps := i * kzg4844.CellProofsPerBlob; ps+kzg4844.CellProofsPerBlob <= len(cs.Proofs) {
pf = make([]kzg4844.Proof, kzg4844.CellProofsPerBlob)
copy(pf, cs.Proofs[ps:ps+kzg4844.CellProofsPerBlob])
}
c.entries[v] = &cachedBlob{
cell: blobCells,
custody: cs.Custody,
commitment: cs.Commitments[i],
proofs: pf,
version: cs.Version,
}
cacheEntriesGauge.Inc(1)
}
}
// loadBlobs loads the blobs whose vhash is in wantSet from ptx.
func (c *Cache) loadBlobs(ptx *BlobTxForPool, wantSet map[common.Hash]struct{}) {
if ptx.CellSidecar.Custody.OneCount() < kzg4844.DataPerBlob {
return
}
// blobs will be computed inside of sidecar()
sidecar, err := ptx.sidecar()
if err != nil || sidecar == nil {
return
}
c.mu.Lock()
defer c.mu.Unlock()
for i, v := range sidecar.BlobHashes() {
if _, ok := wantSet[v]; !ok {
continue
}
if _, exists := c.entries[v]; exists {
continue
}
var pf []kzg4844.Proof
switch sidecar.Version {
case types.BlobSidecarVersion0:
pf = []kzg4844.Proof{sidecar.Proofs[i]}
case types.BlobSidecarVersion1:
cellProofs, err := sidecar.CellProofsAt(i)
if err != nil {
log.Error("Failed to get cell proofs", "txhash", ptx.Tx.Hash(), "err", err)
continue
}
pf = cellProofs
}
c.entries[v] = &cachedBlob{
blob: &sidecar.Blobs[i],
commitment: sidecar.Commitments[i],
proofs: pf,
version: sidecar.Version,
}
cacheEntriesGauge.Inc(1)
}
}
// selectTopTxs returns the vhashes of the top K most profitable pending blob
// transactions, up to the active fork's maxBlobsPerBlock.
func (c *Cache) selectTopTxs() []common.Hash {
@ -373,8 +525,9 @@ func (c *Cache) selectTopTxs() []common.Hash {
baseFee := eip1559.CalcBaseFee(config, head)
filter := txpool.PendingFilter{
BlobTxs: true,
BaseFee: uint256.MustFromBig(baseFee),
BlobTxs: true,
BaseFee: uint256.MustFromBig(baseFee),
PartialCells: c.needCell,
}
if head.ExcessBlobGas != nil {
filter.BlobFee = uint256.MustFromBig(eip4844.CalcBlobFee(config, head))

7
core/txpool/blobpool/cache_test.go
View file

@ -146,7 +146,10 @@ func (tc *testCache) inject(t *testing.T, spec txSpec) []common.Hash {
tx := makeMultiBlobTx(0, spec.tip, 1_000_000, 1_000_000, spec.blobs, tc.offset, key)
tc.offset += spec.blobs
ptx := newBlobTxForPool(tx)
ptx, err := newBlobTxForPool(tx)
if err != nil {
t.Fatalf("new blob tx for pool: %v", err)
}
tc.blobpool.lock.Lock()
defer tc.blobpool.lock.Unlock()
@ -155,7 +158,7 @@ func (tc *testCache) inject(t *testing.T, spec txSpec) []common.Hash {
if err != nil {
t.Fatalf("store put: %v", err)
}
meta := newBlobTxMeta(id, ptx.TxSize(), tc.blobpool.store.Size(id), ptx)
meta := newBlobTxMeta(id, tc.blobpool.store.Size(id), ptx)
addr := crypto.PubkeyToAddress(key.PublicKey)
tc.blobpool.index[addr] = append(tc.blobpool.index[addr], meta)
tc.blobpool.lookup.track(meta)

2
core/txpool/blobpool/config.go
View file

@ -25,6 +25,8 @@ type Config struct {
Datadir string // Data directory containing the currently executable blobs
Datacap uint64 // Soft-cap of database storage (hard cap is larger due to overhead)
PriceBump uint64 // Minimum price bump percentage to replace an already existing nonce
FetchProbability uint64 // EIP-8070: full blob fetch probability for sparse blobpool
}
// DefaultConfig contains the default configurations for the transaction pool.

207
core/txpool/blobpool/conversion.go Normal file
View file

@ -0,0 +1,207 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package blobpool
import (
"errors"
"slices"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/log"
)
// maxPendingConversionTasks caps the number of pending conversion tasks. This
// prevents excessive memory usage; the worst-case scenario (2k transactions
// with 6 blobs each) would consume approximately 1.5GB of memory.
const maxPendingConversionTasks = 2048
type convertResult struct {
ptx *BlobTxForPool
err error
}
// txConvert represents a conversion task with an attached legacy blob transaction.
type txConvert struct {
tx *types.Transaction // Legacy blob transaction
done chan convertResult // Channel for signaling back if the conversion succeeds
}
// conversionQueue is a dedicated queue for converting legacy blob transactions
// received from the network after the Osaka fork. Since conversion is expensive,
// it is performed in the background by a single thread, ensuring the main Geth
// process is not overloaded.
type conversionQueue struct {
tasks chan *txConvert
startConversion chan func()
quit chan struct{}
closed chan struct{}
queue []func()
taskDone chan struct{}
}
// newConversionQueue constructs the conversion queue.
func newConversionQueue() *conversionQueue {
q := &conversionQueue{
tasks: make(chan *txConvert),
startConversion: make(chan func()),
quit: make(chan struct{}),
closed: make(chan struct{}),
}
go q.loop()
return q
}
// convert accepts a legacy blob transaction with version-0 blobs and queues it
// for conversion.
//
// This function may block for a long time until the transaction is processed.
func (q *conversionQueue) convert(tx *types.Transaction) (*BlobTxForPool, error) {
done := make(chan convertResult, 1)
select {
case q.tasks <- &txConvert{tx: tx, done: done}:
res := <-done
return res.ptx, res.err
case <-q.closed:
return nil, errors.New("conversion queue closed")
}
}
// launchConversion starts a conversion task in the background.
func (q *conversionQueue) launchConversion(fn func()) error {
select {
case q.startConversion <- fn:
return nil
case <-q.closed:
return errors.New("conversion queue closed")
}
}
// close terminates the conversion queue.
func (q *conversionQueue) close() {
select {
case <-q.closed:
return
default:
close(q.quit)
<-q.closed
}
}
// run converts a batch of legacy blob txs to the new cell proof format.
func (q *conversionQueue) run(tasks []*txConvert, done chan struct{}, interrupt *atomic.Int32) {
defer close(done)
for _, t := range tasks {
if interrupt != nil && interrupt.Load() != 0 {
t.done <- convertResult{err: errors.New("conversion is interrupted")}
continue
}
// Run the conversion, the original sidecar will be mutated in place
start := time.Now()
ptx, err := newBlobTxForPool(t.tx)
t.done <- convertResult{ptx: ptx, err: err}
log.Trace("Converted legacy blob tx", "hash", t.tx.Hash(), "err", err, "elapsed", common.PrettyDuration(time.Since(start)))
}
}
func (q *conversionQueue) loop() {
defer close(q.closed)
var (
done chan struct{} // Non-nil if background routine is active
interrupt *atomic.Int32 // Flag to signal conversion interruption
// The pending tasks for sidecar conversion. We assume the number of legacy
// blob transactions requiring conversion will not be excessive. However,
// a hard cap is applied as a protective measure.
txTasks []*txConvert
)
for {
select {
case t := <-q.tasks:
if len(txTasks) >= maxPendingConversionTasks {
t.done <- convertResult{err: errors.New("conversion queue is overloaded")}
continue
}
txTasks = append(txTasks, t)
// Launch the background conversion thread if it's idle
if done == nil {
done, interrupt = make(chan struct{}), new(atomic.Int32)
tasks := slices.Clone(txTasks)
txTasks = txTasks[:0]
go q.run(tasks, done, interrupt)
}
case <-done:
done, interrupt = nil, nil
if len(txTasks) > 0 {
done, interrupt = make(chan struct{}), new(atomic.Int32)
tasks := slices.Clone(txTasks)
txTasks = txTasks[:0]
go q.run(tasks, done, interrupt)
}
case fn := <-q.startConversion:
q.queue = append(q.queue, fn)
q.runNextTask()
case <-q.taskDone:
q.runNextTask()
case <-q.quit:
if done != nil {
log.Debug("Waiting for blob proof conversion to exit")
interrupt.Store(1)
<-done
}
if q.taskDone != nil {
log.Debug("Waiting for blobpool billy conversion to exit")
<-q.taskDone
}
// Signal any tasks that were queued for the next batch but never started
// so callers blocked in convert() receive an error instead of hanging.
for _, t := range txTasks {
// Best-effort notify; t.done is a buffered channel of size 1
// created by convert(), and we send exactly once per task.
t.done <- convertResult{err: errors.New("conversion queue closed")}
}
// Drop references to allow GC of the backing array.
txTasks = txTasks[:0]
return
}
}
}
func (q *conversionQueue) runNextTask() {
if len(q.queue) == 0 {
q.taskDone = nil
return
}
fn := q.queue[0]
q.queue = append(q.queue[:0], q.queue[1:]...)
done := make(chan struct{})
go func() { defer close(done); fn() }()
q.taskDone = done
}

114
core/txpool/blobpool/conversion_test.go Normal file
View file

@ -0,0 +1,114 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package blobpool
import (
"sync"
"testing"
"time"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/params"
)
func TestConversionQueueBasic(t *testing.T) {
queue := newConversionQueue()
defer queue.close()
key, _ := crypto.GenerateKey()
tx := makeTx(0, 1, 1, 1, key)
ptx, err := queue.convert(tx)
if err != nil {
t.Fatalf("Expected successful conversion, got error: %v", err)
}
if ptx == nil {
t.Fatal("Expected a converted transaction, got nil")
}
if ptx.Tx.Hash() != tx.Hash() {
t.Errorf("Converted tx hash mismatch: have %s, want %s", ptx.Tx.Hash(), tx.Hash())
}
if len(ptx.CellSidecar.Cells) == 0 {
t.Error("Expected cells to be computed during conversion")
}
}
func TestConversionQueueClosed(t *testing.T) {
queue := newConversionQueue()
queue.close()
key, _ := crypto.GenerateKey()
tx := makeTx(0, 1, 1, 1, key)
if _, err := queue.convert(tx); err == nil {
t.Fatal("Expected error when converting on closed queue, got nil")
}
}
func TestConversionQueueDoubleClose(t *testing.T) {
queue := newConversionQueue()
queue.close()
queue.close() // Should not panic
}
func TestConversionQueueAutoRestartBatch(t *testing.T) {
queue := newConversionQueue()
defer queue.close()
key, _ := crypto.GenerateKey()
// Create a heavy transaction to ensure the first batch runs long enough
// for subsequent tasks to be queued while it is active.
heavy := makeMultiBlobTx(0, 1, 1, 1, int(params.BlobTxMaxBlobs), 0, key)
var wg sync.WaitGroup
wg.Add(1)
heavyDone := make(chan error, 1)
go func() {
defer wg.Done()
_, err := queue.convert(heavy)
heavyDone <- err
}()
// Give the conversion worker a head start so that the following tasks are
// enqueued while the first batch is running.
time.Sleep(200 * time.Millisecond)
tx1 := makeTx(1, 1, 1, 1, key)
tx2 := makeTx(2, 1, 1, 1, key)
wg.Add(2)
done1 := make(chan error, 1)
done2 := make(chan error, 1)
go func() { defer wg.Done(); _, err := queue.convert(tx1); done1 <- err }()
go func() { defer wg.Done(); _, err := queue.convert(tx2); done2 <- err }()
for _, c := range []struct {
name string
done chan error
}{{"tx1", done1}, {"tx2", done2}, {"heavy", heavyDone}} {
select {
case err := <-c.done:
if err != nil {
t.Fatalf("%s conversion error: %v", c.name, err)
}
case <-time.After(30 * time.Second):
t.Fatalf("timeout waiting for %s conversion", c.name)
}
}
wg.Wait()
}

60
core/txpool/blobpool/limbo.go
View file

@ -33,7 +33,7 @@ import (
type limboBlob struct {
TxHash common.Hash // Owner transaction's hash to support resurrecting reorged txs
Block uint64 // Block in which the blob transaction was included
Ptx *blobTxForPool
Ptx *BlobTxForPool
}
// limbo is a light, indexed database to temporarily store recently included
@ -49,7 +49,7 @@ type limbo struct {
}
// newLimbo opens and indexes a set of limboed blob transactions.
func newLimbo(config *params.ChainConfig, datadir string) (*limbo, error) {
func newLimbo(config *params.ChainConfig, datadir string) (*limbo, []uint64, error) {
l := &limbo{
index: make(map[common.Hash]uint64),
groups: make(map[uint64]map[uint64]common.Hash),
@ -61,7 +61,7 @@ func newLimbo(config *params.ChainConfig, datadir string) (*limbo, error) {
// See if we need to migrate the limbo after fusaka.
slotter, err := tryMigrate(config, slotter, datadir)
if err != nil {
return nil, err
return nil, nil, err
}
// Index all limboed blobs on disk and delete anything unprocessable
@ -71,62 +71,30 @@ func newLimbo(config *params.ChainConfig, datadir string) (*limbo, error) {
)
index := func(id uint64, size uint32, data []byte) {
err := l.parseBlob(id, data)
if err != nil {
fails = append(fails, id)
}
if errors.Is(err, errLegacyTx) {
convert = append(convert, id)
return
}
if err != nil {
fails = append(fails, id)
}
}
store, err := billy.Open(billy.Options{Path: datadir, Repair: true}, slotter, index)
if err != nil {
return nil, err
return nil, nil, err
}
l.store = store
// Migrate legacy limbo entries to blobTxForPool. Note all ids in `converted` are also
// in `fails`, and the converted entries will be deleted by the loop that handles
// `fails`.
for _, id := range convert {
data, err := l.store.Get(id)
if err != nil {
continue
}
var legacy struct {
TxHash common.Hash
Block uint64
Tx *types.Transaction
}
if err := rlp.DecodeBytes(data, &legacy); err != nil {
continue
}
blob, err := rlp.EncodeToBytes(&limboBlob{
TxHash: legacy.TxHash,
Block: legacy.Block,
Ptx: newBlobTxForPool(legacy.Tx),
})
if err != nil {
continue
}
newID, err := l.store.Put(blob)
if err != nil {
continue
}
if err := l.parseBlob(newID, blob); err != nil {
fails = append(fails, newID)
}
}
if len(fails) > 0 {
log.Warn("Dropping invalidated limboed blobs", "ids", fails)
for _, id := range fails {
if err := l.store.Delete(id); err != nil {
l.Close()
return nil, err
return nil, nil, err
}
}
}
return l, nil
return l, convert, nil
}
// Close closes down the underlying persistent store.
@ -201,7 +169,9 @@ func (l *limbo) finalize(final *types.Header) {
// push stores a new blob transaction into the limbo, waiting until finality for
// it to be automatically evicted.
func (l *limbo) push(ptx *blobTxForPool, block uint64) error {
func (l *limbo) push(ptx *BlobTxForPool, block uint64) error {
// If the blobs are already tracked by the limbo, consider it a programming
// error. There's not much to do against it, but be loud.
hash := ptx.Tx.Hash()
if _, ok := l.index[hash]; ok {
log.Error("Limbo cannot push already tracked blobs", "tx", hash)
@ -217,7 +187,7 @@ func (l *limbo) push(ptx *blobTxForPool, block uint64) error {
// pull retrieves a previously pushed set of blobs back from the limbo, removing
// it at the same time. This method should be used when a previously included blob
// transaction gets reorged out.
func (l *limbo) pull(tx common.Hash) (*blobTxForPool, error) {
func (l *limbo) pull(tx common.Hash) (*BlobTxForPool, error) {
// If the blobs are not tracked by the limbo, there's not much to do. This
// can happen for example if a blob transaction is mined without pushing it
// into the network first.
@ -294,7 +264,7 @@ func (l *limbo) getAndDrop(id uint64) (*limboBlob, error) {
// setAndIndex assembles a limbo blob database entry and stores it, also updating
// the in-memory indices.
func (l *limbo) setAndIndex(ptx *blobTxForPool, block uint64) error {
func (l *limbo) setAndIndex(ptx *BlobTxForPool, block uint64) error {
txhash := ptx.Tx.Hash()
item := &limboBlob{
TxHash: txhash,

23
core/txpool/blobpool/limbo_test.go
View file

@ -1,4 +1,4 @@
// Copyright 2026 The go-ethereum Authors
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@ -27,8 +27,9 @@ import (
"github.com/holiman/billy"
)
// TestLimboLegacyMigration checks whether limbo entries in the legacy limboBlob type
// are migrated to the blobTxForPool layout on startup instead of being dropped.
// TestLimboLegacyMigration checks that a limbo entry in the legacy limboBlob
// layout is flagged for conversion by newLimbo instead of being dropped, so the
// conversion queue can migrate it to the blobTxForPool layout.
func TestLimboLegacyMigration(t *testing.T) {
key, _ := crypto.GenerateKey()
tx := makeMultiBlobTx(0, 10, 100, 100, 2, 0, key)
@ -49,24 +50,20 @@ func TestLimboLegacyMigration(t *testing.T) {
if err != nil {
t.Fatalf("failed to encode legacy entry: %v", err)
}
if _, err := store.Put(data); err != nil {
id, err := store.Put(data)
if err != nil {
t.Fatalf("failed to store legacy entry: %v", err)
}
store.Close()
// Open the limbo, which should migrate the legacy entry.
l, err := newLimbo(new(params.ChainConfig), dir)
// Open the limbo, which should flag the legacy entry for conversion.
l, convert, err := newLimbo(new(params.ChainConfig), dir)
if err != nil {
t.Fatalf("failed to open limbo: %v", err)
}
defer l.Close()
// The migrated transaction must be tracked and reconstruct the original.
ptx, err := l.pull(tx.Hash())
if err != nil {
t.Fatalf("failed to pull migrated tx: %v", err)
}
if got := ptx.ToTx().Hash(); got != tx.Hash() {
t.Fatalf("migrated tx hash mismatch: got %x, want %x", got, tx.Hash())
if len(convert) != 1 || convert[0] != id {
t.Fatalf("legacy entry not flagged for conversion: got %v, want [%d]", convert, id)
}
}

24
core/txpool/blobpool/lookup.go
View file

@ -18,11 +18,15 @@ package blobpool
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
type txMetadata struct {
id uint64 // the billy id of transaction
size uint64 // the RLP encoded size of transaction (blobs are included)
id uint64 // the billy id of transaction
size uint64 // the RLP encoded size of transaction (blobs are included)
sizeWithoutBlob uint64 // the RLP encoded size without blob data (for ETH/72 announcements)
custody types.CustodyBitmap
vhashes []common.Hash // blob versioned hashes for the transaction
}
// lookup maps blob versioned hashes to transaction hashes that include them,
@ -56,6 +60,15 @@ func (l *lookup) storeidOfTx(txhash common.Hash) (uint64, bool) {
return meta.id, true
}
// blobHashesOfTx returns the blob versioned hashes for a transaction.
func (l *lookup) blobHashesOfTx(txhash common.Hash) ([]common.Hash, bool) {
meta, ok := l.txIndex[txhash]
if !ok {
return nil, false
}
return meta.vhashes, true
}
// storeidOfBlob returns the datastore storage item id of a blob.
func (l *lookup) storeidOfBlob(vhash common.Hash) (uint64, bool) {
// If the blob is unknown, return a miss
@ -91,8 +104,11 @@ func (l *lookup) track(tx *blobTxMeta) {
}
// Map the transaction hash to the datastore id and RLP-encoded transaction size
l.txIndex[tx.hash] = &txMetadata{
id: tx.id,
size: tx.size,
id: tx.id,
size: tx.size,
sizeWithoutBlob: tx.sizeWithoutBlob,
custody: *tx.custody,
vhashes: tx.vhashes,
}
}

3
core/txpool/errors.go
View file

@ -74,4 +74,7 @@ var (
// ErrKZGVerificationError is returned when a KZG proof was not verified correctly.
ErrKZGVerificationError = errors.New("KZG verification error")
// ErrSidecarFormatError is returned when sidecar is malformed
ErrSidecarFormatError = errors.New("Wrong sidecar format")
)

2
core/txpool/legacypool/legacypool.go
View file

@ -1010,7 +1010,7 @@ func (pool *LegacyPool) get(hash common.Hash) *types.Transaction {
}
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
func (pool *LegacyPool) GetRLP(hash common.Hash) []byte {
func (pool *LegacyPool) GetRLP(hash common.Hash, _ uint) []byte {
tx := pool.all.Get(hash)
if tx == nil {
return nil

12
core/txpool/subpool.go
View file

@ -80,14 +80,16 @@ type PendingFilter struct {
// When BlobTxs true, return only blob transactions (block blob-space filling)
// when false, return only non-blob txs (peer-join announces, block space filling)
BlobTxs bool
BlobVersion byte // Blob tx version to include. 0 means pre-Osaka, 1 means Osaka and later
BlobTxs bool
PartialCells bool
BlobVersion byte // Blob tx version to include. 0 means pre-Osaka, 1 means Osaka and later
}
// TxMetadata denotes the metadata of a transaction.
type TxMetadata struct {
Type uint8 // The type of the transaction
Size uint64 // The length of the 'rlp encoding' of a transaction
Type uint8 // The type of the transaction
Size uint64 // The length of the 'rlp encoding' of a transaction (including blobs)
SizeWithoutBlob uint64 // The length without blob data (for ETH/72 announcements)
}
// SubPool represents a specialized transaction pool that lives on its own (e.g.
@ -132,7 +134,7 @@ type SubPool interface {
Get(hash common.Hash) *types.Transaction
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
GetRLP(hash common.Hash) []byte
GetRLP(hash common.Hash, version uint) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.

4
core/txpool/txpool.go
View file

@ -288,9 +288,9 @@ func (p *TxPool) Get(hash common.Hash) *types.Transaction {
}
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
func (p *TxPool) GetRLP(hash common.Hash) []byte {
func (p *TxPool) GetRLP(hash common.Hash, version uint) []byte {
for _, subpool := range p.subpools {
encoded := subpool.GetRLP(hash)
encoded := subpool.GetRLP(hash, version)
if len(encoded) != 0 {
return encoded
}

84
core/txpool/validation.go
View file

@ -65,9 +65,6 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
if opts.Accept&(1<<tx.Type()) == 0 {
return fmt.Errorf("%w: tx type %v not supported by this pool", core.ErrTxTypeNotSupported, tx.Type())
}
if blobCount := len(tx.BlobHashes()); blobCount > opts.MaxBlobCount {
return fmt.Errorf("%w: blob count %v, limit %v", ErrTxBlobLimitExceeded, blobCount, opts.MaxBlobCount)
}
// Before performing any expensive validations, sanity check that the tx is
// smaller than the maximum limit the pool can meaningfully handle
if tx.Size() > opts.MaxSize {
@ -161,7 +158,7 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.GasTipCap(), opts.MinTip)
}
if tx.Type() == types.BlobTxType {
return validateBlobTx(tx)
return validateBlobSidecar(tx, head, opts)
}
if tx.Type() == types.SetCodeTxType {
if len(tx.SetCodeAuthorizations()) == 0 {
@ -171,42 +168,67 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
return nil
}
// validateBlobTx implements the blob-transaction specific validations.
func validateBlobTx(tx *types.Transaction) error {
// validateBlobSidecar implements the blob sidecar validation.
// Note that this doesn't verify the consistency between blobs(cells) and
// proofs. For proof verification, use validateCells.
func validateBlobSidecar(tx *types.Transaction, head *types.Header, opts *ValidationOptions) error {
if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
}
sidecar := tx.BlobTxSidecar()
if sidecar == nil {
return errors.New("missing sidecar in blob transaction")
}
// Ensure the sidecar is constructed with the correct version
if sidecar.Version != types.BlobSidecarVersion1 {
return fmt.Errorf("unexpected sidecar version, want: %d, got: %d", types.BlobSidecarVersion1, sidecar.Version)
}
// Ensure the blob fee cap satisfies the minimum blob gas price
if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
}
// Ensure the number of items in the blob transaction and various side
// data match up before doing any expensive validations
hashes := tx.BlobHashes()
if len(hashes) == 0 {
return errors.New("blobless blob transaction")
}
if len(hashes) > params.BlobTxMaxBlobs {
return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.BlobTxMaxBlobs)
}
if len(sidecar.Blobs) != len(hashes) {
return fmt.Errorf("invalid number of %d blobs compared to %d blob hashes", len(sidecar.Blobs), len(hashes))
}
if err := sidecar.ValidateBlobCommitmentHashes(hashes); err != nil {
return err
}
return validateBlobSidecarOsaka(sidecar, hashes)
}
func validateBlobSidecarOsaka(sidecar *types.BlobTxSidecar, hashes []common.Hash) error {
if len(sidecar.Proofs) != len(hashes)*kzg4844.CellProofsPerBlob {
return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes)*kzg4844.CellProofsPerBlob)
if len(hashes) > opts.MaxBlobCount {
return fmt.Errorf("%w: blob count %v, limit %v", ErrTxBlobLimitExceeded, len(hashes), opts.MaxBlobCount)
}
if err := kzg4844.VerifyCellProofs(sidecar.Blobs, sidecar.Commitments, sidecar.Proofs); err != nil {
if sidecar.Version != types.BlobSidecarVersion1 {
return fmt.Errorf("%w: unexpected sidecar version, want: %d, got: %d", ErrSidecarFormatError, types.BlobSidecarVersion1, sidecar.Version)
}
if len(sidecar.Proofs) != len(sidecar.Commitments)*kzg4844.CellProofsPerBlob {
return fmt.Errorf("%w: invalid number of %d blob proofs expected %d", ErrSidecarFormatError, len(sidecar.Proofs), len(sidecar.Commitments)*kzg4844.CellProofsPerBlob)
}
return nil
}
func ValidateCells(sidecar *types.BlobTxCellSidecar) error {
// Two checks here (custody count check and blobCount check) is duplicated in buffer.go
// However it is required to 1) serve eth71 peer and direct submission 2) catch any bug in
// merging cell delivery.
if sidecar.Custody.OneCount() == 0 {
return errors.New("blobless blob transaction")
}
// Verify whether the blob count is consistent with other parts of the sidecar and the transaction
blobCount := len(sidecar.Cells) / sidecar.Custody.OneCount()
if blobCount == 0 {
return errors.New("blobless blob transaction")
}
if blobCount != len(sidecar.Commitments) {
return fmt.Errorf("invalid number of %d blobs compared to %d commitments", blobCount, len(sidecar.Commitments))
}
if len(sidecar.Proofs) != len(sidecar.Commitments)*kzg4844.CellProofsPerBlob {
return fmt.Errorf("invalid number of %d proofs compared to %d commitments", len(sidecar.Proofs), len(sidecar.Commitments))
}
if sidecar.Version != types.BlobSidecarVersion1 {
return fmt.Errorf("unexpected sidecar version, want: %d, got: %d", types.BlobSidecarVersion1, sidecar.Version)
}
return validateCellsOsaka(sidecar)
}
func validateCellsOsaka(sidecar *types.BlobTxCellSidecar) error {
indices := sidecar.Custody.Indices()
cellProofs := make([]kzg4844.Proof, 0)
for blobIdx := range len(sidecar.Commitments) {
for _, proofIdx := range indices {
idx := blobIdx*kzg4844.CellProofsPerBlob + int(proofIdx)
cellProofs = append(cellProofs, sidecar.Proofs[idx])
}
}
if err := kzg4844.VerifyCells(sidecar.Cells, sidecar.Commitments, cellProofs, sidecar.Custody.Indices()); err != nil {
return fmt.Errorf("%w: %v", ErrKZGVerificationError, err)
}
return nil

152
core/types/custody_bitmap.go Normal file
View file

@ -0,0 +1,152 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package types
import (
"fmt"
"math/bits"
"math/rand"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
// CustodyBitmap is a bitmap to represent which custody index to store (little endian).
type CustodyBitmap [16]byte
// MarshalText implements encoding.TextMarshaler.
func (b CustodyBitmap) MarshalText() ([]byte, error) {
return []byte(hexutil.Encode(b[:])), nil
}
// UnmarshalText implements encoding.TextUnmarshaler.
func (b *CustodyBitmap) UnmarshalText(input []byte) error {
decoded, err := hexutil.Decode(string(input))
if err != nil {
return fmt.Errorf("custody bitmap: %v", err)
}
if len(decoded) != len(b) {
return fmt.Errorf("custody bitmap: invalid length %d, want %d", len(decoded), len(b))
}
copy(b[:], decoded)
return nil
}
var (
CustodyBitmapAll = func() CustodyBitmap {
var result CustodyBitmap
for i := range result {
result[i] = 0xFF
}
return result
}()
CustodyBitmapData = func() CustodyBitmap {
var result CustodyBitmap
for i := 0; i < kzg4844.DataPerBlob/8; i++ {
result[i] = 0xFF
}
return result
}()
)
func NewCustodyBitmap(indices []uint64) CustodyBitmap {
var result CustodyBitmap
for _, i := range indices {
if i >= uint64(kzg4844.CellsPerBlob) {
panic("CustodyBitmap: bit index out of range")
}
result[i/8] |= 1 << (i % 8)
}
return result
}
// NewRandomCustodyBitmap creates a CustodyBitmap with n randomly selected indices.
// This should be used only for tests.
func NewRandomCustodyBitmap(n int) CustodyBitmap {
if n <= 0 || n > kzg4844.CellsPerBlob {
panic("CustodyBitmap: invalid number of indices")
}
indices := make([]uint64, 0, n)
used := make(map[uint64]bool)
for len(indices) < n {
idx := uint64(rand.Intn(kzg4844.CellsPerBlob))
if !used[idx] {
used[idx] = true
indices = append(indices, idx)
}
}
return NewCustodyBitmap(indices)
}
// IsSet returns whether bit i is set.
func (b CustodyBitmap) IsSet(i uint64) bool {
if i >= uint64(kzg4844.CellsPerBlob) {
return false
}
return (b[i/8]>>(i%8))&1 == 1
}
// OneCount returns the number of bits set to 1.
func (b CustodyBitmap) OneCount() int {
total := 0
for _, v := range b {
total += bits.OnesCount8(v)
}
return total
}
// Indices returns the bit positions set to 1, in ascending order.
func (b CustodyBitmap) Indices() []uint64 {
out := make([]uint64, 0, b.OneCount())
for byteIdx, val := range b {
v := val
for v != 0 {
tz := bits.TrailingZeros8(v)
out = append(out, uint64(byteIdx*8+tz))
v &^= 1 << tz
}
}
return out
}
// Difference returns b AND NOT set (bits in b but not in set).
func (b CustodyBitmap) Difference(set CustodyBitmap) CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] &^ set[i]
}
return out
}
// Intersection returns b AND set.
func (b CustodyBitmap) Intersection(set CustodyBitmap) CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] & set[i]
}
return out
}
// Union returns b OR set.
func (b CustodyBitmap) Union(set CustodyBitmap) CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] | set[i]
}
return out
}

10
core/types/tx_blob.go
View file

@ -176,6 +176,16 @@ func (sc *BlobTxSidecar) Copy() *BlobTxSidecar {
}
}
// BlobTxCellSidecar is a sidecar that carries cells instead of blobs.
// The Custody field represents which cells of each blob this sidecar contains.
type BlobTxCellSidecar struct {
Version byte
Cells []kzg4844.Cell
Commitments []kzg4844.Commitment
Proofs []kzg4844.Proof
Custody CustodyBitmap
}
// blobTxWithBlobs represents blob tx with its corresponding sidecar.
// This is an interface because sidecars are versioned.
type blobTxWithBlobs interface {

22
eth/backend.go
View file

@ -44,6 +44,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/fetcher"
"github.com/ethereum/go-ethereum/eth/gasprice"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/eth/protocols/snap"
@ -349,15 +350,17 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
// Permit the downloader to use the trie cache allowance during fast sync
cacheLimit := options.TrieCleanLimit + options.TrieDirtyLimit + options.SnapshotLimit
if eth.handler, err = newHandler(&handlerConfig{
NodeID: eth.p2pServer.Self().ID(),
Database: chainDb,
Chain: eth.blockchain,
TxPool: eth.txPool,
Network: networkID,
Sync: config.SyncMode,
BloomCache: uint64(cacheLimit),
RequiredBlocks: config.RequiredBlocks,
SnapV2: config.SnapV2,
NodeID: eth.p2pServer.Self().ID(),
Database: chainDb,
Chain: eth.blockchain,
TxPool: eth.txPool,
BlobPool: eth.blobTxPool,
Network: networkID,
Sync: config.SyncMode,
BloomCache: uint64(cacheLimit),
RequiredBlocks: config.RequiredBlocks,
SnapV2: config.SnapV2,
FetchProbability: config.BlobPool.FetchProbability,
}); err != nil {
return nil, err
}
@ -441,6 +444,7 @@ func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager
func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain }
func (s *Ethereum) TxPool() *txpool.TxPool { return s.txPool }
func (s *Ethereum) BlobTxPool() *blobpool.BlobPool { return s.blobTxPool }
func (s *Ethereum) BlobFetcher() *fetcher.BlobFetcher { return s.handler.blobFetcher }
func (s *Ethereum) BlobCache() *blobpool.Cache { return s.blobCache }
func (s *Ethereum) Engine() consensus.Engine { return s.engine }
func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb }

80
eth/catalyst/api.go
View file

@ -22,6 +22,7 @@ import (
"errors"
"fmt"
"reflect"
"slices"
"strconv"
"sync"
"sync/atomic"
@ -219,7 +220,7 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV3(ctx context.Context, update engine.
// ForkchoiceUpdatedV4 is equivalent to V3 with the addition of slot number
// in the payload attributes. It supports only PayloadAttributesV4.
func (api *ConsensusAPI) ForkchoiceUpdatedV4(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) {
func (api *ConsensusAPI) ForkchoiceUpdatedV4(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes, custodyColumns *types.CustodyBitmap) (engine.ForkChoiceResponse, error) {
if params != nil {
switch {
case params.Withdrawals == nil:
@ -232,6 +233,9 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV4(ctx context.Context, update engine.
return engine.STATUS_INVALID, unsupportedForkErr("fcuV4 must only be called for amsterdam payloads")
}
}
if custodyColumns != nil {
api.eth.BlobFetcher().UpdateCustody(*custodyColumns)
}
// TODO(matt): the spec requires that fcu is applied when called on a valid
// hash, even if params are wrong. To do this we need to split up
// forkchoiceUpdate into a function that only updates the head and then a
@ -710,6 +714,63 @@ func (api *ConsensusAPI) getBlobs(ctx context.Context, hashes []common.Hash, v2
return res, nil
}
// GetBlobsV4 returns cell-level blob data from the transaction pool.
// V4 returns only the requested cells as specified by the indices_bitarray.
func (api *ConsensusAPI) GetBlobsV4(hashes []common.Hash, indicesBitarray types.CustodyBitmap) ([]*engine.BlobCellsAndProofsV1, error) {
head := api.eth.BlockChain().CurrentHeader()
// Sparse blobpool is not necessarily coupled with the Amsterdam fork and
// can technically be supported after the Osaka fork
// (where cell proofs are introduced).
if api.config().LatestFork(head.Time) < forks.Osaka {
return nil, nil
}
if len(hashes) > 128 {
return nil, engine.TooLargeRequest.With(fmt.Errorf("requested blob count too large: %v", len(hashes)))
}
cells, proofs, err := api.eth.BlobCache().GetCells(hashes, indicesBitarray)
if err != nil {
return nil, engine.InvalidParams.With(err)
}
var (
res = make([]*engine.BlobCellsAndProofsV1, len(hashes))
hitCount int
)
getBlobsRequestedCounter.Inc(int64(len(hashes)))
for i := range hashes {
if cells[i] == nil || proofs[i] == nil {
continue
}
hitCount++
blobCells := make([]*hexutil.Bytes, len(cells[i]))
for j, cell := range cells[i] {
if cell != nil {
b := hexutil.Bytes(cell[:])
blobCells[j] = &b
}
}
blobProofs := make([]*hexutil.Bytes, len(proofs[i]))
for j, proof := range proofs[i] {
if proof != nil {
b := hexutil.Bytes(proof[:])
blobProofs[j] = &b
}
}
res[i] = &engine.BlobCellsAndProofsV1{
BlobCells: blobCells,
Proofs: blobProofs,
}
}
getBlobsAvailableCounter.Inc(int64(hitCount))
if hitCount == len(hashes) {
getBlobsRequestCompleteHit.Inc(1)
} else if hitCount > 0 {
getBlobsRequestPartialHit.Inc(1)
} else {
getBlobsRequestMiss.Inc(1)
}
return res, nil
}
// HasBlobs reports availability for the requested blob-versioned-hashes.
func (api *ConsensusAPI) HasBlobs(hashes []common.Hash) []bool {
return api.eth.BlobCache().HasBlobs(context.Background(), hashes)
@ -1139,17 +1200,26 @@ func (api *ConsensusAPI) checkFork(timestamp uint64, forks ...forks.Fork) bool {
}
// ExchangeCapabilities returns the current methods provided by this node.
func (api *ConsensusAPI) ExchangeCapabilities([]string) []string {
func (api *ConsensusAPI) ExchangeCapabilities(caps []string) []string {
valueT := reflect.TypeOf(api)
caps := make([]string, 0, valueT.NumMethod())
// If the CL supports getBlobsV4, we call EnableCell() on the
// blob cache to skip the blob recovery process. This is a
// one-directional toggle, which assumes that once the CL
// supports getBlobsV4, it will not fall back to getBlobsV3
// again.
cellmode := slices.Contains(caps, "engine_getBlobsV4")
api.eth.BlobCache().SetCellMode(cellmode)
ourCaps := make([]string, 0, valueT.NumMethod())
for i := 0; i < valueT.NumMethod(); i++ {
name := []rune(valueT.Method(i).Name)
if string(name) == "ExchangeCapabilities" {
continue
}
caps = append(caps, "engine_"+string(unicode.ToLower(name[0]))+string(name[1:]))
ourCaps = append(ourCaps, "engine_"+string(unicode.ToLower(name[0]))+string(name[1:]))
}
return caps
return ourCaps
}
// GetClientVersionV1 exchanges client version data of this node.

164
eth/catalyst/api_test.go
View file

@ -38,6 +38,7 @@ import (
"github.com/ethereum/go-ethereum/consensus/beacon"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/types/bal"
"github.com/ethereum/go-ethereum/crypto"
@ -1969,23 +1970,27 @@ func init() {
// makeMultiBlobTx is a utility method to construct a random blob tx with
// certain number of blobs in its sidecar.
func makeMultiBlobTx(chainConfig *params.ChainConfig, nonce uint64, blobCount int, blobOffset int, key *ecdsa.PrivateKey, version byte) *types.Transaction {
func makeMultiBlobTx(chainConfig *params.ChainConfig, nonce uint64, blobCount int, blobOffset int, key *ecdsa.PrivateKey, version byte, custody types.CustodyBitmap) *blobpool.BlobTxForPool {
indices := custody.Indices()
var (
blobs []kzg4844.Blob
blobHashes []common.Hash
commitments []kzg4844.Commitment
proofs []kzg4844.Proof
cells []kzg4844.Cell
)
for i := 0; i < blobCount; i++ {
blobs = append(blobs, *testBlobs[blobOffset+i])
commitments = append(commitments, testBlobCommits[blobOffset+i])
j := blobOffset + i
blobHashes = append(blobHashes, testBlobVHashes[j])
commitments = append(commitments, testBlobCommits[j])
if version == types.BlobSidecarVersion0 {
proofs = append(proofs, testBlobProofs[blobOffset+i])
proofs = append(proofs, testBlobProofs[j])
} else {
cellProofs, _ := kzg4844.ComputeCellProofs(testBlobs[blobOffset+i])
proofs = append(proofs, cellProofs...)
proofs = append(proofs, testBlobCellProofs[j]...)
}
full, _ := kzg4844.ComputeCells([]kzg4844.Blob{*testBlobs[j]})
for _, idx := range indices {
cells = append(cells, full[idx])
}
blobHashes = append(blobHashes, testBlobVHashes[blobOffset+i])
}
blobtx := &types.BlobTx{
ChainID: uint256.MustFromBig(chainConfig.ChainID),
@ -1996,12 +2001,20 @@ func makeMultiBlobTx(chainConfig *params.ChainConfig, nonce uint64, blobCount in
BlobFeeCap: uint256.NewInt(1000),
BlobHashes: blobHashes,
Value: uint256.NewInt(100),
Sidecar: types.NewBlobTxSidecar(version, blobs, commitments, proofs),
}
return types.MustSignNewTx(key, types.LatestSigner(chainConfig), blobtx)
return &blobpool.BlobTxForPool{
Tx: types.MustSignNewTx(key, types.LatestSigner(chainConfig), blobtx),
CellSidecar: &types.BlobTxCellSidecar{
Version: version,
Cells: cells,
Commitments: commitments,
Proofs: proofs,
Custody: custody,
},
}
}
func newGetBlobEnv(t testing.TB, version byte) (*node.Node, *ConsensusAPI) {
func newGetBlobEnv(t testing.TB, version byte, custody types.CustodyBitmap) (*node.Node, *ConsensusAPI) {
var (
// Create a database pre-initialize with a genesis block
config = *params.MergedTestChainConfig
@ -2030,17 +2043,23 @@ func newGetBlobEnv(t testing.TB, version byte) (*node.Node, *ConsensusAPI) {
}
n, ethServ := startEthService(t, gspec, nil)
// fill blob txs into the pool
tx1 := makeMultiBlobTx(&config, 0, 2, 0, key1, version) // blob[0, 2)
tx2 := makeMultiBlobTx(&config, 0, 2, 2, key2, version) // blob[2, 4)
tx3 := makeMultiBlobTx(&config, 0, 2, 4, key3, version) // blob[4, 6)
ethServ.TxPool().Add([]*types.Transaction{tx1, tx2, tx3}, true)
// fill blob txs into the pool, each holding only the given custody cells
txs := []*blobpool.BlobTxForPool{
makeMultiBlobTx(&config, 0, 2, 0, key1, version, custody), // blob[0, 2)
makeMultiBlobTx(&config, 0, 2, 2, key2, version, custody), // blob[2, 4)
makeMultiBlobTx(&config, 0, 2, 4, key3, version, custody), // blob[4, 6)
}
for _, ptx := range txs {
if err := ethServ.BlobTxPool().AddPooledTx(ptx); err != nil {
t.Fatalf("failed to add blob tx: %v", err)
}
}
api := newConsensusAPIWithoutHeartbeat(ethServ)
return n, api
}
func TestGetBlobsV2And3(t *testing.T) {
n, api := newGetBlobEnv(t, 1)
n, api := newGetBlobEnv(t, 1, types.CustodyBitmapAll)
defer n.Close()
suites := []struct {
@ -2076,7 +2095,7 @@ func TestGetBlobsV2And3(t *testing.T) {
// Benchmark GetBlobsV2 internals
// Note that this is not an RPC-level benchmark, so JSON-RPC overhead is not included.
func BenchmarkGetBlobsV2(b *testing.B) {
n, api := newGetBlobEnv(b, 1)
n, api := newGetBlobEnv(b, 1, types.CustodyBitmapAll)
defer n.Close()
// for blobs in [1, 2, 4, 6], print string and run benchmark
@ -2136,3 +2155,112 @@ func runGetBlobs(t testing.TB, getBlobs getBlobsFn, start, limit int, fillRandom
t.Fatalf("Unexpected result for case %s", name)
}
}
func TestGetBlobsV4(t *testing.T) {
// The pool holds only this set of custody cells.
custody := types.NewCustodyBitmap([]uint64{0, 1, 2, 3, 4})
n, api := newGetBlobEnv(t, 1, custody)
defer n.Close()
masks := []struct {
name string
mask types.CustodyBitmap
}{
{"missing", types.NewCustodyBitmap([]uint64{5})},
{"overlap", types.NewCustodyBitmap([]uint64{0, 2, 5, 127})},
{"aligned", custody},
}
suites := []struct {
start int
limit int
missingBlob bool
}{
{start: 0, limit: 1},
{start: 0, limit: 2},
{start: 1, limit: 3},
{start: 0, limit: 6},
{start: 1, limit: 5},
{start: 0, limit: 6, missingBlob: true},
}
for _, m := range masks {
for i, suite := range suites {
runGetBlobsV4(t, api, custody, m.mask, suite.start, suite.limit, suite.missingBlob, fmt.Sprintf("GetBlobsV4 mask=%s suite=%d", m.name, i))
}
}
}
func runGetBlobsV4(t testing.TB, api *ConsensusAPI, custody, mask types.CustodyBitmap, start, limit int, missingBlob bool, name string) {
// Fill the request for retrieving cells and build the expected response.
var (
vhashes []common.Hash
expect []*engine.BlobCellsAndProofsV1
indices = mask.Indices()
)
for j := start; j < limit; j++ {
vhashes = append(vhashes, testBlobVHashes[j])
cells, err := kzg4844.ComputeCells([]kzg4844.Blob{*testBlobs[j]})
if err != nil {
t.Fatalf("Failed to compute cells for case %s: %v", name, err)
}
blobCells := make([]*hexutil.Bytes, len(indices))
blobProofs := make([]*hexutil.Bytes, len(indices))
for i, idx := range indices {
if !custody.IsSet(idx) {
continue
}
cell := hexutil.Bytes(cells[idx][:])
blobCells[i] = &cell
proof := hexutil.Bytes(testBlobCellProofs[j][idx][:])
blobProofs[i] = &proof
}
expect = append(expect, &engine.BlobCellsAndProofsV1{
BlobCells: blobCells,
Proofs: blobProofs,
})
}
// put random missing blob
if missingBlob {
vhashes = append(vhashes, testrand.Hash())
expect = append(expect, nil)
}
result, err := api.GetBlobsV4(vhashes, mask)
if err != nil {
t.Errorf("Unexpected error for case %s, %v", name, err)
}
if !reflect.DeepEqual(result, expect) {
t.Fatalf("Unexpected result for case %s", name)
}
}
// TestForkchoiceUpdatedV4 tests the custody bitmap argument added in V4.
func TestForkchoiceUpdatedV4(t *testing.T) {
n, api := newGetBlobEnv(t, 1, types.CustodyBitmapAll)
defer n.Close()
head := api.eth.BlockChain().CurrentHeader().Hash()
fcState := engine.ForkchoiceStateV1{
HeadBlockHash: head,
SafeBlockHash: head,
FinalizedBlockHash: head,
}
// Non nil custody bitmap case.
custody := types.NewCustodyBitmap([]uint64{0, 1, 2, 127})
resp, err := api.ForkchoiceUpdatedV4(context.Background(), fcState, nil, &custody)
if err != nil {
t.Fatalf("Unexpected error with custody bitmap: %v", err)
}
if resp.PayloadStatus.Status != engine.VALID {
t.Fatalf("Unexpected status with custody bitmap: got %s, want %s", resp.PayloadStatus.Status, engine.VALID)
}
// Nil custody bitmap case.
resp, err = api.ForkchoiceUpdatedV4(context.Background(), fcState, nil, nil)
if err != nil {
t.Fatalf("Unexpected error with nil custody bitmap: %v", err)
}
if resp.PayloadStatus.Status != engine.VALID {
t.Fatalf("Unexpected status with nil custody bitmap: got %s, want %s", resp.PayloadStatus.Status, engine.VALID)
}
}

959
eth/fetcher/blob_fetcher.go Normal file
View file

@ -0,0 +1,959 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package fetcher
import (
"fmt"
"iter"
"math/rand"
"slices"
"sort"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
type random interface {
Intn(n int) int
}
// BlobFetcher fetches blobs of new type-3 transactions with probability p,
// and for the remaining (1-p) transactions, it performs availability checks.
// For availability checks, it fetches cells from each blob in the transaction
// according to the custody cell indices provided by the consensus client
// connected to this execution client.
var blobFetchTimeout = 5 * time.Second
var blobAvailabilityTimeout = 2 * time.Second
// DefaultFetchProbability is the default probability of fetching the full blob
// payload for the sparse blobpool.
const DefaultFetchProbability = 15
const (
availabilityThreshold = 2
maxPayloadRetrievals = 128
maxPayloadAnnounces = 4096
// maxCellRequests caps the burst of cell requests we can issue at once
// to a single peer. Worst case 256 * 6 = 1536 cells (~3 MB)
maxCellRequests = 256
// refillInterval is the gap between token refill. Combined with
// maxCellRequests and 2-minute buffer timeout, a peer's worst case
// buffered cells is about 16 MB
refillInterval = time.Second / 9
)
type blobTxAnnounce struct {
origin string // Identifier of the peer that sent the announcement
txs []common.Hash // Hashes of transactions announced
cells types.CustodyBitmap // Custody information of transactions being announced
}
type cellRequest struct {
txs []common.Hash // Transactions that have been requested for their cells
cells types.CustodyBitmap // Requested cell indices
time mclock.AbsTime // Timestamp when the request was made
}
type payloadDelivery struct {
origin string // Peer from which the payloads were delivered
txs []common.Hash // Hashes of transactions that were delivered
cells [][]kzg4844.Cell
cellBitmap types.CustodyBitmap
}
type cellWithSeq struct {
seq uint64
cells types.CustodyBitmap
}
// PeerCellDelivery holds cells delivered by a single peer.
type PeerCellDelivery struct {
Cells []kzg4844.Cell // blob-major order as received
Indices []uint64 // custody indices provided by this peer
}
type fetchStatus struct {
fetching types.CustodyBitmap // To avoid fetching cells which had already been fetched / currently being fetched
fetched []uint64 // Custody indices that have been fetched (per-blob, same for all blobs)
deliveries map[string]*PeerCellDelivery // Per-peer cell deliveries
}
type BlobFetcherFunctions struct {
HasPayload func(common.Hash) bool
AddCells func(common.Hash, map[string]*PeerCellDelivery, types.CustodyBitmap)
FetchPayloads func(string, []common.Hash, types.CustodyBitmap) error
DropPeer func(string)
}
// BlobFetcher is responsible for managing type 3 transactions based on peer announcements.
//
// BlobFetcher manages three buffers:
// - Transactions not to be fetched are moved to "waitlist"
// if a payload(blob) seems to be possessed by D(threshold) other peers, request custody cells for that.
// Accept it when the cells are received. Otherwise, it is dropped.
// - Transactions queued to be fetched are moved to "announces"
// if a payload is received, it is added to the blob pool. Otherwise, the transaction is dropped.
// - Transactions to be fetched are moved to "fetching"
// if a payload/cell announcement is received during fetch, the peer is recorded as an alternate source.
type BlobFetcher struct {
notify chan *blobTxAnnounce
cleanup chan *payloadDelivery
drop chan *txDrop
custodyCh chan types.CustodyBitmap
quit chan struct{}
custody types.CustodyBitmap
txSeq uint64 // To make transactions fetched in arrival order
full map[common.Hash]struct{}
partial map[common.Hash]struct{}
// Buffer 1: Set of blob txs whose blob data is waiting for availability confirmation (partial fetch)
waitlist map[common.Hash]map[string]struct{} // Peer set that announced blob availability
waittime map[common.Hash]mclock.AbsTime // Timestamp when added to waitlist
waitslots map[string]map[common.Hash]struct{} // Waiting announcements grouped by peer (DoS protection)
// waitSlots should also include announcements with partial cells
// Buffer 2: Transactions queued for fetching (full fetch + partial fetch)
// "announces" is shared with stage 3, for DoS protection
announces map[string]map[common.Hash]*cellWithSeq // Set of announced transactions, grouped by origin peer
// Buffer 2
// Stage 3: Transactions whose payloads/cells are currently being fetched (full fetch + partial fetch)
fetches map[common.Hash]*fetchStatus // Hash -> Bitmap, in-flight transaction cells
requests map[string][]*cellRequest // In-flight transaction retrievals
alternates map[common.Hash]map[string]types.CustodyBitmap // In-flight transaction alternate origins (in case the peer is dropped)
fn BlobFetcherFunctions // callbacks
fetchProbability uint64
// peerTokens tracks each peer's remaining cell request token.
peerTokens map[string]*token
step chan struct{} // Notification channel when the fetcher loop iterates
clock mclock.Clock // Monotonic clock or simulated clock for tests
realTime func() time.Time // Real system time or simulated time for tests
rand random // Randomizer
}
// token is a per peer token bucket for outgoing cell requests.
type token struct {
amount int64
last mclock.AbsTime
}
func NewBlobFetcher(fn BlobFetcherFunctions, custody types.CustodyBitmap, rand random, fetchProbability uint64) *BlobFetcher {
if fetchProbability < DefaultFetchProbability {
fetchProbability = DefaultFetchProbability
}
return &BlobFetcher{
notify: make(chan *blobTxAnnounce),
cleanup: make(chan *payloadDelivery),
drop: make(chan *txDrop),
custodyCh: make(chan types.CustodyBitmap),
quit: make(chan struct{}),
full: make(map[common.Hash]struct{}),
partial: make(map[common.Hash]struct{}),
waitlist: make(map[common.Hash]map[string]struct{}),
waittime: make(map[common.Hash]mclock.AbsTime),
waitslots: make(map[string]map[common.Hash]struct{}),
announces: make(map[string]map[common.Hash]*cellWithSeq),
fetches: make(map[common.Hash]*fetchStatus),
requests: make(map[string][]*cellRequest),
alternates: make(map[common.Hash]map[string]types.CustodyBitmap),
peerTokens: make(map[string]*token),
fn: fn,
fetchProbability: fetchProbability,
custody: custody,
clock: mclock.System{},
realTime: time.Now,
rand: rand,
}
}
// Notify is called when a Type 3 transaction is observed on the network. (TransactionPacket / NewPooledTransactionHashesPacket)
func (f *BlobFetcher) Notify(peer string, txs []common.Hash, cells types.CustodyBitmap) error {
blobAnnounceInMeter.Mark(int64(len(txs)))
anns := make([]common.Hash, 0)
for _, tx := range txs {
if f.fn.HasPayload(tx) {
continue
}
anns = append(anns, tx)
}
blobAnnounce := &blobTxAnnounce{origin: peer, txs: anns, cells: cells}
select {
case f.notify <- blobAnnounce:
return nil
case <-f.quit:
return errTerminated
}
}
// Enqueue inserts a batch of received blob payloads into the blob pool.
// This is triggered by ethHandler upon receiving direct request responses.
func (f *BlobFetcher) Enqueue(peer string, hashes []common.Hash, cells [][]kzg4844.Cell, cellBitmap types.CustodyBitmap) error {
blobReplyInMeter.Mark(int64(len(hashes)))
if len(hashes) != len(cells) {
return fmt.Errorf("cells reply length mismatch: %d hashes, %d cells", len(hashes), len(cells))
}
// An empty custody mask means "no cells"; a peer delivering cells alongside it
// is malformed and should be dropped.
if cellBitmap.OneCount() == 0 {
for _, c := range cells {
if len(c) != 0 {
return fmt.Errorf("cells delivered with empty custody mask (%d cells)", len(c))
}
}
}
select {
case f.cleanup <- &payloadDelivery{origin: peer, txs: hashes, cells: cells, cellBitmap: cellBitmap}:
case <-f.quit:
return errTerminated
}
return nil
}
func (f *BlobFetcher) Drop(peer string) error {
select {
case f.drop <- &txDrop{peer: peer}:
return nil
case <-f.quit:
return errTerminated
}
}
// UpdateCustody hands a new custody bitmap to the fetcher loop. The actual
// swap happens inside the loop so f.custody is never read and written
// concurrently.
func (f *BlobFetcher) UpdateCustody(cells types.CustodyBitmap) {
select {
case f.custodyCh <- cells:
case <-f.quit:
}
}
func (f *BlobFetcher) Start() {
go f.loop()
}
func (f *BlobFetcher) Stop() {
close(f.quit)
}
func (f *BlobFetcher) loop() {
var (
waitTimer = new(mclock.Timer) // Timer for waitlist (availability)
waitTrigger = make(chan struct{}, 1)
timeoutTimer = new(mclock.Timer) // Timer for payload fetch request
timeoutTrigger = make(chan struct{}, 1)
)
for {
select {
case ann := <-f.notify:
// Drop part of the announcements if too many have accumulated from that peer
// This prevents a peer from dominating the queue with txs without responding to the request
used := len(f.waitslots[ann.origin]) + len(f.announces[ann.origin])
if used >= maxPayloadAnnounces {
blobAnnounceDOSMeter.Mark(int64(len(ann.txs)))
break
}
want := used + len(ann.txs)
if want >= maxPayloadAnnounces {
blobAnnounceDOSMeter.Mark(int64(want - maxPayloadAnnounces))
ann.txs = ann.txs[:maxPayloadAnnounces-used]
}
var (
idleWait = len(f.waittime) == 0
_, oldPeer = f.announces[ann.origin]
nextSeq = func() uint64 {
seq := f.txSeq
f.txSeq++
return seq
}
reschedule = make(map[string]struct{})
)
for _, hash := range ann.txs {
if oldPeer && f.announces[ann.origin][hash] != nil {
// Ignore already announced information
// We also have to prevent reannouncement by changing cells field.
// Considering cell custody transition is notified in advance of its finalization by consensus client,
// there is no reason to reannounce cells, and it has to be prevented.
continue
}
// Decide full or partial request
if _, ok := f.full[hash]; !ok {
if _, ok := f.partial[hash]; !ok {
// Not decided yet
var randomValue int
if f.rand == nil {
randomValue = rand.Intn(100)
} else {
randomValue = f.rand.Intn(100)
}
// For eager mode, always fetch immediately
if uint64(randomValue) < f.fetchProbability || f.custody.OneCount() >= kzg4844.DataPerBlob {
f.full[hash] = struct{}{}
} else {
f.partial[hash] = struct{}{}
// Register for availability check
f.waitlist[hash] = make(map[string]struct{})
f.waittime[hash] = f.clock.Now()
}
}
}
if _, ok := f.full[hash]; ok {
// 1) Decided to send full request of the tx
// if the cells are not set to all bitmap, ignore.
if ann.cells != types.CustodyBitmapAll {
continue
}
if f.announces[ann.origin] == nil {
f.announces[ann.origin] = make(map[common.Hash]*cellWithSeq)
}
f.announces[ann.origin][hash] = &cellWithSeq{
cells: types.CustodyBitmapData,
seq: nextSeq(),
}
reschedule[ann.origin] = struct{}{}
continue
}
if _, ok := f.partial[hash]; ok {
// 2) Decided to send partial request of the tx
if f.waitlist[hash] != nil {
// it is already waiting for availability check
if ann.cells != types.CustodyBitmapAll {
// Availability check is only meaningful with full availability announcements
continue
}
// Transaction is at the stage of availability check
// Add the peer to the peer list with full availability (waitlist)
f.waitlist[hash][ann.origin] = struct{}{}
if waitslots := f.waitslots[ann.origin]; waitslots != nil {
waitslots[hash] = struct{}{}
} else {
f.waitslots[ann.origin] = map[common.Hash]struct{}{
hash: {},
}
}
if len(f.waitlist[hash]) >= availabilityThreshold {
// Passed availability check, move to fetching stage
blobFetcherWaitTime.Update(int64(time.Duration(f.clock.Now() - f.waittime[hash])))
for peer := range f.waitlist[hash] {
if f.announces[peer] == nil {
f.announces[peer] = make(map[common.Hash]*cellWithSeq)
}
f.announces[peer][hash] = &cellWithSeq{
cells: f.custody,
seq: nextSeq(),
}
delete(f.waitslots[peer], hash)
if len(f.waitslots[peer]) == 0 {
delete(f.waitslots, peer)
}
reschedule[peer] = struct{}{}
}
delete(f.waitlist, hash)
delete(f.waittime, hash)
}
continue
}
if ann.cells.Intersection(f.custody).OneCount() == 0 {
// If there's no custody overlapping in ann, it can be ignored
continue
}
// Add this peer as a possible fetch source
if f.announces[ann.origin] == nil {
f.announces[ann.origin] = make(map[common.Hash]*cellWithSeq)
}
f.announces[ann.origin][hash] = &cellWithSeq{
cells: ann.cells.Intersection(f.custody),
seq: nextSeq(),
}
reschedule[ann.origin] = struct{}{}
}
}
// If a new item was added to the waitlist, schedule its timeout
if idleWait && len(f.waittime) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
// If this is a new peer and that peer sent transaction with payload flag,
// schedule transaction fetches from it
if !oldPeer && len(f.announces[ann.origin]) > 0 {
f.scheduleFetches(timeoutTimer, timeoutTrigger, reschedule)
}
case <-waitTrigger:
// At least one transaction's waiting time ran out. Instead of dropping,
// convert timed-out partial fetches to full fetches so we don't lose
// the transaction. All peers in the waitlist announced full custody
// (that was the entry condition), so they can serve as full fetch sources.
reschedule := make(map[string]struct{})
for hash, instance := range f.waittime {
if time.Duration(f.clock.Now()-instance)+txGatherSlack > blobAvailabilityTimeout {
// partial -> full conversion
delete(f.partial, hash)
f.full[hash] = struct{}{}
blobAnnounceTimeoutMeter.Mark(1)
for peer := range f.waitlist[hash] {
if f.announces[peer] == nil {
f.announces[peer] = make(map[common.Hash]*cellWithSeq)
}
f.announces[peer][hash] = &cellWithSeq{
cells: types.CustodyBitmapData,
seq: f.txSeq,
}
f.txSeq++
delete(f.waitslots[peer], hash)
if len(f.waitslots[peer]) == 0 {
delete(f.waitslots, peer)
}
reschedule[peer] = struct{}{}
}
delete(f.waittime, hash)
delete(f.waitlist, hash)
}
}
if len(reschedule) > 0 {
f.scheduleFetches(timeoutTimer, timeoutTrigger, reschedule)
}
// If transactions are still waiting for availability, reschedule the wait timer
if len(f.waittime) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
case <-timeoutTrigger:
// Clean up any expired retrievals and avoid re-requesting them from the
// same peer (either overloaded or malicious, useless in both cases).
// Update blobpool according to availability result.
for peer, requests := range f.requests {
newRequests := make([]*cellRequest, 0)
for _, req := range requests {
if time.Duration(f.clock.Now()-req.time)+txGatherSlack > blobFetchTimeout {
blobRequestTimeoutMeter.Mark(int64(len(req.txs)))
for _, hash := range req.txs {
// Do not request the same tx from this peer
delete(f.announces[peer], hash)
delete(f.alternates[hash], peer)
if f.fetches[hash] == nil {
// already completed or discarded via another peer
continue
}
// Allow other candidates to be requested these cells
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(req.cells)
// Drop the tx entirely if the remaining sources can no longer complete it
if !f.recoverable(hash) {
f.discard(hash)
}
}
if len(f.announces[peer]) == 0 {
delete(f.announces, peer)
}
} else {
newRequests = append(newRequests, req)
}
}
f.requests[peer] = newRequests
if len(f.requests[peer]) == 0 {
delete(f.requests, peer)
}
}
// Schedule a new transaction retrieval
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
// Trigger timeout for new schedule
f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
case delivery := <-f.cleanup:
// Remove from announce
var requestId int
var request *cellRequest
for _, hash := range delivery.txs {
// Find the request
for i, req := range f.requests[delivery.origin] {
if slices.Contains(req.txs, hash) && req.cells == delivery.cellBitmap {
request = req
requestId = i
break
}
}
if request != nil {
break
}
}
if request == nil {
// peer sent cells not requested. ignore
break
}
for i, hash := range delivery.txs {
if !slices.Contains(request.txs, hash) || f.fetches[hash] == nil {
// Unexpected or canceled hash, ignore
continue
}
indices := delivery.cellBitmap.Indices()
if len(indices) > 0 {
status := f.fetches[hash]
status.deliveries[delivery.origin] = &PeerCellDelivery{
Cells: delivery.cells[i],
Indices: indices,
}
status.fetched = append(status.fetched, indices...)
}
// Update announces of this peer
delete(f.announces[delivery.origin], hash)
if len(f.announces[delivery.origin]) == 0 {
delete(f.announces, delivery.origin)
}
delete(f.alternates[hash], delivery.origin)
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
}
// Check whether the all required cells are fetched
completed := false
if _, ok := f.full[hash]; ok && len(f.fetches[hash].fetched) >= kzg4844.DataPerBlob {
completed = true
} else if _, ok := f.partial[hash]; ok {
fetched := make([]uint64, len(f.fetches[hash].fetched))
copy(fetched, f.fetches[hash].fetched)
slices.Sort(fetched)
custodyIndices := f.custody.Indices()
completed = slices.Equal(fetched, custodyIndices)
}
if completed {
blobFetcherFetchTime.Update(int64(time.Duration(f.clock.Now() - request.time)))
status := f.fetches[hash]
collectedCustody := types.NewCustodyBitmap(status.fetched)
f.fn.AddCells(hash, status.deliveries, collectedCustody)
f.discard(hash)
}
}
blobRequestDoneMeter.Mark(int64(len(delivery.txs)))
// Remove the request
f.requests[delivery.origin][requestId] = f.requests[delivery.origin][len(f.requests[delivery.origin])-1]
f.requests[delivery.origin] = f.requests[delivery.origin][:len(f.requests[delivery.origin])-1]
if len(f.requests[delivery.origin]) == 0 {
delete(f.requests, delivery.origin)
}
// Reschedule missing transactions in the request
// Anything not delivered should be re-scheduled (with or without
// this peer, depending on the response cutoff)
delivered := make(map[common.Hash]struct{})
for _, hash := range delivery.txs {
delivered[hash] = struct{}{}
}
cutoff := len(request.txs)
for i, hash := range request.txs {
if _, ok := delivered[hash]; ok {
cutoff = i
continue
}
}
// Reschedule missing hashes from alternates, not-fulfilled from alt+self
for i, hash := range request.txs {
if _, ok := delivered[hash]; !ok {
// Not delivered
if i < cutoff {
// Remove origin from candidate sources for partial responses
delete(f.alternates[hash], delivery.origin)
delete(f.announces[delivery.origin], hash)
if len(f.announces[delivery.origin]) == 0 {
delete(f.announces, delivery.origin)
}
}
// Mark cells deliverable by other peers
if f.fetches[hash] != nil {
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(delivery.cellBitmap)
}
}
}
// Something was delivered, try to reschedule requests
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too
case drop := <-f.drop:
// A peer was dropped, remove all traces of it
delete(f.peerTokens, drop.peer)
if _, ok := f.waitslots[drop.peer]; ok {
for hash := range f.waitslots[drop.peer] {
delete(f.waitlist[hash], drop.peer)
if len(f.waitlist[hash]) == 0 {
delete(f.waitlist, hash)
delete(f.waittime, hash)
}
}
delete(f.waitslots, drop.peer)
if len(f.waitlist) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
}
// Clean up general announcement tracking
if _, ok := f.announces[drop.peer]; ok {
for hash := range f.announces[drop.peer] {
delete(f.alternates[hash], drop.peer)
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
}
}
delete(f.announces, drop.peer)
}
delete(f.announces, drop.peer)
// Clean up any active requests
if request, ok := f.requests[drop.peer]; ok && len(request) != 0 {
for _, req := range request {
for _, hash := range req.txs {
if f.fetches[hash] == nil {
// already canceled
continue
}
// Undelivered hash, reschedule if there's an alternative origin available
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(req.cells)
delete(f.alternates[hash], drop.peer)
if !f.recoverable(hash) {
f.discard(hash)
}
}
}
delete(f.requests, drop.peer)
// If a request was cancelled, check if anything needs to be rescheduled
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
}
case cells := <-f.custodyCh:
f.custody = cells
case <-f.quit:
return
}
// Update metrics gauges
blobFetcherWaitingPeers.Update(int64(len(f.waitslots)))
blobFetcherWaitingHashes.Update(int64(len(f.waitlist)))
blobFetcherQueueingPeers.Update(int64(len(f.announces) - len(f.requests)))
blobFetcherQueueingHashes.Update(int64(len(f.announces)))
blobFetcherFetchingPeers.Update(int64(len(f.requests)))
blobFetcherFetchingHashes.Update(int64(len(f.fetches)))
// Loop did something, ping the step notifier if needed (tests)
if f.step != nil {
f.step <- struct{}{}
}
}
}
func (f *BlobFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) {
if *timer != nil {
(*timer).Stop()
}
now := f.clock.Now()
earliest := now
for _, instance := range f.waittime {
if earliest > instance {
earliest = instance
if blobAvailabilityTimeout-time.Duration(now-earliest) < txGatherSlack {
break
}
}
}
*timer = f.clock.AfterFunc(blobAvailabilityTimeout-time.Duration(now-earliest), func() {
trigger <- struct{}{}
})
}
// Exactly same as the one in TxFetcher
func (f *BlobFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) {
if *timer != nil {
(*timer).Stop()
}
now := f.clock.Now()
earliest := now
for _, requests := range f.requests {
for _, req := range requests {
// If this request already timed out, skip it altogether
if req.txs == nil {
continue
}
if earliest > req.time {
earliest = req.time
if blobFetchTimeout-time.Duration(now-earliest) < txGatherSlack {
break
}
}
}
}
*timer = f.clock.AfterFunc(blobFetchTimeout-time.Duration(now-earliest), func() {
trigger <- struct{}{}
})
}
// consumeToken consumes n tokens from peer's cell-request budget.
// It returns false if the remaining tokens cannot cover n.
func (f *BlobFetcher) consumeToken(peer string, n int) bool {
now := f.clock.Now()
b, ok := f.peerTokens[peer]
if !ok {
b = &token{amount: maxCellRequests, last: now}
f.peerTokens[peer] = b
} else {
// Here, fractional remaining elapsed time is left in b.last
// so that it can be carried over to the next call
elapsed := time.Duration(now - b.last)
if add := int64(elapsed / refillInterval); add > 0 {
b.amount += add
if b.amount > maxCellRequests {
b.amount = maxCellRequests
}
b.last += mclock.AbsTime(time.Duration(add) * refillInterval)
}
}
if b.amount < int64(n) {
return false
}
b.amount -= int64(n)
return true
}
// recoverable checks whether the missing cells of a tx can be fully
// received from the alternative peers.
func (f *BlobFetcher) recoverable(hash common.Hash) bool {
// union of already fetched cells and the cells still announced by peers
var covered types.CustodyBitmap
if status := f.fetches[hash]; status != nil {
covered = types.NewCustodyBitmap(status.fetched)
}
for _, anns := range f.announces {
if cs, ok := anns[hash]; ok {
covered = covered.Union(cs.cells)
}
}
if _, ok := f.full[hash]; ok {
// Full case: peer coverage is enough to recover the blob
return covered.OneCount() >= kzg4844.DataPerBlob
}
// Partial case:
// return true if our custody can be covered
return f.custody.Difference(covered).OneCount() == 0
}
// discard removes every trace of a transaction from the fetcher: its fetch
// state and all pending announcements/alternates across peers.
func (f *BlobFetcher) discard(hash common.Hash) {
for peer, anns := range f.announces {
if _, ok := anns[hash]; ok {
delete(anns, hash)
if len(anns) == 0 {
delete(f.announces, peer)
}
}
}
delete(f.alternates, hash)
delete(f.fetches, hash)
delete(f.full, hash)
delete(f.partial, hash)
}
func (f *BlobFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, whitelist map[string]struct{}) {
// Gather the set of peers we want to retrieve from (default to all)
actives := whitelist
if actives == nil {
actives = make(map[string]struct{})
for peer := range f.announces {
actives[peer] = struct{}{}
}
}
if len(actives) == 0 {
return
}
wasIdle := len(f.requests) == 0
// For each active peer, try to schedule some payload fetches.
for peer := range f.peers(actives) {
if len(f.announces[peer]) == 0 || len(f.requests[peer]) != 0 {
continue
}
var (
hashes []common.Hash
custodies []types.CustodyBitmap
)
for hash, cells := range f.announcesByArrival(f.announces[peer]) {
if !f.recoverable(hash) {
continue
}
var unfetched types.CustodyBitmap
if f.fetches[hash] == nil {
// tx is not being fetched
unfetched = cells
} else {
unfetched = cells.Difference(f.fetches[hash].fetching)
}
// Mark fetching for unfetched cells if the peer has enough token.
// Otherwise, the next peer who announced the hash and has token will be selected
// in the next loop
if unfetched.OneCount() > 0 && f.consumeToken(peer, unfetched.OneCount()) {
if f.fetches[hash] == nil {
f.fetches[hash] = &fetchStatus{
fetching: unfetched,
fetched: make([]uint64, 0),
deliveries: make(map[string]*PeerCellDelivery),
}
} else {
f.fetches[hash].fetching = f.fetches[hash].fetching.Union(unfetched)
}
// Accumulate the hash and stop if the limit was reached
hashes = append(hashes, hash)
custodies = append(custodies, unfetched)
}
// Mark alternatives
if f.alternates[hash] == nil {
f.alternates[hash] = map[string]types.CustodyBitmap{
peer: cells,
}
} else {
f.alternates[hash][peer] = cells
}
// Stop once we've accumulated enough hashes for this peer
if len(hashes) >= maxPayloadRetrievals {
break
}
}
// If any hashes were allocated, request them from the peer
if len(hashes) > 0 {
// Group hashes by custody bitmap
requestByCustody := make(map[types.CustodyBitmap]*cellRequest)
for i, hash := range hashes {
key := custodies[i]
if _, ok := requestByCustody[key]; !ok {
requestByCustody[key] = &cellRequest{
txs: []common.Hash{},
cells: custodies[i],
time: f.clock.Now(),
}
}
requestByCustody[key].txs = append(requestByCustody[key].txs, hash)
}
// construct request
var request []*cellRequest
for _, cr := range requestByCustody {
request = append(request, cr)
}
f.requests[peer] = request
// Iterate over a copy: the loop may swap-truncate f.requests[peer]
// concurrently when a delivery for this peer arrives.
reqs := slices.Clone(request)
go func() {
for _, req := range reqs {
blobRequestOutMeter.Mark(int64(len(req.txs)))
if err := f.fn.FetchPayloads(peer, req.txs, req.cells); err != nil {
blobRequestFailMeter.Mark(int64(len(req.txs)))
f.Drop(peer)
break
}
}
}()
}
}
// If a new request was fired, schedule a timeout timer
if wasIdle && len(f.requests) > 0 {
f.rescheduleTimeout(timer, timeout)
}
}
// announcesByArrival returns an iterator over the given announcements
// in arrival order. We enforce an arrival ordering to minimize
// the chances of transaction nonce-gaps, which result in
// transactions being rejected by the txpool.
func (f *BlobFetcher) announcesByArrival(announces map[common.Hash]*cellWithSeq) iter.Seq2[common.Hash, types.CustodyBitmap] {
return func(yield func(hash common.Hash, cells types.CustodyBitmap) bool) {
type announcement struct {
hash common.Hash
cells types.CustodyBitmap
seq uint64
}
// Process announcements by their arrival order
list := make([]announcement, 0, len(announces))
for hash, entry := range announces {
list = append(list, announcement{hash: hash, cells: entry.cells, seq: entry.seq})
}
sort.Slice(list, func(i, j int) bool {
return list[i].seq < list[j].seq
})
for i := range list {
if !yield(list[i].hash, list[i].cells) {
return
}
}
}
}
// peers returns an iterator over a map of peers in production, but during
// testing it does a deterministic sorted random to allow reproducing issues.
func (f *BlobFetcher) peers(peers map[string]struct{}) iter.Seq[string] {
return func(yield func(peer string) bool) {
// If we're running production(step == nil), use whatever Go's map gives us
if f.step == nil {
for peer := range peers {
if !yield(peer) {
return
}
}
return
}
// We're running the test suite, make iteration deterministic (sorted by peer id)
list := make([]string, 0, len(peers))
for peer := range peers {
list = append(list, peer)
}
sort.Strings(list)
for _, peer := range list {
if !yield(peer) {
return
}
}
}
}

1152
eth/fetcher/blob_fetcher_test.go Normal file
View file

File diff suppressed because it is too large Load diff

22
eth/fetcher/metrics.go
View file

@ -57,4 +57,26 @@ var (
// to become "unfrozen", either by eventually replying to the request
// or by being dropped, measuring from the moment the request was sent.
txFetcherSlowWait = metrics.NewRegisteredHistogram("eth/fetcher/transaction/slow/wait", nil, metrics.NewExpDecaySample(1028, 0.015))
blobAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/in", nil)
blobAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/dos", nil)
// This metric tracks partial→full conversions due to availability timeout
blobAnnounceTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/timeout", nil)
blobRequestOutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/out", nil)
blobRequestFailMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/fail", nil)
blobRequestDoneMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/done", nil)
blobRequestTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/timeout", nil)
blobReplyInMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/replies/in", nil)
blobFetcherWaitingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/waiting/peers", nil)
blobFetcherWaitingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/waiting/hashes", nil)
blobFetcherQueueingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/queueing/peers", nil)
blobFetcherQueueingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/queueing/hashes", nil)
blobFetcherFetchingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/fetching/peers", nil)
blobFetcherFetchingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/fetching/hashes", nil)
blobFetcherWaitTime = metrics.NewRegisteredHistogram("eth/fetcher/blob/wait/time", nil, metrics.NewExpDecaySample(1028, 0.015))
blobFetcherFetchTime = metrics.NewRegisteredHistogram("eth/fetcher/blob/fetch/time", nil, metrics.NewExpDecaySample(1028, 0.015))
)

176
eth/fetcher/tx_fetcher.go
View file

@ -29,8 +29,11 @@ import (
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
)
const (
@ -186,6 +189,8 @@ type TxFetcher struct {
dropPeer func(string) // Drops a peer in case of announcement violation
onAccepted func(peer string, hashes []common.Hash) // Optional: notified with accepted tx hashes per peer
buffer *blobpool.BlobBuffer
step chan struct{} // Notification channel when the fetcher loop iterates
clock mclock.Clock // Monotonic clock or simulated clock for tests
realTime func() time.Time // Real system time or simulated time for tests
@ -195,8 +200,9 @@ type TxFetcher struct {
// NewTxFetcher creates a transaction fetcher to retrieve transaction
// based on hash announcements.
// Chain can be nil to disable on-chain checks.
func NewTxFetcher(chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), onAccepted func(string, []common.Hash)) *TxFetcher {
return NewTxFetcherForTests(chain, validateMeta, addTxs, fetchTxs, dropPeer, onAccepted, mclock.System{}, time.Now, nil)
func NewTxFetcher(chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error,
dropPeer func(string), onAccepted func(string, []common.Hash), buffer *blobpool.BlobBuffer) *TxFetcher {
return NewTxFetcherForTests(chain, validateMeta, addTxs, fetchTxs, dropPeer, onAccepted, buffer, mclock.System{}, time.Now, nil)
}
// NewTxFetcherForTests is a testing method to mock out the realtime clock with
@ -204,7 +210,7 @@ func NewTxFetcher(chain *core.BlockChain, validateMeta func(common.Hash, byte) e
// Chain can be nil to disable on-chain checks.
func NewTxFetcherForTests(
chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), onAccepted func(string, []common.Hash),
clock mclock.Clock, realTime func() time.Time, rand *mrand.Rand) *TxFetcher {
buffer *blobpool.BlobBuffer, clock mclock.Clock, realTime func() time.Time, rand *mrand.Rand) *TxFetcher {
return &TxFetcher{
notify: make(chan *txAnnounce),
cleanup: make(chan *txDelivery),
@ -225,6 +231,7 @@ func NewTxFetcherForTests(
addTxs: addTxs,
fetchTxs: fetchTxs,
dropPeer: dropPeer,
buffer: buffer,
onAccepted: onAccepted,
clock: clock,
realTime: realTime,
@ -233,8 +240,8 @@ func NewTxFetcherForTests(
}
// Notify announces the fetcher of the potential availability of a new batch of
// transactions in the network.
func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []common.Hash) error {
// transactions in the network. It returns array of hashes decided to be fetched.
func (f *TxFetcher) Notify(peer string, kinds []byte, sizes []uint32, hashes []common.Hash) ([]common.Hash, error) {
// Keep track of all the announced transactions
txAnnounceInMeter.Mark(int64(len(hashes)))
@ -247,13 +254,18 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
unknownHashes = make([]common.Hash, 0, len(hashes))
unknownMetas = make([]txMetadata, 0, len(hashes))
blobFetchHashes = make([]common.Hash, 0, len(hashes))
duplicate int64
onchain int64
underpriced int64
)
for i, hash := range hashes {
err := f.validateMeta(hash, types[i])
err := f.validateMeta(hash, kinds[i])
if errors.Is(err, txpool.ErrAlreadyKnown) {
if kinds[i] == types.BlobTxType {
blobFetchHashes = append(blobFetchHashes, hash)
}
duplicate++
continue
}
@ -273,11 +285,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
}
unknownHashes = append(unknownHashes, hash)
if kinds[i] == types.BlobTxType {
blobFetchHashes = append(blobFetchHashes, hash)
}
// Transaction metadata has been available since eth68, and all
// legacy eth protocols (prior to eth68) have been deprecated.
// Therefore, metadata is always expected in the announcement.
unknownMetas = append(unknownMetas, txMetadata{kind: types[i], size: sizes[i]})
unknownMetas = append(unknownMetas, txMetadata{kind: kinds[i], size: sizes[i]})
}
txAnnounceKnownMeter.Mark(duplicate)
txAnnounceUnderpricedMeter.Mark(underpriced)
@ -285,14 +300,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
// If anything's left to announce, push it into the internal loop
if len(unknownHashes) == 0 {
return nil
return blobFetchHashes, nil
}
announce := &txAnnounce{origin: peer, hashes: unknownHashes, metas: unknownMetas}
select {
case f.notify <- announce:
return nil
return blobFetchHashes, nil
case <-f.quit:
return errTerminated
return nil, errTerminated
}
}
@ -306,26 +321,36 @@ func (f *TxFetcher) isKnownUnderpriced(hash common.Hash) bool {
return ok
}
type deliveryMetrics struct {
inMeter *metrics.Meter
knownMeter *metrics.Meter
underpricedMeter *metrics.Meter
otherRejectMeter *metrics.Meter
}
// Enqueue imports a batch of received transaction into the transaction pool
// and the fetcher. This method may be called by both transaction broadcasts and
// direct request replies. The differentiation is important so the fetcher can
// re-schedule missing transactions as soon as possible.
func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error {
var (
inMeter = txReplyInMeter
knownMeter = txReplyKnownMeter
underpricedMeter = txReplyUnderpricedMeter
otherRejectMeter = txReplyOtherRejectMeter
violation error
)
func (f *TxFetcher) Enqueue(peer string, version uint, txs []*types.Transaction, direct bool) error {
var violation error
metrics := deliveryMetrics{
inMeter: txReplyInMeter,
knownMeter: txReplyKnownMeter,
underpricedMeter: txReplyUnderpricedMeter,
otherRejectMeter: txReplyOtherRejectMeter,
}
if !direct {
inMeter = txBroadcastInMeter
knownMeter = txBroadcastKnownMeter
underpricedMeter = txBroadcastUnderpricedMeter
otherRejectMeter = txBroadcastOtherRejectMeter
metrics = deliveryMetrics{
inMeter: txBroadcastInMeter,
knownMeter: txBroadcastKnownMeter,
underpricedMeter: txBroadcastUnderpricedMeter,
otherRejectMeter: txBroadcastOtherRejectMeter,
}
}
// Keep track of all the propagated transactions
inMeter.Mark(int64(len(txs)))
metrics.inMeter.Mark(int64(len(txs)))
// Push all the transactions into the pool, tracking underpriced ones to avoid
// re-requesting them and dropping the peer in case of malicious transfers.
@ -339,41 +364,41 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool)
if end > len(txs) {
end = len(txs)
}
var (
duplicate int64
underpriced int64
otherreject int64
)
batch := txs[i:end]
var accepted []common.Hash
for j, err := range f.addTxs(batch) {
// Track the transaction hash if the price is too low for us.
// Avoid re-request this transaction when we receive another
// announcement.
if errors.Is(err, txpool.ErrUnderpriced) || errors.Is(err, txpool.ErrReplaceUnderpriced) || errors.Is(err, txpool.ErrTxGasPriceTooLow) {
f.underpriced.Add(batch[j].Hash(), batch[j].Time())
var (
poolTxs []*types.Transaction
blobTxs []*types.Transaction
)
if version >= eth.ETH72 {
for _, tx := range batch {
if tx.Type() == types.BlobTxType {
blobTxs = append(blobTxs, tx)
} else {
poolTxs = append(poolTxs, tx)
}
}
// Track a few interesting failure types
switch {
case err == nil:
} else {
poolTxs = batch
}
batch = append(poolTxs, blobTxs...)
// Add regular tx to pool, blob tx to buffer.
errs := append(f.addTxs(poolTxs), f.buffer.AddTx(blobTxs, peer)...)
hashes := make([]common.Hash, len(batch))
for j := range batch {
hashes[j] = batch[j].Hash()
}
var accepted []common.Hash
for j, err := range errs {
if err == nil {
accepted = append(accepted, batch[j].Hash())
case errors.Is(err, txpool.ErrAlreadyKnown):
duplicate++
case errors.Is(err, txpool.ErrUnderpriced) || errors.Is(err, txpool.ErrReplaceUnderpriced) || errors.Is(err, txpool.ErrTxGasPriceTooLow):
underpriced++
case errors.Is(err, txpool.ErrKZGVerificationError):
}
if errors.Is(err, txpool.ErrKZGVerificationError) || errors.Is(err, txpool.ErrSidecarFormatError) {
// KZG verification failed, terminate transaction processing immediately.
// Since KZG verification is computationally expensive, this acts as a
// defensive measure against potential DoS attacks.
violation = err
default:
otherreject++
}
added = append(added, batch[j].Hash())
metas = append(metas, txMetadata{
@ -386,17 +411,16 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool)
break
}
}
knownMeter.Mark(duplicate)
underpricedMeter.Mark(underpriced)
otherRejectMeter.Mark(otherreject)
otherreject := f.handleAddErrors(hashes, errs, metrics)
// Notify the tracker which txs from this peer were accepted.
if f.onAccepted != nil && len(accepted) > 0 {
f.onAccepted(peer, accepted)
}
// If 'other reject' is >25% of the deliveries in any batch, sleep a bit.
if otherreject > int64((len(batch)+3)/4) {
log.Debug("Peer delivering stale or invalid transactions", "peer", peer, "rejected", otherreject)
// If 'other reject' is >25% of the deliveries in any batch, sleep a bit
// to throttle the misbehaving peer.
if otherreject > int64((len(hashes)+3)/4) {
log.Debug("Peer delivering stale or invalid transactions", "rejected", otherreject)
time.Sleep(200 * time.Millisecond)
}
// If we encountered a protocol violation, disconnect this peer.
@ -412,6 +436,36 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool)
}
}
func (f *TxFetcher) handleAddErrors(txs []common.Hash, errs []error, metrics deliveryMetrics) (otherreject int64) {
var (
duplicate int64
underpriced int64
)
for i, err := range errs {
// Track a few interesting failure types
switch {
case err == nil: // Noop, but need to handle to not count these
case errors.Is(err, txpool.ErrAlreadyKnown):
duplicate++
// Track the transaction hash if the price is too low for us.
// Avoid re-request this transaction when we receive another
// announcement.
case errors.Is(err, txpool.ErrUnderpriced) || errors.Is(err, txpool.ErrReplaceUnderpriced) || errors.Is(err, txpool.ErrTxGasPriceTooLow):
f.underpriced.Add(txs[i], f.realTime())
underpriced++
default:
otherreject++
}
}
metrics.knownMeter.Mark(duplicate)
metrics.underpricedMeter.Mark(underpriced)
metrics.otherRejectMeter.Mark(otherreject)
return otherreject
}
// Drop should be called when a peer disconnects. It cleans up all the internal
// data structures of the given node.
func (f *TxFetcher) Drop(peer string) error {
@ -457,6 +511,14 @@ func (f *TxFetcher) loop() {
}
for {
txs, errs := f.buffer.Flush()
f.handleAddErrors(txs, errs, deliveryMetrics{
inMeter: txReplyInMeter,
knownMeter: txReplyKnownMeter,
underpricedMeter: txReplyUnderpricedMeter,
otherRejectMeter: txReplyOtherRejectMeter,
})
select {
case ann := <-f.notify:
// Drop part of the new announcements if there are too many accumulated.

32
eth/fetcher/tx_fetcher_test.go
View file

@ -28,9 +28,11 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/params"
"github.com/holiman/uint256"
)
@ -60,9 +62,10 @@ type doTxNotify struct {
sizes []uint32
}
type doTxEnqueue struct {
peer string
txs []*types.Transaction
direct bool
peer string
version uint
txs []*types.Transaction
direct bool
}
type doWait struct {
time time.Duration
@ -87,6 +90,16 @@ type txFetcherTest struct {
steps []interface{}
}
// newTestBlobBuffer returns a BlobBuffer with no-op callbacks for tests that
// don't exercise blob handling but still need a non-nil buffer.
func newTestBlobBuffer() *blobpool.BlobBuffer {
return blobpool.NewBlobBuffer(blobpool.BlobBufferFunctions{
ValidateTx: func(*types.Transaction) error { return nil },
AddToPool: func(*blobpool.BlobTxForPool) error { return nil },
DropPeer: func(string) {},
})
}
// newTestTxFetcher creates a tx fetcher with noop callbacks, simulated clock,
// and deterministic randomness.
func newTestTxFetcher() *TxFetcher {
@ -97,7 +110,9 @@ func newTestTxFetcher() *TxFetcher {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil, nil,
nil,
nil,
newTestBlobBuffer(),
)
}
@ -1888,7 +1903,7 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) {
// Process the original or expanded steps
switch step := step.(type) {
case doTxNotify:
if err := fetcher.Notify(step.peer, step.types, step.sizes, step.hashes); err != nil {
if _, err := fetcher.Notify(step.peer, step.types, step.sizes, step.hashes); err != nil {
t.Errorf("step %d: %v", i, err)
}
<-wait // Fetcher needs to process this, wait until it's done
@ -1899,7 +1914,7 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) {
}
case doTxEnqueue:
if err := fetcher.Enqueue(step.peer, step.txs, step.direct); err != nil {
if err := fetcher.Enqueue(step.peer, step.version, step.txs, step.direct); err != nil {
t.Errorf("step %d: %v", i, err)
}
<-wait // Fetcher needs to process this, wait until it's done
@ -2204,6 +2219,7 @@ func TestTransactionForgotten(t *testing.T) {
func(string, []common.Hash) error { return nil },
func(string) {},
nil,
newTestBlobBuffer(),
mockClock,
mockTime,
rand.New(rand.NewSource(0)), // Use fixed seed for deterministic behavior
@ -2220,7 +2236,7 @@ func TestTransactionForgotten(t *testing.T) {
tx2.SetTime(now)
// Initial state: both transactions should be marked as underpriced
if err := fetcher.Enqueue("peer", []*types.Transaction{tx1, tx2}, false); err != nil {
if err := fetcher.Enqueue("peer", eth.ETH70, []*types.Transaction{tx1, tx2}, false); err != nil {
t.Fatal(err)
}
if !fetcher.isKnownUnderpriced(tx1.Hash()) {
@ -2269,7 +2285,7 @@ func TestTransactionForgotten(t *testing.T) {
// Re-enqueue tx1 with updated timestamp
tx1.SetTime(mockTime())
if err := fetcher.Enqueue("peer", []*types.Transaction{tx1}, false); err != nil {
if err := fetcher.Enqueue("peer", eth.ETH70, []*types.Transaction{tx1}, false); err != nil {
t.Fatal(err)
}
if !fetcher.isKnownUnderpriced(tx1.Hash()) {

76
eth/handler.go
View file

@ -32,7 +32,9 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/fetcher"
@ -76,7 +78,7 @@ type txPool interface {
// GetRLP retrieves the RLP-encoded transaction from local txpool
// with given tx hash.
GetRLP(hash common.Hash) []byte
GetRLP(hash common.Hash, version uint) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
@ -98,18 +100,31 @@ type txPool interface {
FilterType(kind byte) bool
}
// blobPool defines the methods needed from a blob pool implementation to
// support cell-based blob data availability.
type blobPool interface {
Has(hash common.Hash) bool
GetBlobHashes(hash common.Hash) []common.Hash
GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error)
GetCustody(hash common.Hash) *types.CustodyBitmap
AddPooledTx(pooledTx *blobpool.BlobTxForPool) error
ValidateTxBasics(pooledTx *types.Transaction) error
}
// handlerConfig is the collection of initialization parameters to create a full
// node network handler.
type handlerConfig struct {
NodeID enode.ID // P2P node ID used for tx propagation topology
Database ethdb.Database // Database for direct sync insertions
Chain *core.BlockChain // Blockchain to serve data from
TxPool txPool // Transaction pool to propagate from
Network uint64 // Network identifier to advertise
Sync ethconfig.SyncMode // Whether to snap or full sync
BloomCache uint64 // Megabytes to alloc for snap sync bloom
RequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges
SnapV2 bool // Whether to advertise and sync via the snap/2 protocol
NodeID enode.ID // P2P node ID used for tx propagation topology
Database ethdb.Database // Database for direct sync insertions
Chain *core.BlockChain // Blockchain to serve data from
TxPool txPool // Transaction pool to propagate from
BlobPool blobPool // Blob pool for cell-based blob data availability
Network uint64 // Network identifier to advertise
Sync ethconfig.SyncMode // Whether to snap or full sync
BloomCache uint64 // Megabytes to alloc for snap sync bloom
RequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges
SnapV2 bool // Whether to advertise and sync via the snap/2 protocol
FetchProbability uint64 // Full blob fetch probability for sparse blobpool (blobFetcher)
}
type handler struct {
@ -119,11 +134,13 @@ type handler struct {
database ethdb.Database
txpool txPool
blobpool blobPool
chain *core.BlockChain
maxPeers int
downloader *downloader.Downloader
txFetcher *fetcher.TxFetcher
blobFetcher *fetcher.BlobFetcher
txTracker *txtracker.Tracker
peers *peerSet
txBroadcastKey [16]byte
@ -150,6 +167,7 @@ func newHandler(config *handlerConfig) (*handler, error) {
networkID: config.Network,
database: config.Database,
txpool: config.TxPool,
blobpool: config.BlobPool,
chain: config.Chain,
peers: newPeerSet(),
txBroadcastKey: newBroadcastChoiceKey(),
@ -172,11 +190,19 @@ func newHandler(config *handlerConfig) (*handler, error) {
}
return p.RequestTxs(hashes)
}
// Construct the blob buffer for assembling blob txs from separate tx and cell deliveries.
blobBuffer := blobpool.NewBlobBuffer(blobpool.BlobBufferFunctions{
ValidateTx: h.blobpool.ValidateTxBasics,
AddToPool: h.blobpool.AddPooledTx,
DropPeer: h.removePeer,
})
addTxs := func(txs []*types.Transaction) []error {
return h.txpool.Add(txs, false)
}
validateMeta := func(tx common.Hash, kind byte) error {
if h.txpool.Has(tx) {
if h.txpool.Has(tx) || blobBuffer.HasTx(tx) {
return txpool.ErrAlreadyKnown
}
if !h.txpool.FilterType(kind) {
@ -185,7 +211,30 @@ func newHandler(config *handlerConfig) (*handler, error) {
return nil
}
h.txTracker = txtracker.New()
h.txFetcher = fetcher.NewTxFetcher(h.chain, validateMeta, addTxs, fetchTx, h.removePeer, h.txTracker.NotifyAccepted)
h.txFetcher = fetcher.NewTxFetcher(h.chain, validateMeta, addTxs, fetchTx, h.removePeer, h.txTracker.NotifyAccepted, blobBuffer)
// Construct the blob fetcher for cell-based blob data availability
blobCallbacks := fetcher.BlobFetcherFunctions{
FetchPayloads: func(peer string, hashes []common.Hash, cells types.CustodyBitmap) error {
p := h.peers.peer(peer)
if p == nil {
return errors.New("unknown peer")
}
return p.RequestPayload(hashes, cells)
},
HasPayload: func(hash common.Hash) bool {
return h.blobpool.Has(hash) || blobBuffer.HasCells(hash)
},
AddCells: func(hash common.Hash, deliveries map[string]*fetcher.PeerCellDelivery, custody types.CustodyBitmap) {
converted := make(map[string]*blobpool.PeerDelivery, len(deliveries))
for peer, d := range deliveries {
converted[peer] = &blobpool.PeerDelivery{Cells: d.Cells, Indices: d.Indices}
}
blobBuffer.AddCells(hash, converted, custody)
},
DropPeer: h.removePeer,
}
h.blobFetcher = fetcher.NewBlobFetcher(blobCallbacks, types.CustodyBitmapAll, nil, config.FetchProbability)
return h, nil
}
@ -406,6 +455,7 @@ func (h *handler) unregisterPeer(id string) {
}
h.downloader.UnregisterPeer(id)
h.txFetcher.Drop(id)
h.blobFetcher.Drop(id)
h.txTracker.NotifyPeerDrop(id)
if err := h.peers.unregisterPeer(id); err != nil {
@ -429,6 +479,7 @@ func (h *handler) Start(maxPeers int) {
// start sync handlers
h.txFetcher.Start()
h.blobFetcher.Start()
// Start the transaction tracker (records tx deliveries, credits peer inclusions).
h.txTracker.Start(h.chain)
@ -442,6 +493,7 @@ func (h *handler) Stop() {
h.txsSub.Unsubscribe() // quits txBroadcastLoop
h.blockRange.stop()
h.txFetcher.Stop()
h.blobFetcher.Stop()
h.downloader.Terminate()
// Quit chainSync and txsync64.

38
eth/handler_eth.go
View file

@ -23,8 +23,10 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/params"
)
// ethHandler implements the eth.Backend interface to handle the various network
@ -33,6 +35,7 @@ type ethHandler handler
func (h *ethHandler) Chain() *core.BlockChain { return h.chain }
func (h *ethHandler) TxPool() eth.TxPool { return h.txpool }
func (h *ethHandler) BlobPool() eth.BlobPool { return h.blobpool }
// RunPeer is invoked when a peer joins on the `eth` protocol.
func (h *ethHandler) RunPeer(peer *eth.Peer, hand eth.Handler) error {
@ -58,8 +61,19 @@ func (h *ethHandler) AcceptTxs() bool {
func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
// Consume any broadcasts and announces, forwarding the rest to the downloader
switch packet := packet.(type) {
case *eth.NewPooledTransactionHashesPacket:
return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
case *eth.NewPooledTransactionHashesPacket72:
hashes, err := h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
if err != nil {
return err
}
if len(hashes) != 0 {
return h.blobFetcher.Notify(peer.ID(), hashes, packet.Mask)
}
return nil
case *eth.NewPooledTransactionHashesPacket71:
_, err := h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
return err
case *eth.TransactionsPacket:
txs, err := packet.Items()
@ -69,7 +83,7 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
if err := handleTransactions(peer, txs, true); err != nil {
return fmt.Errorf("Transactions: %v", err)
}
return h.txFetcher.Enqueue(peer.ID(), txs, false)
return h.txFetcher.Enqueue(peer.ID(), peer.Version(), txs, false)
case *eth.PooledTransactionsPacket:
txs, err := packet.List.Items()
@ -79,7 +93,23 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
if err := handleTransactions(peer, txs, false); err != nil {
return fmt.Errorf("PooledTransactions: %v", err)
}
return h.txFetcher.Enqueue(peer.ID(), txs, true)
return h.txFetcher.Enqueue(peer.ID(), peer.Version(), txs, true)
case *eth.CellsResponse:
outer, err := packet.Cells.Items()
if err != nil {
return fmt.Errorf("Cells: %v", err)
}
cells := make([][]kzg4844.Cell, len(outer))
for i := range outer {
if outer[i].Len() > params.BlobTxMaxBlobs*kzg4844.CellsPerBlob {
return fmt.Errorf("Cells: cells per tx exceeded the possible maximum")
}
if cells[i], err = outer[i].Items(); err != nil {
return fmt.Errorf("Cells: %v", err)
}
}
return h.blobFetcher.Enqueue(peer.ID(), packet.Hashes, cells, packet.Mask)
default:
return fmt.Errorf("unexpected eth packet type: %T", packet)

34
eth/handler_eth_test.go
View file

@ -44,13 +44,14 @@ type testEthHandler struct {
func (h *testEthHandler) Chain() *core.BlockChain { panic("no backing chain") }
func (h *testEthHandler) TxPool() eth.TxPool { panic("no backing tx pool") }
func (h *testEthHandler) BlobPool() eth.BlobPool { return nil }
func (h *testEthHandler) AcceptTxs() bool { return true }
func (h *testEthHandler) RunPeer(*eth.Peer, eth.Handler) error { panic("not used in tests") }
func (h *testEthHandler) PeerInfo(enode.ID) interface{} { panic("not used in tests") }
func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
switch packet := packet.(type) {
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket71:
h.txAnnounces.Send(packet.Hashes)
return nil
@ -105,10 +106,12 @@ func testForkIDSplit(t *testing.T, protocol uint) {
_, blocksNoFork, _ = core.GenerateChainWithGenesis(gspecNoFork, engine, 2, nil)
_, blocksProFork, _ = core.GenerateChainWithGenesis(gspecProFork, engine, 2, nil)
txPool = newTestTxPool()
ethNoFork, _ = newHandler(&handlerConfig{
Database: dbNoFork,
Chain: chainNoFork,
TxPool: newTestTxPool(),
TxPool: txPool,
BlobPool: txPool,
Network: 1,
Sync: ethconfig.FullSync,
BloomCache: 1,
@ -116,7 +119,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
ethProFork, _ = newHandler(&handlerConfig{
Database: dbProFork,
Chain: chainProFork,
TxPool: newTestTxPool(),
TxPool: txPool,
BlobPool: txPool,
Network: 1,
Sync: ethconfig.FullSync,
BloomCache: 1,
@ -137,8 +141,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
peerNoFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil)
peerProFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil)
peerNoFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil, nil)
peerProFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@ -168,8 +172,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil, nil)
peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@ -199,8 +203,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
peerNoFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil)
peerNoFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@ -249,8 +253,8 @@ func testRecvTransactions(t *testing.T, protocol uint) {
defer p2pSrc.Close()
defer p2pSink.Close()
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, nil)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, nil)
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, handler.txpool, nil)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, handler.txpool, nil)
defer src.Close()
defer sink.Close()
@ -305,8 +309,8 @@ func testSendTransactions(t *testing.T, protocol uint) {
defer p2pSrc.Close()
defer p2pSink.Close()
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, nil)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, nil)
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, handler.blobpool, nil)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, handler.blobpool, nil)
defer src.Close()
defer sink.Close()
@ -380,8 +384,8 @@ func testTransactionPropagation(t *testing.T, protocol uint) {
defer sourcePipe.Close()
defer sinkPipe.Close()
sourcePeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{byte(i + 1)}, "", nil, sourcePipe), sourcePipe, source.txpool, nil)
sinkPeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, sink.txpool, nil)
sourcePeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{byte(i + 1)}, "", nil, sourcePipe), sourcePipe, source.txpool, source.txpool, nil)
sinkPeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, sink.txpool, sink.txpool, nil)
defer sourcePeer.Close()
defer sinkPeer.Close()

140
eth/handler_test.go
View file

@ -29,8 +29,10 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/ethdb"
@ -54,7 +56,10 @@ var (
// Its goal is to get around setting up a valid statedb for the balance and nonce
// checks.
type testTxPool struct {
pool map[common.Hash]*types.Transaction // Hash map of collected transactions
txPool map[common.Hash]*types.Transaction // Hash map of collected transactions
cellPool map[common.Hash][]kzg4844.Cell
custody map[common.Hash]types.CustodyBitmap
txFeed event.Feed // Notification feed to allow waiting for inclusion
lock sync.RWMutex // Protects the transaction pool
@ -63,7 +68,9 @@ type testTxPool struct {
// newTestTxPool creates a mock transaction pool.
func newTestTxPool() *testTxPool {
return &testTxPool{
pool: make(map[common.Hash]*types.Transaction),
txPool: make(map[common.Hash]*types.Transaction),
cellPool: make(map[common.Hash][]kzg4844.Cell),
custody: make(map[common.Hash]types.CustodyBitmap),
}
}
@ -73,7 +80,16 @@ func (p *testTxPool) Has(hash common.Hash) bool {
p.lock.Lock()
defer p.lock.Unlock()
return p.pool[hash] != nil
return p.txPool[hash] != nil
}
// Has returns an indicator whether txpool has a transaction
// cached with the given hash.
func (p *testTxPool) HasPayload(hash common.Hash) bool {
p.lock.Lock()
defer p.lock.Unlock()
return p.cellPool[hash] != nil
}
// Get retrieves the transaction from local txpool with given
@ -81,16 +97,16 @@ func (p *testTxPool) Has(hash common.Hash) bool {
func (p *testTxPool) Get(hash common.Hash) *types.Transaction {
p.lock.Lock()
defer p.lock.Unlock()
return p.pool[hash]
return p.txPool[hash]
}
// Get retrieves the transaction from local txpool with given
// tx hash.
func (p *testTxPool) GetRLP(hash common.Hash) []byte {
func (p *testTxPool) GetRLP(hash common.Hash, _ uint) []byte {
p.lock.Lock()
defer p.lock.Unlock()
tx := p.pool[hash]
tx := p.txPool[hash]
if tx != nil {
blob, _ := rlp.EncodeToBytes(tx)
return blob
@ -104,7 +120,7 @@ func (p *testTxPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
p.lock.Lock()
defer p.lock.Unlock()
tx := p.pool[hash]
tx := p.txPool[hash]
if tx != nil {
return &txpool.TxMetadata{
Type: tx.Type(),
@ -121,7 +137,7 @@ func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error {
defer p.lock.Unlock()
for _, tx := range txs {
p.pool[tx.Hash()] = tx
p.txPool[tx.Hash()] = tx
}
p.txFeed.Send(core.NewTxsEvent{Txs: txs})
return make([]error, len(txs))
@ -134,7 +150,7 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]
var count int
batches := make(map[common.Address][]*types.Transaction)
for _, tx := range p.pool {
for _, tx := range p.txPool {
from, _ := types.Sender(types.HomesteadSigner{}, tx)
batches[from] = append(batches[from], tx)
}
@ -164,6 +180,87 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]
func (p *testTxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool) event.Subscription {
return p.txFeed.Subscribe(ch)
}
func (p *testTxPool) GetBlobHashes(hash common.Hash) []common.Hash {
p.lock.RLock()
defer p.lock.RUnlock()
tx, exists := p.txPool[hash]
if !exists {
return nil
}
return tx.BlobHashes()
}
func (p *testTxPool) GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error) {
p.lock.RLock()
defer p.lock.RUnlock()
requestedIndices := mask.Indices()
cells := make([][]*kzg4844.Cell, len(vhashes))
proofs := make([][]*kzg4844.Proof, len(vhashes))
for i, vhash := range vhashes {
// Find the tx containing this versioned hash
var foundTx *types.Transaction
var blobIdx int
for _, tx := range p.txPool {
for j, bh := range tx.BlobHashes() {
if bh == vhash {
foundTx = tx
blobIdx = j
break
}
}
if foundTx != nil {
break
}
}
if foundTx == nil {
continue
}
txCells, ok := p.cellPool[foundTx.Hash()]
if !ok {
continue
}
_ = blobIdx // cells in the mock are stored flat by cell index
blobCells := make([]*kzg4844.Cell, len(requestedIndices))
for j, idx := range requestedIndices {
if int(idx) < len(txCells) {
cell := txCells[idx]
blobCells[j] = &cell
}
}
cells[i] = blobCells
}
return cells, proofs, nil
}
func (p *testTxPool) GetCustody(hash common.Hash) *types.CustodyBitmap {
p.lock.RLock()
defer p.lock.RUnlock()
mask, ok := p.custody[hash]
if !ok {
return nil
}
return &mask
}
// AddCells adds cells for a specific transaction hash (for testing)
func (p *testTxPool) AddCells(hash common.Hash, cells []kzg4844.Cell, mask types.CustodyBitmap) {
p.lock.Lock()
defer p.lock.Unlock()
p.cellPool[hash] = cells
p.custody[hash] = mask
}
func (p *testTxPool) AddPooledTx(pooledTx *blobpool.BlobTxForPool) error {
p.lock.Lock()
defer p.lock.Unlock()
hash := pooledTx.Tx.Hash()
p.cellPool[hash] = pooledTx.CellSidecar.Cells
p.txPool[hash] = pooledTx.Tx
return nil
}
// FilterType should check whether the pool supports the given type of transactions.
func (p *testTxPool) FilterType(kind byte) bool {
@ -174,14 +271,19 @@ func (p *testTxPool) FilterType(kind byte) bool {
return false
}
func (p *testTxPool) ValidateTxBasics(_ *types.Transaction) error {
return nil
}
// testHandler is a live implementation of the Ethereum protocol handler, just
// preinitialized with some sane testing defaults and the transaction pool mocked
// out.
type testHandler struct {
db ethdb.Database
chain *core.BlockChain
txpool *testTxPool
handler *handler
db ethdb.Database
chain *core.BlockChain
txpool *testTxPool
blobpool *testTxPool
handler *handler
}
// newTestHandler creates a new handler for testing purposes with no blocks.
@ -210,6 +312,7 @@ func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler
Database: db,
Chain: chain,
TxPool: txpool,
BlobPool: txpool,
Network: 1,
Sync: mode,
BloomCache: 1,
@ -217,10 +320,11 @@ func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler
handler.Start(1000)
return &testHandler{
db: db,
chain: chain,
txpool: txpool,
handler: handler,
db: db,
chain: chain,
txpool: txpool,
blobpool: txpool,
handler: handler,
}
}
@ -317,7 +421,7 @@ func createTestPeers(rand *rand.Rand, n int) []*ethPeer {
var id enode.ID
rand.Read(id[:])
p2pPeer := p2p.NewPeer(id, "test", nil)
ep := eth.NewPeer(eth.ETH69, p2pPeer, nil, nil, nil)
ep := eth.NewPeer(eth.ETH69, p2pPeer, nil, nil, nil, nil)
peers[i] = &ethPeer{Peer: ep}
}
return peers

39
eth/protocols/eth/broadcast.go
View file

@ -113,29 +113,58 @@ func (p *Peer) announceTransactions() {
pending []common.Hash
pendingTypes []byte
pendingSizes []uint32
mask types.CustodyBitmap
size common.StorageSize
processed = make(map[int]bool)
)
for count = 0; count < len(queue) && size < maxTxPacketSize; count++ {
if meta := p.txpool.GetMetadata(queue[count]); meta != nil {
custody := p.blobpool.GetCustody(queue[count])
if custody != nil {
// Blob txs should be batched into the same announcement
// if they share the same custody.
if mask.OneCount() == 0 {
// This is the first blob tx in this batch, so use its
// custody as a mask in this batch.
mask = *custody
} else {
if mask != *custody {
// Leave this in the queue so that it can be included
// in a later announcement.
continue
}
}
}
pending = append(pending, queue[count])
pendingTypes = append(pendingTypes, meta.Type)
pendingSizes = append(pendingSizes, uint32(meta.Size))
if p.version >= ETH72 && meta.SizeWithoutBlob > 0 {
pendingSizes = append(pendingSizes, uint32(meta.SizeWithoutBlob))
} else {
pendingSizes = append(pendingSizes, uint32(meta.Size))
}
size += common.HashLength
}
processed[count] = true
}
// Shift and trim queue
queue = queue[:copy(queue, queue[count:])]
// Shift and trim queue using processed map
var remaining []common.Hash
for i, h := range queue {
if !processed[i] {
remaining = append(remaining, h)
}
}
queue = remaining
// If there's anything available to transfer, fire up an async writer
if len(pending) > 0 {
done = make(chan struct{})
go func() {
if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes); err != nil {
if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes, mask); err != nil {
fail <- err
return
}
close(done)
p.Log().Trace("Sent transaction announcements", "count", len(pending))
p.Log().Trace("Sent transaction announcements", "count", len(pending), "mask", mask, "tx", pending)
}()
}
}

40
eth/protocols/eth/handler.go
View file

@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
@ -71,6 +72,9 @@ type Backend interface {
// TxPool retrieves the transaction pool object to serve data.
TxPool() TxPool
// BlobPool retrieves the blob pool object to serve cell requests.
BlobPool() BlobPool
// AcceptTxs retrieves whether transaction processing is enabled on the node
// or if inbound transactions should simply be dropped.
AcceptTxs() bool
@ -90,6 +94,18 @@ type Backend interface {
Handle(peer *Peer, packet Packet) error
}
// BlobPool defines the methods needed by the protocol handler to serve cell requests.
type BlobPool interface {
// GetBlobHashes returns the blob versioned hashes for a given transaction hash.
GetBlobHashes(hash common.Hash) []common.Hash
// GetBlobCells retrieves cells and proofs for given versioned blob hashes filtered by the custody bitmap.
GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error)
// GetCustody returns the custody bitmap for a given transaction hash.
GetCustody(hash common.Hash) *types.CustodyBitmap
// Has returns whether the blob pool contains a transaction with the given hash.
Has(hash common.Hash) bool
}
// TxPool defines the methods needed by the protocol handler to serve transactions.
type TxPool interface {
// Get retrieves the transaction from the local txpool with the given hash.
@ -97,7 +113,7 @@ type TxPool interface {
// GetRLP retrieves the RLP-encoded transaction from the local txpool with
// the given hash.
GetRLP(hash common.Hash) []byte
GetRLP(hash common.Hash, version uint) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
@ -113,7 +129,7 @@ func MakeProtocols(backend Backend, network uint64, disc enode.Iterator) []p2p.P
Version: version,
Length: protocolLengths[version],
Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
peer := NewPeer(version, p, rw, backend.TxPool(), backend.Chain().Config())
peer := NewPeer(version, p, rw, backend.TxPool(), backend.BlobPool(), backend.Chain().Config())
defer peer.Close()
return backend.RunPeer(peer, func(peer *Peer) error {
@ -216,6 +232,24 @@ var eth71 = map[uint64]msgHandler{
BlockAccessListsMsg: handleBlockAccessLists,
}
var eth72 = map[uint64]msgHandler{
TransactionsMsg: handleTransactions,
NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes72,
GetBlockHeadersMsg: handleGetBlockHeaders,
BlockHeadersMsg: handleBlockHeaders,
GetBlockBodiesMsg: handleGetBlockBodies,
BlockBodiesMsg: handleBlockBodies,
GetReceiptsMsg: handleGetReceipts70,
ReceiptsMsg: handleReceipts70,
GetPooledTransactionsMsg: handleGetPooledTransactions,
PooledTransactionsMsg: handlePooledTransactions,
BlockRangeUpdateMsg: handleBlockRangeUpdate,
GetBlockAccessListsMsg: handleGetBlockAccessLists,
BlockAccessListsMsg: handleBlockAccessLists,
GetCellsMsg: handleGetCells,
CellsMsg: handleCells,
}
// handleMessage is invoked whenever an inbound message is received from a remote
// peer. The remote connection is torn down upon returning any error.
func handleMessage(backend Backend, peer *Peer) error {
@ -237,6 +271,8 @@ func handleMessage(backend Backend, peer *Peer) error {
handlers = eth70
case ETH71:
handlers = eth71
case ETH72:
handlers = eth72
default:
return fmt.Errorf("unknown eth protocol version: %v", peer.version)
}

15
eth/protocols/eth/handler_test.go
View file

@ -63,9 +63,10 @@ func u64(val uint64) *uint64 { return &val }
// purpose is to allow testing the request/reply workflows and wire serialization
// in the `eth` protocol without actually doing any data processing.
type testBackend struct {
db ethdb.Database
chain *core.BlockChain
txpool *txpool.TxPool
db ethdb.Database
chain *core.BlockChain
txpool *txpool.TxPool
blobpool *blobpool.BlobPool
}
// newTestBackend creates an empty chain and wraps it into a mock backend.
@ -143,9 +144,10 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, cancun bool, generat
txpool, _ := txpool.New(txconfig.PriceLimit, chain, []txpool.SubPool{legacyPool, blobPool})
return &testBackend{
db: db,
chain: chain,
txpool: txpool,
db: db,
chain: chain,
txpool: txpool,
blobpool: blobPool,
}
}
@ -157,6 +159,7 @@ func (b *testBackend) close() {
func (b *testBackend) Chain() *core.BlockChain { return b.chain }
func (b *testBackend) TxPool() TxPool { return b.txpool }
func (b *testBackend) BlobPool() BlobPool { return b.blobpool }
func (b *testBackend) RunPeer(peer *Peer, handler Handler) error {
// Normally the backend would do peer maintenance and handshakes. All that

103
eth/protocols/eth/handlers.go
View file

@ -26,6 +26,7 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/tracker"
"github.com/ethereum/go-ethereum/rlp"
@ -568,7 +569,27 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer)
if !backend.AcceptTxs() {
return nil
}
ann := new(NewPooledTransactionHashesPacket)
ann := new(NewPooledTransactionHashesPacket71)
if err := msg.Decode(ann); err != nil {
return err
}
if len(ann.Hashes) != len(ann.Types) || len(ann.Hashes) != len(ann.Sizes) {
return fmt.Errorf("NewPooledTransactionHashes: invalid len of fields in %v %v %v", len(ann.Hashes), len(ann.Types), len(ann.Sizes))
}
// Schedule all the unknown hashes for retrieval
for _, hash := range ann.Hashes {
peer.MarkTransaction(hash)
}
return backend.Handle(peer, ann)
}
func handleNewPooledTransactionHashes72(backend Backend, msg Decoder, peer *Peer) error {
// New transaction announcement arrived, make sure we have
// a valid and fresh chain to handle them
if !backend.AcceptTxs() {
return nil
}
ann := new(NewPooledTransactionHashesPacket72)
if err := msg.Decode(ann); err != nil {
return err
}
@ -588,11 +609,11 @@ func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error
if err := msg.Decode(&query); err != nil {
return err
}
hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsRequest)
hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsRequest, peer.version)
return peer.ReplyPooledTransactionsRLP(query.RequestId, hashes, txs)
}
func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsRequest) ([]common.Hash, []rlp.RawValue) {
func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsRequest, version uint) ([]common.Hash, []rlp.RawValue) {
// Gather transactions until the fetch or network limits is reached
var (
bytes int
@ -604,7 +625,7 @@ func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsReq
break
}
// Retrieve the requested transaction, skipping if unknown to us
encoded := backend.TxPool().GetRLP(hash)
encoded := backend.TxPool().GetRLP(hash, version)
if len(encoded) == 0 {
continue
}
@ -667,6 +688,80 @@ func handleBlockRangeUpdate(backend Backend, msg Decoder, peer *Peer) error {
return nil
}
func handleGetCells(backend Backend, msg Decoder, peer *Peer) error {
// Decode the cell retrieval message
var query GetCellsRequestPacket
if err := msg.Decode(&query); err != nil {
return err
}
hashes, cells, custody := answerGetCells(backend, query.GetCellsRequest)
return peer.ReplyCells(query.RequestId, hashes, cells, custody)
}
func answerGetCells(backend Backend, query GetCellsRequest) ([]common.Hash, [][]kzg4844.Cell, types.CustodyBitmap) {
var (
cellCounts int
hashes []common.Hash
cells [][]kzg4844.Cell
)
maxCells := softResponseLimit / 2048
for _, hash := range query.Hashes {
if cellCounts >= maxCells {
break
}
// Look up the blob versioned hashes for this transaction
vhashes := backend.BlobPool().GetBlobHashes(hash)
if len(vhashes) == 0 {
continue
}
blobCells, _, _ := backend.BlobPool().GetBlobCells(vhashes, query.Mask)
// Flatten per-blob cells into a single slice. If any blob has a nil
// entry (unavailable cell), skip the entire transaction.
var flat []kzg4844.Cell
skip := false
for _, bc := range blobCells {
if bc == nil {
skip = true
break
}
for _, c := range bc {
if c == nil {
skip = true
break
}
flat = append(flat, *c)
}
if skip {
break
}
}
if skip || len(flat) == 0 {
continue
}
hashes = append(hashes, hash)
cells = append(cells, flat)
cellCounts += len(flat)
}
return hashes, cells, query.Mask
}
func handleCells(backend Backend, msg Decoder, peer *Peer) error {
var cellsResponse CellsPacket
if err := msg.Decode(&cellsResponse); err != nil {
return err
}
tresp := tracker.Response{
ID: cellsResponse.RequestId,
MsgCode: CellsMsg,
Size: cellsResponse.CellsResponse.Cells.Len(),
}
if err := peer.tracker.Fulfil(tresp); err != nil {
return fmt.Errorf("Cells: %w", err)
}
return backend.Handle(peer, &cellsResponse.CellsResponse)
}
// handleGetBlockAccessLists serves a GetBlockAccessLists request.
func handleGetBlockAccessLists(backend Backend, msg Decoder, peer *Peer) error {
var query GetBlockAccessListsPacket

2
eth/protocols/eth/handshake_test.go
View file

@ -77,7 +77,7 @@ func testHandshake(t *testing.T, protocol uint) {
defer app.Close()
defer net.Close()
peer := NewPeer(protocol, p2p.NewPeer(enode.ID{}, "peer", nil), net, nil, nil)
peer := NewPeer(protocol, p2p.NewPeer(enode.ID{}, "peer", nil), net, nil, nil, nil)
defer peer.Close()
// Send the junk test with one peer, check the handshake failure

66
eth/protocols/eth/peer.go
View file

@ -27,6 +27,7 @@ import (
mapset "github.com/deckarep/golang-set/v2"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/tracker"
"github.com/ethereum/go-ethereum/params"
@ -66,7 +67,8 @@ type Peer struct {
version uint // Protocol version negotiated
lastRange atomic.Pointer[BlockRangeUpdatePacket]
txpool TxPool // Transaction pool used by the broadcasters for liveness checks
txpool TxPool // Transaction pool used by the broadcasters for liveness checks
blobpool BlobPool
knownTxs *knownCache // Set of transaction hashes known to be known by this peer
txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests
txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests
@ -86,11 +88,11 @@ type Peer struct {
// NewPeer creates a wrapper for a network connection and negotiated protocol
// version.
func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool, chainConfig *params.ChainConfig) *Peer {
func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool, blobpool BlobPool, chainConfig *params.ChainConfig) *Peer {
cap := p2p.Cap{Name: ProtocolName, Version: version}
id := p.ID().String()
peer := &Peer{
id: p.ID().String(),
id: id,
Peer: p,
rw: rw,
version: version,
@ -102,6 +104,7 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool, cha
reqCancel: make(chan *cancel),
resDispatch: make(chan *response),
txpool: txpool,
blobpool: blobpool,
chainConfig: chainConfig,
receiptBuffer: make(map[uint64]*receiptRequest),
term: make(chan struct{}),
@ -188,13 +191,13 @@ func (p *Peer) AsyncSendTransactions(hashes []common.Hash) {
// This method is a helper used by the async transaction announcer. Don't call it
// directly as the queueing (memory) and transmission (bandwidth) costs should
// not be managed directly.
func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32) error {
if err := p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket{Types: types, Sizes: sizes, Hashes: hashes}); err != nil {
return err
}
func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32, cells types.CustodyBitmap) error {
// Mark all the transactions as known, but ensure we don't overflow our limits
p.knownTxs.Add(hashes...)
return nil
if p.version >= ETH72 {
return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket72{Types: types, Sizes: sizes, Hashes: hashes, Mask: cells})
}
return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket71{Types: types, Sizes: sizes, Hashes: hashes})
}
// AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually
@ -248,6 +251,53 @@ func (p *Peer) ReplyReceiptsRLP69(id uint64, receipts rlp.RawList[*ReceiptList])
})
}
// ReplyCells is the response to GetCells.
func (p *Peer) ReplyCells(id uint64, hashes []common.Hash, cells [][]kzg4844.Cell, mask types.CustodyBitmap) error {
inner := make([]rlp.RawList[kzg4844.Cell], len(cells))
for i, c := range cells {
raw, err := rlp.EncodeToRawList(c)
if err != nil {
return err
}
inner[i] = raw
}
rawCells, err := rlp.EncodeToRawList(inner)
if err != nil {
return err
}
return p2p.Send(p.rw, CellsMsg, &CellsPacket{
RequestId: id,
CellsResponse: CellsResponse{
Hashes: hashes,
Cells: rawCells,
Mask: mask,
},
})
}
// RequestPayload fetches a batch of cells from a remote node.
func (p *Peer) RequestPayload(hashes []common.Hash, cell types.CustodyBitmap) error {
p.Log().Debug("Fetching batch of cells", "txcount", len(hashes), "cellcount", cell.OneCount())
id := rand.Uint64()
err := p.tracker.Track(tracker.Request{
ID: id,
ReqCode: GetCellsMsg,
RespCode: CellsMsg,
Size: len(hashes),
})
if err != nil {
return err
}
return p2p.Send(p.rw, GetCellsMsg, &GetCellsRequestPacket{
RequestId: id,
GetCellsRequest: GetCellsRequest{
Hashes: hashes,
Mask: cell,
},
})
}
// ReplyReceiptsRLP70 is the response to GetReceipts.
func (p *Peer) ReplyReceiptsRLP70(id uint64, receipts rlp.RawList[*ReceiptList], lastBlockIncomplete bool) error {
return p2p.Send(p.rw, ReceiptsMsg, &ReceiptsPacket70{

2
eth/protocols/eth/peer_test.go
View file

@ -45,7 +45,7 @@ func newTestPeer(name string, version uint, backend Backend) (*testPeer, <-chan
var id enode.ID
rand.Read(id[:])
peer := NewPeer(version, p2p.NewPeer(id, name, nil), net, backend.TxPool(), nil)
peer := NewPeer(version, p2p.NewPeer(id, name, nil), net, backend.TxPool(), backend.BlobPool(), nil)
errc := make(chan error, 1)
go func() {
defer app.Close()

60
eth/protocols/eth/protocol.go
View file

@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/forkid"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/rlp"
)
@ -32,6 +33,7 @@ const (
ETH69 = 69
ETH70 = 70
ETH71 = 71
ETH72 = 72
)
// ProtocolName is the official short name of the `eth` protocol used during
@ -40,11 +42,11 @@ const ProtocolName = "eth"
// ProtocolVersions are the supported versions of the `eth` protocol (first
// is primary).
var ProtocolVersions = []uint{ETH71, ETH70, ETH69}
var ProtocolVersions = []uint{ETH72, ETH71, ETH70, ETH69}
// protocolLengths are the number of implemented message corresponding to
// different protocol versions.
var protocolLengths = map[uint]uint64{ETH71: 20, ETH69: 18, ETH70: 18}
var protocolLengths = map[uint]uint64{ETH69: 18, ETH70: 18, ETH71: 20, ETH72: 22}
// maxMessageSize is the maximum cap on the size of a protocol message.
const maxMessageSize = 10 * 1024 * 1024
@ -69,6 +71,8 @@ const (
BlockRangeUpdateMsg = 0x11
GetBlockAccessListsMsg = 0x12
BlockAccessListsMsg = 0x13
GetCellsMsg = 0x14
CellsMsg = 0x15
)
var (
@ -248,13 +252,23 @@ type ReceiptsPacket70 struct {
// ReceiptsRLPResponse is used for receipts, when we already have it encoded
type ReceiptsRLPResponse []rlp.RawValue
// NewPooledTransactionHashesPacket represents a transaction announcement packet on eth/68 and newer.
type NewPooledTransactionHashesPacket struct {
// NewPooledTransactionHashesPacket71 represents a transaction announcement packet on protocol version
// less than or equal to 71.
type NewPooledTransactionHashesPacket71 struct {
Types []byte
Sizes []uint32
Hashes []common.Hash
}
// NewPooledTransactionHashesPacket72 represents a transaction announcement packet on ETH/72
// with an additional custody bitmap field for cell-based blob data availability.
type NewPooledTransactionHashesPacket72 struct {
Types []byte
Sizes []uint32
Hashes []common.Hash
Mask types.CustodyBitmap
}
// GetPooledTransactionsRequest represents a transaction query.
type GetPooledTransactionsRequest []common.Hash
@ -291,6 +305,31 @@ type BlockRangeUpdatePacket struct {
LatestBlockHash common.Hash
}
// GetCellsRequest represents a request for cells of blob transactions.
type GetCellsRequest struct {
Hashes []common.Hash
Mask types.CustodyBitmap
}
// GetCellsRequestPacket represents a cell request with request ID wrapping.
type GetCellsRequestPacket struct {
RequestId uint64
GetCellsRequest
}
// CellsResponse represents a response containing cells for blob transactions.
type CellsResponse struct {
Hashes []common.Hash
Cells rlp.RawList[rlp.RawList[kzg4844.Cell]]
Mask types.CustodyBitmap
}
// CellsPacket represents a cells response with request ID wrapping.
type CellsPacket struct {
RequestId uint64
CellsResponse
}
type GetBlockAccessListsRequest []common.Hash
type GetBlockAccessListsPacket struct {
@ -326,8 +365,11 @@ func (*GetBlockBodiesRequest) Kind() byte { return GetBlockBodiesMsg }
func (*BlockBodiesResponse) Name() string { return "BlockBodies" }
func (*BlockBodiesResponse) Kind() byte { return BlockBodiesMsg }
func (*NewPooledTransactionHashesPacket) Name() string { return "NewPooledTransactionHashes" }
func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransactionHashesMsg }
func (*NewPooledTransactionHashesPacket71) Name() string { return "NewPooledTransactionHashes" }
func (*NewPooledTransactionHashesPacket71) Kind() byte { return NewPooledTransactionHashesMsg }
func (*NewPooledTransactionHashesPacket72) Name() string { return "NewPooledTransactionHashes" }
func (*NewPooledTransactionHashesPacket72) Kind() byte { return NewPooledTransactionHashesMsg }
func (*GetPooledTransactionsRequest) Name() string { return "GetPooledTransactions" }
func (*GetPooledTransactionsRequest) Kind() byte { return GetPooledTransactionsMsg }
@ -352,3 +394,9 @@ func (*GetBlockAccessListsRequest) Kind() byte { return GetBlockAccessListsMsg
func (*BlockAccessListResponse) Name() string { return "BlockAccessLists" }
func (*BlockAccessListResponse) Kind() byte { return BlockAccessListsMsg }
func (*GetCellsRequest) Name() string { return "GetCells" }
func (*GetCellsRequest) Kind() byte { return GetCellsMsg }
func (*CellsResponse) Name() string { return "Cells" }
func (*CellsResponse) Kind() byte { return CellsMsg }

4
eth/sync_test.go
View file

@ -50,8 +50,8 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) {
defer emptyPipeEth.Close()
defer fullPipeEth.Close()
emptyPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{1}, "", caps), emptyPipeEth, empty.txpool, nil)
fullPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{2}, "", caps), fullPipeEth, full.txpool, nil)
emptyPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{1}, "", caps), emptyPipeEth, empty.txpool, empty.blobpool, nil)
fullPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{2}, "", caps), fullPipeEth, full.txpool, full.blobpool, nil)
defer emptyPeerEth.Close()
defer fullPeerEth.Close()

22
tests/fuzzers/txfetcher/txfetcher_fuzzer.go
View file

@ -25,22 +25,28 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/eth/fetcher"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
)
var (
peers []string
txs []*types.Transaction
peers []string
peerVersions map[string]uint
txs []*types.Transaction
)
func init() {
// Random is nice, but we need it deterministic
rand := rand.New(rand.NewSource(0x3a29))
supportedVersions := []uint{eth.ETH69, eth.ETH70, eth.ETH72}
peers = make([]string, 10)
peerVersions = make(map[string]uint, len(peers))
for i := 0; i < len(peers); i++ {
peers[i] = fmt.Sprintf("Peer #%d", i)
peerVersions[peers[i]] = supportedVersions[i%len(supportedVersions)]
}
txs = make([]*types.Transaction, 65536) // We need to bump enough to hit all the limits
for i := 0; i < len(txs); i++ {
@ -84,7 +90,13 @@ func fuzz(input []byte) int {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
nil, nil,
nil,
nil,
blobpool.NewBlobBuffer(blobpool.BlobBufferFunctions{
ValidateTx: func(*types.Transaction) error { return nil },
AddToPool: func(*blobpool.BlobTxForPool) error { return nil },
DropPeer: func(string) {},
}),
clock,
func() time.Time {
nanoTime := int64(clock.Now())
@ -139,7 +151,7 @@ func fuzz(input []byte) int {
if verbose {
fmt.Println("Notify", peer, announceIdxs)
}
if err := f.Notify(peer, types, sizes, announces); err != nil {
if _, err := f.Notify(peer, types, sizes, announces); err != nil {
panic(err)
}
@ -180,7 +192,7 @@ func fuzz(input []byte) int {
if verbose {
fmt.Println("Enqueue", peer, deliverIdxs, direct)
}
if err := f.Enqueue(peer, deliveries, direct); err != nil {
if err := f.Enqueue(peer, peerVersions[peer], deliveries, direct); err != nil {
panic(err)
}