diff --git a/beacon/engine/types.go b/beacon/engine/types.go
index 9af2af31c6..9d820ac772 100644
--- a/beacon/engine/types.go
+++ b/beacon/engine/types.go
@@ -173,6 +173,11 @@ type BlobAndProofV2 struct {
CellProofs []hexutil.Bytes `json:"proofs"` // proofs MUST contain exactly CELLS_PER_EXT_BLOB cell proofs.
}
+type BlobCellsAndProofsV1 struct {
+ BlobCells []*hexutil.Bytes `json:"blob_cells"`
+ Proofs []*hexutil.Bytes `json:"proofs"`
+}
+
// BlobAndProofListV2 is a list of BlobAndProofV2 with a hand-rolled JSON marshaler
// that avoids the overhead of encoding/json for large blob payloads.
type BlobAndProofListV2 []*BlobAndProofV2
diff --git a/cmd/devp2p/internal/ethtest/conn.go b/cmd/devp2p/internal/ethtest/conn.go
index fb655eba18..9c402d161e 100644
--- a/cmd/devp2p/internal/ethtest/conn.go
+++ b/cmd/devp2p/internal/ethtest/conn.go
@@ -181,7 +181,7 @@ func (c *Conn) ReadEth() (any, error) {
case eth.TransactionsMsg:
msg = new(eth.TransactionsPacket)
case eth.NewPooledTransactionHashesMsg:
- msg = new(eth.NewPooledTransactionHashesPacket)
+ msg = new(eth.NewPooledTransactionHashesPacket71)
case eth.GetPooledTransactionsMsg:
msg = new(eth.GetPooledTransactionsPacket)
case eth.PooledTransactionsMsg:
diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go
index fa883806c8..37ba594f71 100644
--- a/cmd/devp2p/internal/ethtest/suite.go
+++ b/cmd/devp2p/internal/ethtest/suite.go
@@ -976,7 +976,7 @@ the transactions using a GetPooledTransactions request.`)
}
// Send announcement.
- ann := eth.NewPooledTransactionHashesPacket{Types: txTypes, Sizes: sizes, Hashes: hashes}
+ ann := eth.NewPooledTransactionHashesPacket71{Types: txTypes, Sizes: sizes, Hashes: hashes}
err = conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann)
if err != nil {
t.Fatalf("failed to write to connection: %v", err)
@@ -994,7 +994,7 @@ the transactions using a GetPooledTransactions request.`)
t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsRequest))
}
return
- case *eth.NewPooledTransactionHashesPacket:
+ case *eth.NewPooledTransactionHashesPacket71:
continue
case *eth.TransactionsPacket:
continue
@@ -1060,12 +1060,12 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
t2 = s.makeBlobTxs(2, 3, 0x2)
)
for _, test := range []struct {
- ann eth.NewPooledTransactionHashesPacket
+ ann eth.NewPooledTransactionHashesPacket71
resp eth.PooledTransactionsResponse
}{
// Invalid tx size.
{
- ann: eth.NewPooledTransactionHashesPacket{
+ ann: eth.NewPooledTransactionHashesPacket71{
Types: []byte{types.BlobTxType, types.BlobTxType},
Sizes: []uint32{uint32(t1[0].Size()), uint32(t1[1].Size() + 10)},
Hashes: []common.Hash{t1[0].Hash(), t1[1].Hash()},
@@ -1074,7 +1074,7 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
},
// Wrong tx type.
{
- ann: eth.NewPooledTransactionHashesPacket{
+ ann: eth.NewPooledTransactionHashesPacket71{
Types: []byte{types.DynamicFeeTxType, types.BlobTxType},
Sizes: []uint32{uint32(t2[0].Size()), uint32(t2[1].Size())},
Hashes: []common.Hash{t2[0].Hash(), t2[1].Hash()},
@@ -1203,7 +1203,7 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types
return
}
- ann := eth.NewPooledTransactionHashesPacket{
+ ann := eth.NewPooledTransactionHashesPacket71{
Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(badTx.Size())},
Hashes: []common.Hash{badTx.Hash()},
@@ -1254,7 +1254,7 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types
return
}
- ann := eth.NewPooledTransactionHashesPacket{
+ ann := eth.NewPooledTransactionHashesPacket71{
Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(tx.Size())},
Hashes: []common.Hash{tx.Hash()},
diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go
index 855e799b4f..186d747b84 100644
--- a/cmd/devp2p/internal/ethtest/transaction.go
+++ b/cmd/devp2p/internal/ethtest/transaction.go
@@ -74,7 +74,7 @@ func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error {
for _, tx := range txs {
got[tx.Hash()] = true
}
- case *eth.NewPooledTransactionHashesPacket:
+ case *eth.NewPooledTransactionHashesPacket71:
for _, hash := range msg.Hashes {
got[hash] = true
}
@@ -164,7 +164,7 @@ func (s *Suite) sendInvalidTxs(t *utesting.T, txs []*types.Transaction) error {
return fmt.Errorf("received bad tx: %s", tx.Hash())
}
}
- case *eth.NewPooledTransactionHashesPacket:
+ case *eth.NewPooledTransactionHashesPacket71:
for _, hash := range msg.Hashes {
if _, ok := invalids[hash]; ok {
return fmt.Errorf("received bad tx: %s", hash)
diff --git a/cmd/geth/main.go b/cmd/geth/main.go
index b5802f51e7..4aadb5fb87 100644
--- a/cmd/geth/main.go
+++ b/cmd/geth/main.go
@@ -76,6 +76,7 @@ var (
utils.BlobPoolDataDirFlag,
utils.BlobPoolDataCapFlag,
utils.BlobPoolPriceBumpFlag,
+ utils.BlobPoolFetchProbabilityFlag,
utils.SyncModeFlag,
utils.SyncTargetFlag,
utils.ExitWhenSyncedFlag,
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
index 8b06772d87..f264ed58e8 100644
--- a/cmd/utils/flags.go
+++ b/cmd/utils/flags.go
@@ -48,6 +48,7 @@ import (
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth"
"github.com/ethereum/go-ethereum/eth/ethconfig"
+ "github.com/ethereum/go-ethereum/eth/fetcher"
"github.com/ethereum/go-ethereum/eth/filters"
"github.com/ethereum/go-ethereum/eth/gasprice"
"github.com/ethereum/go-ethereum/eth/syncer"
@@ -502,6 +503,12 @@ var (
Value: ethconfig.Defaults.BlobPool.PriceBump,
Category: flags.BlobPoolCategory,
}
+ BlobPoolFetchProbabilityFlag = &cli.Uint64Flag{
+ Name: "blobpool.fetchprobability",
+ Usage: "Probability of fetching the full blob payload for sparse blobpool (min=15, max=100)",
+ Value: fetcher.DefaultFetchProbability,
+ Category: flags.BlobPoolCategory,
+ }
// Performance tuning settings
CacheFlag = &cli.IntFlag{
Name: "cache",
@@ -1695,6 +1702,9 @@ func setBlobPool(ctx *cli.Context, cfg *blobpool.Config) {
if ctx.IsSet(BlobPoolPriceBumpFlag.Name) {
cfg.PriceBump = ctx.Uint64(BlobPoolPriceBumpFlag.Name)
}
+ if ctx.IsSet(BlobPoolFetchProbabilityFlag.Name) {
+ cfg.FetchProbability = ctx.Uint64(BlobPoolFetchProbabilityFlag.Name)
+ }
}
func setMiner(ctx *cli.Context, cfg *miner.Config) {
diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go
index ee73c8e897..c013b7b1e3 100644
--- a/core/txpool/blobpool/blobpool.go
+++ b/core/txpool/blobpool/blobpool.go
@@ -129,9 +129,12 @@ type blobTxMeta struct {
announced bool // Whether the tx has been announced to listeners
- id uint64 // Storage ID in the pool's persistent store
- storageSize uint32 // Byte size in the pool's persistent store
- size uint64 // RLP-encoded size of transaction including the attached blob
+ id uint64 // Storage ID in the pool's persistent store
+ storageSize uint32 // Byte size in the pool's persistent store
+ size uint64 // RLP-encoded size of transaction including the attached blob
+ sizeWithoutBlob uint64 // RLP-encoded size of transaction without blob data (for ETH/72)
+
+ custody *types.CustodyBitmap
nonce uint64 // Needed to prioritize inclusion order within an account
costCap *uint256.Int // Needed to validate cumulative balance sufficiency
@@ -149,70 +152,109 @@ type blobTxMeta struct {
evictionBlobFeeJumps float64 // Worse blob fee (converted to fee jumps) across all previous nonces
}
-// blobTxForPool is the storage representation of a blob transaction in the
+// BlobTxForPool is the storage representation of a blob transaction in the
// blobpool.
-type blobTxForPool struct {
+type BlobTxForPool struct {
Tx *types.Transaction // tx without sidecar
- Version byte
- Commitments []kzg4844.Commitment
- Proofs []kzg4844.Proof
- Blobs []kzg4844.Blob
+ CellSidecar *types.BlobTxCellSidecar
}
-// Sidecar returns BlobTxSidecar of ptx.
-func (ptx *blobTxForPool) Sidecar() *types.BlobTxSidecar {
- return types.NewBlobTxSidecar(ptx.Version, ptx.Blobs, ptx.Commitments, ptx.Proofs)
+// Sidecar returns BlobTxSidecar of pooled transaction. Since this function
+// recovers the blob field in sidecar, it is expansive and needs to be
+// avoided if possible. Returns error if recovery fails (e.g. insufficient cells).
+func (ptx *BlobTxForPool) sidecar() (*types.BlobTxSidecar, error) {
+ sidecar := ptx.CellSidecar
+ blobs, err := kzg4844.RecoverBlobs(sidecar.Cells, sidecar.Custody.Indices())
+ if err != nil {
+ return nil, err
+ }
+ return types.NewBlobTxSidecar(sidecar.Version, blobs, sidecar.Commitments, sidecar.Proofs), nil
}
// TxSize returns the transaction size on the network without
// reconstructing the transaction.
-func (ptx *blobTxForPool) TxSize() uint64 {
- var blobs, commitments, proofs uint64
- for i := range ptx.Blobs {
- blobs += rlp.BytesSize(ptx.Blobs[i][:])
+func (ptx *BlobTxForPool) txSize() uint64 {
+ sidecar := ptx.CellSidecar
+
+ var commitments, proofs uint64
+ for i := range sidecar.Commitments {
+ commitments += rlp.BytesSize(sidecar.Commitments[i][:])
}
- for i := range ptx.Commitments {
- commitments += rlp.BytesSize(ptx.Commitments[i][:])
- }
- for i := range ptx.Proofs {
- proofs += rlp.BytesSize(ptx.Proofs[i][:])
+ for i := range sidecar.Proofs {
+ proofs += rlp.BytesSize(sidecar.Proofs[i][:])
}
+ var blob kzg4844.Blob
+ blobs := uint64(len(sidecar.Commitments)) * rlp.BytesSize(blob[:])
return ptx.Tx.Size() + rlp.ListSize(rlp.ListSize(blobs)+rlp.ListSize(commitments)+rlp.ListSize(proofs))
}
+func (ptx *BlobTxForPool) txSizeWithoutBlob() uint64 {
+ sidecar := ptx.CellSidecar
+
+ var commitments, proofs uint64
+ for i := range sidecar.Commitments {
+ commitments += rlp.BytesSize(sidecar.Commitments[i][:])
+ }
+ for i := range sidecar.Proofs {
+ proofs += rlp.BytesSize(sidecar.Proofs[i][:])
+ }
+ return ptx.Tx.Size() + rlp.ListSize(rlp.ListSize(0)+rlp.ListSize(commitments)+rlp.ListSize(proofs))
+}
+
// ToTx reconstructs a full Transaction with the sidecar attached.
-func (ptx *blobTxForPool) ToTx() *types.Transaction {
- return ptx.Tx.WithBlobTxSidecar(ptx.Sidecar())
+func (ptx *BlobTxForPool) toTx() (*types.Transaction, error) {
+ sc, err := ptx.sidecar()
+ if err != nil {
+ return nil, err
+ }
+ return ptx.Tx.WithBlobTxSidecar(sc), nil
}
// newBlobTxForPool decomposes a blob transaction into blobTxForPool type.
-func newBlobTxForPool(tx *types.Transaction) *blobTxForPool {
+func newBlobTxForPool(tx *types.Transaction) (*BlobTxForPool, error) {
sc := tx.BlobTxSidecar()
if sc == nil {
- panic("missing blob tx sidecar")
+ return nil, errors.New("missing blob tx sidecar")
}
- return &blobTxForPool{
- Tx: tx.WithoutBlobTxSidecar(),
+ cells, err := kzg4844.ComputeCells(sc.Blobs)
+ if err != nil {
+ return nil, err
+ }
+ sidecar := types.BlobTxCellSidecar{
Version: sc.Version,
Commitments: sc.Commitments,
Proofs: sc.Proofs,
- Blobs: sc.Blobs,
+ Cells: cells,
+ Custody: types.CustodyBitmapAll,
}
+ return &BlobTxForPool{
+ Tx: tx.WithoutBlobTxSidecar(),
+ CellSidecar: &sidecar,
+ }, nil
}
-// encodeForNetwork transforms stored blobTxForPool RLP into the standard
-// network transaction encoding. This is used for getRLP.
+// encodeForNetwork transforms stored BlobTxForPool RLP into the network
+// transaction encoding for the given eth protocol version. Used for getRLP.
//
-// Stored RLP: [type_byte || tx_fields, version, [comms], [proofs], [blobs]]
-// V0: type_byte || rlp([tx_fields, [blobs], [comms], [proofs]])
-// V1: type_byte || rlp([tx_fields, version, [blobs], [comms], [proofs]])
-func encodeForNetwork(storedRLP []byte) ([]byte, error) {
+// Stored RLP: [type_byte || tx_fields, [version, [cells], [comms], [proofs], custody]]
+//
+// eth/69, eth/70: [blobs] is recovered from stored cells via kzg.
+//
+// V0: type_byte || rlp([tx_fields, [blobs], [comms], [proofs]])
+// V1: type_byte || rlp([tx_fields, version, [blobs], [comms], [proofs]])
+//
+// eth/72: [blobs] is replaced by an empty list (cells are fetched separately
+//
+// via GetCells).
+// V0: type_byte || rlp([tx_fields, [], [comms], [proofs]])
+// V1: type_byte || rlp([tx_fields, version, [], [comms], [proofs]])
+func encodeForNetwork(storedRLP []byte, version uint) ([]byte, error) {
elems, err := rlp.SplitListValues(storedRLP)
if err != nil {
- return nil, fmt.Errorf("invalid blobTxForPool RLP: %w", err)
+ return nil, fmt.Errorf("invalid BlobTxForPool RLP: %w", err)
}
- if len(elems) < 5 {
- return nil, fmt.Errorf("blobTxForPool has %d elements, need at least 5", len(elems))
+ if len(elems) < 2 {
+ return nil, fmt.Errorf("BlobTxForPool has %d elements, need at least 2", len(elems))
}
// 1. Extract tx byte and other tx fields
@@ -226,19 +268,53 @@ func encodeForNetwork(storedRLP []byte) ([]byte, error) {
typeByte := txBytes[0]
txRLP := txBytes[1:]
- // 2. Find the version of sidecar.
- version, _, err := rlp.SplitUint64(elems[1])
- if err != nil || version > 255 || version == 0 {
+ // 2. Split the nested CellSidecar list.
+ sidecarElems, err := rlp.SplitListValues(elems[1])
+ if err != nil {
+ return nil, fmt.Errorf("invalid CellSidecar RLP: %w", err)
+ }
+ if len(sidecarElems) < 5 {
+ return nil, fmt.Errorf("CellSidecar has %d elements, need at least 5", len(sidecarElems))
+ }
+
+ // 3. Find the version of sidecar.
+ sidecarVersion, _, err := rlp.SplitUint64(sidecarElems[0])
+ if err != nil || sidecarVersion > 255 || sidecarVersion == 0 {
return nil, fmt.Errorf("invalid version: %w", err)
}
- // 3. Extract sidecar elements.
- commitmentsRLP := elems[2]
- proofsRLP := elems[3]
- blobsRLP := elems[4]
- // 4. Reconstruct into the network format.
- outer := [][]byte{txRLP, elems[1], blobsRLP, commitmentsRLP, proofsRLP}
+ // 4. Extract sidecar elements.
+ commitmentsRLP := sidecarElems[2]
+ proofsRLP := sidecarElems[3]
+ // 5. Build the [blobs] field for the wire format.
+ var blobsField []byte
+ // todo - Didn't use eth.ETH72 due to circular import error in test
+ if version >= 72 {
+ // eth/72 omits the blob payload; peers fetch cells separately via GetCells.
+ blobsField = []byte{0xc0} // RLP-encoded empty list
+ } else {
+ // eth/69, eth/70 need actual blobs: recover them from stored cells.
+ var cells []kzg4844.Cell
+ if err := rlp.DecodeBytes(sidecarElems[1], &cells); err != nil {
+ return nil, fmt.Errorf("invalid cells RLP: %w", err)
+ }
+ var custody types.CustodyBitmap
+ if err := rlp.DecodeBytes(sidecarElems[4], &custody); err != nil {
+ return nil, fmt.Errorf("invalid custody RLP: %w", err)
+ }
+ blobs, err := kzg4844.RecoverBlobs(cells, custody.Indices())
+ if err != nil {
+ return nil, fmt.Errorf("failed to recover blobs: %w", err)
+ }
+ blobsField, err = rlp.EncodeToBytes(blobs)
+ if err != nil {
+ return nil, fmt.Errorf("failed to encode blobs: %w", err)
+ }
+ }
+
+ // 6. Reconstruct into the network format.
+ outer := [][]byte{txRLP, sidecarElems[0], blobsField, commitmentsRLP, proofsRLP}
body, err := rlp.MergeListValues(outer)
if err != nil {
return nil, err
@@ -252,21 +328,23 @@ func encodeForNetwork(storedRLP []byte) ([]byte, error) {
// newBlobTxMeta retrieves the indexed metadata fields from a pooled blob
// transaction and assembles a helper struct to track in memory.
-func newBlobTxMeta(id uint64, size uint64, storageSize uint32, ptx *blobTxForPool) *blobTxMeta {
+func newBlobTxMeta(id uint64, storageSize uint32, ptx *BlobTxForPool) *blobTxMeta {
meta := &blobTxMeta{
- hash: ptx.Tx.Hash(),
- vhashes: ptx.Tx.BlobHashes(),
- version: ptx.Version,
- id: id,
- storageSize: storageSize,
- size: size,
- nonce: ptx.Tx.Nonce(),
- costCap: uint256.MustFromBig(ptx.Tx.Cost()),
- execTipCap: uint256.MustFromBig(ptx.Tx.GasTipCap()),
- execFeeCap: uint256.MustFromBig(ptx.Tx.GasFeeCap()),
- blobFeeCap: uint256.MustFromBig(ptx.Tx.BlobGasFeeCap()),
- execGas: ptx.Tx.Gas(),
- blobGas: ptx.Tx.BlobGas(),
+ hash: ptx.Tx.Hash(),
+ vhashes: ptx.Tx.BlobHashes(),
+ version: ptx.CellSidecar.Version,
+ id: id,
+ storageSize: storageSize,
+ size: ptx.txSize(),
+ sizeWithoutBlob: ptx.txSizeWithoutBlob(),
+ nonce: ptx.Tx.Nonce(),
+ costCap: uint256.MustFromBig(ptx.Tx.Cost()),
+ execTipCap: uint256.MustFromBig(ptx.Tx.GasTipCap()),
+ execFeeCap: uint256.MustFromBig(ptx.Tx.GasFeeCap()),
+ blobFeeCap: uint256.MustFromBig(ptx.Tx.BlobGasFeeCap()),
+ execGas: ptx.Tx.Gas(),
+ blobGas: ptx.Tx.BlobGas(),
+ custody: &ptx.CellSidecar.Custody,
}
meta.basefeeJumps = dynamicFeeJumps(meta.execFeeCap)
meta.blobfeeJumps = dynamicBlobFeeJumps(meta.blobFeeCap)
@@ -465,8 +543,10 @@ type BlobPool struct {
stored uint64 // Useful data size of all transactions on disk
limbo *limbo // Persistent data store for the non-finalized blobs
- gapped map[common.Address][]*types.Transaction // Transactions that are currently gapped (nonce too high)
- gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion
+ cQueue *conversionQueue
+
+ gapped map[common.Address][]*BlobTxForPool // Transactions that are currently gapped (nonce too high)
+ gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion
signer types.Signer // Transaction signer to use for sender recovery
chain BlockChain // Chain object to access the state through
@@ -501,7 +581,7 @@ func New(config Config, chain BlockChain, hasPendingAuth func(common.Address) bo
lookup: newLookup(),
index: make(map[common.Address][]*blobTxMeta),
spent: make(map[common.Address]*uint256.Int),
- gapped: make(map[common.Address][]*types.Transaction),
+ gapped: make(map[common.Address][]*BlobTxForPool),
gappedSource: make(map[common.Hash]common.Address),
}
}
@@ -564,11 +644,14 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
)
index := func(id uint64, size uint32, blob []byte) {
err := p.parseTransaction(id, size, blob)
- if err != nil {
- toDelete = append(toDelete, id)
- }
+ // Transactions in legacy format will be queued for cell computation.
+ // This entry will be swapped from the store after the conversion.
if errors.Is(err, errLegacyTx) {
convertTxs = append(convertTxs, id)
+ return
+ }
+ if err != nil {
+ toDelete = append(toDelete, id)
}
}
store, err := billy.Open(billy.Options{Path: queuedir, Repair: true}, slotter, index)
@@ -577,45 +660,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
}
p.store = store
- // Migrate legacy transactions (types.Transaction) to pooledBlobTx format.
- if len(convertTxs) > 0 {
- for _, id := range convertTxs {
- var tx types.Transaction
- data, err := p.store.Get(id)
- if err != nil {
- continue
- }
- err = rlp.DecodeBytes(data, &tx)
- if err != nil {
- continue
- }
- if tx.BlobTxSidecar() == nil {
- continue
- }
- ptx := newBlobTxForPool(&tx)
- blob, err := rlp.EncodeToBytes(ptx)
- if err != nil {
- continue
- }
- id, err := p.store.Put(blob)
- if err != nil {
- continue
- }
- meta := newBlobTxMeta(id, ptx.TxSize(), p.store.Size(id), ptx)
-
- // If the newly inserted transaction fails to be tracked,
- // it should also be removed with those in `toDelete`
- sender, err := types.Sender(p.signer, ptx.Tx)
- if err != nil {
- toDelete = append(toDelete, id)
- continue
- }
- if err := p.trackTransaction(meta, sender); err != nil {
- toDelete = append(toDelete, id)
- continue
- }
- }
- }
+ p.cQueue = newConversionQueue()
if len(toDelete) > 0 {
log.Warn("Dropping invalidated blob transactions", "ids", toDelete)
@@ -659,7 +704,8 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
// Pool initialized, attach the blob limbo to it to track blobs included
// recently but not yet finalized
- p.limbo, err = newLimbo(p.chain.Config(), limbodir)
+ var convertLimbo []uint64
+ p.limbo, convertLimbo, err = newLimbo(p.chain.Config(), limbodir)
if err != nil {
p.Close()
return err
@@ -678,12 +724,22 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
// Update the metrics and return the constructed pool
datacapGauge.Update(int64(p.config.Datacap))
p.updateStorageMetrics()
+
+ if len(convertTxs) > 0 {
+ p.cQueue.launchConversion(func() { p.convertLegacyTxs(convertTxs) })
+ }
+ if len(convertLimbo) > 0 {
+ p.cQueue.launchConversion(func() { p.convertLegacyLimbo(convertLimbo) })
+ }
return nil
}
// Close closes down the underlying persistent store.
func (p *BlobPool) Close() error {
var errs []error
+ if p.cQueue != nil {
+ p.cQueue.close()
+ }
if p.limbo != nil { // Close might be invoked due to error in constructor, before p,limbo is set
if err := p.limbo.Close(); err != nil {
errs = append(errs, err)
@@ -704,25 +760,119 @@ func (p *BlobPool) Close() error {
// parseTransaction is a callback method on pool creation that gets called for
// each transaction on disk to create the in-memory metadata index.
-// Return value `bool` is set to true when the entry has old Transaction type.
+// Announced state is not initialized here, it needs to be initialized separately.
+//
+// If a legacy types.Transaction is found on disk, it is returned for migration
+// in Init (the old ID will be deleted and a new pooledBlobTx written).
+// If a pooledBlobTx is found, it is indexed directly and nil is returned.
func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error {
- var ptx blobTxForPool
+ var ptx BlobTxForPool
if err := rlp.DecodeBytes(blob, &ptx); err != nil {
kind, content, _, splitErr := rlp.Split(blob)
// check whether it is legacy tx type
if splitErr == nil && kind == rlp.String && len(content) > 1 && content[0] == 3 {
return errLegacyTx
}
+ log.Error("Failed to decode blob pool entry", "id", id, "err", err)
return err
}
- meta := newBlobTxMeta(id, ptx.TxSize(), size, &ptx)
+ meta := newBlobTxMeta(id, size, &ptx)
sender, err := types.Sender(p.signer, ptx.Tx)
if err != nil {
+ // This path is impossible unless the signature validity changes across
+ // restarts. For that ever improbable case, recover gracefully by ignoring
+ // this data entry.
+ log.Error("Failed to recover blob tx sender", "id", id, "hash", ptx.Tx.Hash(), "err", err)
return err
}
return p.trackTransaction(meta, sender)
}
+func (p *BlobPool) convertLegacyTxs(ids []uint64) {
+ start := time.Now()
+ var converted, discarded int
+
+ for _, id := range ids {
+ p.lock.Lock()
+ data, err := p.store.Get(id)
+ if err != nil {
+ p.lock.Unlock()
+ continue
+ }
+ if derr := p.store.Delete(id); derr != nil {
+ log.Error("Failed to delete legacy blob tx", "id", id, "err", derr)
+ p.lock.Unlock()
+ continue
+ }
+ p.lock.Unlock()
+ var tx types.Transaction
+ if err := rlp.DecodeBytes(data, &tx); err != nil {
+ log.Error("Failed to decode legacy blob tx", "id", id, "err", err)
+ continue
+ }
+ ptx, err := newBlobTxForPool(&tx)
+ if err != nil {
+ log.Error("Failed to convert legacy blob tx", "hash", tx.Hash(), "err", err)
+ continue
+ }
+ if err := p.AddPooledTx(ptx); err != nil {
+ log.Debug("Discarded converted blob tx", "hash", tx.Hash(), "err", err)
+ discarded++
+ } else {
+ converted++
+ }
+ }
+
+ log.Info("Completed blob transaction conversion", "converted", converted, "discarded", discarded, "elapsed", common.PrettyDuration(time.Since(start)))
+}
+
+func (p *BlobPool) convertLegacyLimbo(ids []uint64) {
+ start := time.Now()
+ var converted, discarded int
+
+ for _, id := range ids {
+ p.lock.Lock()
+ data, err := p.limbo.store.Get(id)
+ if err != nil {
+ p.lock.Unlock()
+ continue
+ }
+ if derr := p.limbo.store.Delete(id); derr != nil {
+ log.Error("Failed to delete legacy blob tx", "id", id, "err", derr)
+ p.lock.Unlock()
+ continue
+ }
+ p.lock.Unlock()
+ var legacy struct {
+ TxHash common.Hash
+ Block uint64
+ Tx *types.Transaction
+ }
+ if err := rlp.DecodeBytes(data, &legacy); err != nil {
+ log.Error("Failed to decode legacy limbo entry", "id", id, "err", err)
+ continue
+ }
+ if legacy.Tx == nil || legacy.Tx.BlobTxSidecar() == nil {
+ continue
+ }
+ ptx, err := newBlobTxForPool(legacy.Tx)
+ if err != nil {
+ log.Error("Failed to convert legacy limbo entry", "hash", legacy.TxHash, "err", err)
+ continue
+ }
+ p.lock.Lock()
+ if err := p.limbo.setAndIndex(ptx, legacy.Block); err != nil {
+ log.Error("Failed to re-store converted limbo entry", "hash", legacy.TxHash, "err", err)
+ discarded++
+ } else {
+ converted++
+ }
+ p.lock.Unlock()
+ }
+
+ log.Info("Completed limbo blob conversion", "converted", converted, "discarded", discarded, "elapsed", common.PrettyDuration(time.Since(start)))
+}
+
// trackTransaction registers a transaction's metadata in the pool's indices.
func (p *BlobPool) trackTransaction(meta *blobTxMeta, sender common.Address) error {
if p.lookup.exists(meta.hash) {
@@ -1006,7 +1156,7 @@ func (p *BlobPool) offload(addr common.Address, nonce uint64, id uint64, inclusi
log.Error("Blobs missing for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err)
return
}
- var ptx blobTxForPool
+ var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
log.Error("Blobs corrupted for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err)
return
@@ -1094,7 +1244,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) {
log.Error("Blobs missing for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err)
continue
}
- var ptx blobTxForPool
+ var ptx BlobTxForPool
if err = rlp.DecodeBytes(data, &ptx); err != nil {
log.Error("Blobs corrupted for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err)
continue
@@ -1131,7 +1281,7 @@ func (p *BlobPool) vhashesByTx() map[common.Hash][]common.Hash {
// getByVhash reads and decodes the blob transaction which has the given
// versioned hash. Returns nil if unavailable.
-func (p *BlobPool) getByVhash(vhash common.Hash) *blobTxForPool {
+func (p *BlobPool) getByVhash(vhash common.Hash) *BlobTxForPool {
p.lock.RLock()
txID, exists := p.lookup.storeidOfBlob(vhash)
p.lock.RUnlock()
@@ -1143,7 +1293,7 @@ func (p *BlobPool) getByVhash(vhash common.Hash) *blobTxForPool {
log.Error("Tracked blob transaction missing from store", "id", txID, "err", err)
return nil
}
- var ptx blobTxForPool
+ var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
log.Error("Blobs corrupted for tracked transaction", "id", txID, "err", err)
return nil
@@ -1300,7 +1450,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error {
// A reorged-out legacy blob transaction can therefore not be re-added, so drop
// it on the floor instead of putting it back.
head := p.head.Load()
- if p.chain.Config().IsOsaka(head.Number, head.Time) && ptx.Version == types.BlobSidecarVersion0 {
+ if p.chain.Config().IsOsaka(head.Number, head.Time) && ptx.CellSidecar.Version == types.BlobSidecarVersion0 {
log.Debug("Dropping reorged legacy blob transaction", "hash", txhash)
return errors.New("legacy blob sidecar unsupported post-osaka")
}
@@ -1314,7 +1464,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error {
log.Error("Failed to write transaction into storage", "hash", ptx.Tx.Hash(), "err", err)
return err
}
- meta := newBlobTxMeta(id, ptx.TxSize(), p.store.Size(id), ptx)
+ meta := newBlobTxMeta(id, p.store.Size(id), ptx)
if _, ok := p.index[addr]; !ok {
if err := p.reserver.Hold(addr); err != nil {
log.Warn("Failed to reserve account for blob pool", "tx", ptx.Tx.Hash(), "from", addr, "err", err)
@@ -1543,6 +1693,7 @@ func (p *BlobPool) Has(hash common.Hash) bool {
return poolHas || gapped
}
+// getRLP returns the raw RLP-encoded pooledBlobTx data from the store.
func (p *BlobPool) getRLP(hash common.Hash) []byte {
// Track the amount of time waiting to retrieve a fully resolved blob tx from
// the pool and the amount of time actually spent on pulling the data from disk.
@@ -1569,32 +1720,34 @@ func (p *BlobPool) getRLP(hash common.Hash) []byte {
}
// Get returns a transaction if it is contained in the pool, or nil otherwise.
+// Note that this function always try to recover full blobs
func (p *BlobPool) Get(hash common.Hash) *types.Transaction {
data := p.getRLP(hash)
if len(data) == 0 {
return nil
}
- var ptx blobTxForPool
+ var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
id, _ := p.lookup.storeidOfTx(hash)
-
- log.Error("Blobs corrupted for traced transaction",
- "hash", hash, "id", id, "err", err)
+ log.Error("Blobs corrupted for traced transaction", "hash", hash, "id", id, "err", err)
return nil
}
- return ptx.ToTx()
+ tx, err := ptx.toTx()
+ if err != nil {
+ log.Error("Failed to recover transaction in blobpool", "hash", hash, "err", err)
+ return nil
+ }
+ return tx
}
-// GetRLP returns a RLP-encoded transaction for network if it is contained in the pool.
-// It converts the pool's internal type to the RLP format used by the eth protocol:
-// e.g. type_byte || [..., version, [blobs], [comms], [proofs]]
-func (p *BlobPool) GetRLP(hash common.Hash) []byte {
+// GetRLP returns an RLP-encoded transaction if it is contained in the pool.
+func (p *BlobPool) GetRLP(hash common.Hash, version uint) []byte {
data := p.getRLP(hash)
if len(data) == 0 {
// Not in this pool, do not log.
return nil
}
- rlp, err := encodeForNetwork(data)
+ rlp, err := encodeForNetwork(data, version)
if err != nil {
log.Error("Failed to encode pooled tx into the network type", "hash", hash, "err", err)
return nil
@@ -1611,13 +1764,14 @@ func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
p.lock.RLock()
defer p.lock.RUnlock()
- size, ok := p.lookup.sizeOfTx(hash)
+ meta, ok := p.lookup.txIndex[hash]
if !ok {
return nil
}
return &txpool.TxMetadata{
- Type: types.BlobTxType,
- Size: size,
+ Type: types.BlobTxType,
+ Size: meta.size,
+ SizeWithoutBlob: meta.sizeWithoutBlob,
}
}
@@ -1657,12 +1811,16 @@ func (p *BlobPool) getBlobs(vhashes []common.Hash, version byte) (_ []*kzg4844.B
}
// Decode the blob transaction
- var ptx blobTxForPool
+ var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
log.Error("Blobs corrupted for traced transaction", "id", txID, "err", err)
continue
}
- sidecar := ptx.Sidecar()
+ sidecar, err := ptx.sidecar()
+ if err != nil {
+ log.Error("Failed to recover sidecar in blobpool", "id", txID, "err", err)
+ continue
+ }
// Traverse the blobs in the transaction
for i, hash := range ptx.Tx.BlobHashes() {
list, ok := indices[hash]
@@ -1699,6 +1857,90 @@ func (p *BlobPool) getBlobs(vhashes []common.Hash, version byte) (_ []*kzg4844.B
return blobs, commitments, proofs, nil
}
+// GetBlobHashes returns the blob versioned hashes for a given transaction hash.
+func (p *BlobPool) GetBlobHashes(txHash common.Hash) []common.Hash {
+ p.lock.RLock()
+ defer p.lock.RUnlock()
+ vhashes, ok := p.lookup.blobHashesOfTx(txHash)
+ if !ok {
+ return nil
+ }
+ return vhashes
+}
+
+// GetBlobCells returns cells for the given versioned blob hashes. Nil entries
+// mean that the cell was not available.
+func (p *BlobPool) GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error) {
+ var (
+ cells = make([][]*kzg4844.Cell, len(vhashes))
+ proofs = make([][]*kzg4844.Proof, len(vhashes))
+ vindex = make(map[common.Hash][]int) // Indices of versioned hashes in the request
+ filled = make(map[common.Hash]struct{})
+ )
+ for i, h := range vhashes {
+ vindex[h] = append(vindex[h], i)
+ }
+ requestedIndices := mask.Indices()
+
+ for _, vhash := range vhashes {
+ if _, ok := filled[vhash]; ok {
+ continue
+ }
+ p.lock.RLock()
+ txID, exists := p.lookup.storeidOfBlob(vhash)
+ p.lock.RUnlock()
+ if !exists {
+ continue
+ }
+ data, err := p.store.Get(txID)
+ if err != nil {
+ continue
+ }
+ var ptx BlobTxForPool
+ if err := rlp.DecodeBytes(data, &ptx); err != nil {
+ continue
+ }
+ tx := ptx.Tx
+ cellsPerBlob := ptx.CellSidecar.Custody.OneCount()
+ storedIndices := ptx.CellSidecar.Custody.Indices()
+
+ for blobIdx, hash := range tx.BlobHashes() {
+ indices, ok := vindex[hash]
+ if !ok {
+ continue
+ }
+ filled[hash] = struct{}{}
+
+ blobCells := make([]*kzg4844.Cell, len(requestedIndices))
+ blobProofs := make([]*kzg4844.Proof, len(requestedIndices))
+
+ for i, cellIdx := range requestedIndices {
+ pos := -1
+ for k, storedIdx := range storedIndices {
+ if storedIdx == cellIdx {
+ pos = k
+ break
+ }
+ }
+ if pos >= 0 {
+ cell := ptx.CellSidecar.Cells[blobIdx*cellsPerBlob+pos]
+ blobCells[i] = &cell
+ proofIdx := blobIdx*kzg4844.CellProofsPerBlob + int(cellIdx)
+ if proofIdx < len(ptx.CellSidecar.Proofs) {
+ proof := ptx.CellSidecar.Proofs[proofIdx]
+ blobProofs[i] = &proof
+ }
+ }
+ }
+ for _, idx := range indices {
+ cells[idx] = blobCells
+ proofs[idx] = blobProofs
+ }
+ }
+ }
+ return cells, proofs, nil
+}
+
// availableBlobs returns whether the blobs are available in the subpool.
func (p *BlobPool) availableBlobs(vhashes []common.Hash) []bool {
available := make([]bool, len(vhashes))
@@ -1718,14 +1960,23 @@ func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error {
if errs[i] = p.ValidateTxBasics(tx); errs[i] != nil {
continue
}
- errs[i] = p.add(tx)
+ ptx, err := p.cQueue.convert(tx)
+ if err != nil {
+ errs[i] = err
+ continue
+ }
+ if err := txpool.ValidateCells(ptx.CellSidecar); err != nil {
+ errs[i] = err
+ continue
+ }
+ errs[i] = p.AddPooledTx(ptx)
}
return errs
}
// add inserts a new blob transaction into the pool if it passes validation (both
// consensus validity and pool restrictions).
-func (p *BlobPool) add(tx *types.Transaction) (err error) {
+func (p *BlobPool) AddPooledTx(ptx *BlobTxForPool) (err error) {
// The blob pool blocks on adding a transaction. This is because blob txs are
// only even pulled from the network, so this method will act as the overload
// protection for fetches.
@@ -1738,13 +1989,15 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
addtimeHist.Update(time.Since(start).Nanoseconds())
}(time.Now())
- return p.addLocked(tx, true)
+ return p.addLocked(ptx, true)
}
// addLocked inserts a new blob transaction into the pool if it passes validation (both
// consensus validity and pool restrictions). It must be called with the pool lock held.
// Only for internal use.
-func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error) {
+func (p *BlobPool) addLocked(ptx *BlobTxForPool, checkGapped bool) (err error) {
+ tx := ptx.Tx
+
// Ensure the transaction is valid from all perspectives
if err := p.validateTx(tx); err != nil {
log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err)
@@ -1761,7 +2014,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
from, _ := types.Sender(p.signer, tx)
allowance := p.gappedAllowance(from)
if allowance >= 1 && len(p.gappedSource) < maxGapped {
- p.gapped[from] = append(p.gapped[from], tx)
+ p.gapped[from] = append(p.gapped[from], ptx)
p.gappedSource[tx.Hash()] = from
gappedGauge.Update(int64(len(p.gappedSource)))
log.Trace("added tx to gapped blob queue", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
@@ -1807,7 +2060,6 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
}
// Transaction permitted into the pool from a nonce and cost perspective,
// insert it into the database and update the indices
- ptx := newBlobTxForPool(tx)
blob, err := rlp.EncodeToBytes(ptx)
if err != nil {
log.Error("Failed to encode transaction for storage", "hash", tx.Hash(), "err", err)
@@ -1817,7 +2069,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
if err != nil {
return err
}
- meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), ptx)
+ meta := newBlobTxMeta(id, p.store.Size(id), ptx)
var (
next = p.state.GetNonce(from)
@@ -1928,13 +2180,13 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
// We have to add in nonce order, but we want to stable sort to cater for situations
// where transactions are replaced, keeping the original receive order for same nonce
sort.SliceStable(gtxs, func(i, j int) bool {
- return gtxs[i].Nonce() < gtxs[j].Nonce()
+ return gtxs[i].Tx.Nonce() < gtxs[j].Tx.Nonce()
})
for len(gtxs) > 0 {
stateNonce := p.state.GetNonce(from)
firstgap := stateNonce + uint64(len(p.index[from]))
- if gtxs[0].Nonce() > firstgap {
+ if gtxs[0].Tx.Nonce() > firstgap {
// Anything beyond the first gap is not addable yet
break
}
@@ -1942,26 +2194,26 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
// Drop any buffered transactions that became stale in the meantime (included in chain or replaced)
// If we arrive to the transaction in the pending range (between the state Nonce and first gap, we
// try to add them now while removing from here.
- tx := gtxs[0]
+ ptx := gtxs[0]
gtxs[0] = nil
gtxs = gtxs[1:]
- delete(p.gappedSource, tx.Hash())
+ delete(p.gappedSource, ptx.Tx.Hash())
- if tx.Nonce() < stateNonce {
+ if ptx.Tx.Nonce() < stateNonce {
// Stale, drop it. Eventually we could add to limbo here if hash matches.
- log.Trace("Gapped blob transaction became stale", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from]))
+ log.Trace("Gapped blob transaction became stale", "hash", ptx.Tx.Hash(), "from", from, "nonce", ptx.Tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from]))
continue
}
- if tx.Nonce() <= firstgap {
+ if ptx.Tx.Nonce() <= firstgap {
// If we hit the pending range, including the first gap, add it and continue to try to add more.
// We do not recurse here, but continue to loop instead.
// We are under lock, so we can add the transaction directly.
- if err := p.addLocked(tx, false); err == nil {
+ if err := p.addLocked(ptx, false); err == nil {
gappedPromotedMeter.Mark(1)
- log.Trace("Gapped blob transaction added to pool", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
+ log.Trace("Gapped blob transaction added to pool", "hash", ptx.Tx.Hash(), "from", from, "nonce", ptx.Tx.Nonce(), "qlen", len(p.gapped[from]))
} else {
- log.Trace("Gapped blob transaction not accepted", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "err", err)
+ log.Trace("Gapped blob transaction not accepted", "hash", ptx.Tx.Hash(), "from", from, "nonce", ptx.Tx.Nonce(), "err", err)
}
}
}
@@ -2087,6 +2339,10 @@ func (p *BlobPool) Pending(filter txpool.PendingFilter) (map[common.Address][]*t
break // execution gas limit is too high
}
}
+ // Skip transactions without enough cells to recover blobs
+ if !filter.PartialCells && tx.custody != nil && tx.custody.OneCount() < kzg4844.DataPerBlob {
+ break // not enough cells to build a full payload, discard rest of txs from the account
+ }
// Transaction was accepted according to the filter, append to the pending list
lazies = append(lazies, &txpool.LazyTransaction{
Pool: p,
@@ -2229,10 +2485,10 @@ func (p *BlobPool) evictGapped() {
// and we overwrite the slice for this account after filtering.
keep := txs[:0]
for i, gtx := range txs {
- if gtx.Time().Before(cutoff) || gtx.Nonce() < nonce {
+ if gtx.Tx.Time().Before(cutoff) || gtx.Tx.Nonce() < nonce {
// Evict old or stale transactions
// Should we add stale to limbo here if it would belong?
- delete(p.gappedSource, gtx.Hash())
+ delete(p.gappedSource, gtx.Tx.Hash())
txs[i] = nil // Explicitly nil out evicted element
} else {
keep = append(keep, gtx)
@@ -2355,7 +2611,7 @@ func (p *BlobPool) Clear() {
// Reset counters and the gapped buffer
p.stored = 0
- p.gapped = make(map[common.Address][]*types.Transaction)
+ p.gapped = make(map[common.Address][]*BlobTxForPool)
p.gappedSource = make(map[common.Hash]common.Address)
var (
@@ -2364,3 +2620,13 @@ func (p *BlobPool) Clear() {
)
p.evict = newPriceHeap(basefee, blobfee, p.index)
}
+
+// GetCustody returns the custody bitmap for a given transaction hash.
+func (p *BlobPool) GetCustody(hash common.Hash) *types.CustodyBitmap {
+ p.lock.RLock()
+ defer p.lock.RUnlock()
+ if meta := p.lookup.txIndex[hash]; meta != nil {
+ return &meta.custody
+ }
+ return nil
+}
diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go
index a2ec06bed4..56e018d175 100644
--- a/core/txpool/blobpool/blobpool_test.go
+++ b/core/txpool/blobpool/blobpool_test.go
@@ -31,6 +31,7 @@ import (
"slices"
"sync"
"testing"
+ "time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/misc/eip1559"
@@ -233,7 +234,8 @@ func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64,
// encodeForPool encodes a blob transaction in the blobTxForPool storage format.
func encodeForPool(tx *types.Transaction) []byte {
- blob, _ := rlp.EncodeToBytes(newBlobTxForPool(tx))
+ ptx, _ := newBlobTxForPool(tx)
+ blob, _ := rlp.EncodeToBytes(ptx)
return blob
}
@@ -267,6 +269,15 @@ func makeMultiBlobTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCa
return types.MustSignNewTx(key, types.LatestSigner(params.MainnetChainConfig), blobtx)
}
+// removeBlobs returns a copy of tx with the blob payload removed from the
+// sidecar, keeping commitments, proofs and cells intact (simulating what
+// ETH/72 peers send).
+func removeBlobs(tx *types.Transaction) *types.Transaction {
+ sidecar := tx.BlobTxSidecar().Copy()
+ sidecar.Blobs = nil
+ return tx.WithBlobTxSidecar(sidecar)
+}
+
// makeUnsignedTx is a utility method to construct a random blob transaction
// without signing it.
func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64) *types.BlobTx {
@@ -471,7 +482,7 @@ func verifyBlobRetrievals(t *testing.T, pool *BlobPool) {
// - 8. Fully duplicate transactions (matching hash) must be dropped
// - 9. Duplicate nonces from the same account must be dropped
func TestOpenDrops(t *testing.T) {
- //log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true)))
+ // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true)))
// Create a temporary folder for the persistent backend
storage := t.TempDir()
@@ -498,75 +509,76 @@ func TestOpenDrops(t *testing.T) {
S: new(uint256.Int),
})
blob, _ := rlp.EncodeToBytes(tx)
- badsig, _ := store.Put(blob)
+ badsig := tx.Hash()
+ store.Put(blob)
// Insert a sequence of transactions with a nonce gap in between to verify
// that anything gapped will get evicted (case 3).
var (
gapper, _ = crypto.GenerateKey()
- valids = make(map[uint64]struct{})
- gapped = make(map[uint64]struct{})
+ valids = make(map[common.Hash]struct{})
+ gapped = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 3, 4, 6, 7} { // first gap at #2, another at #5
tx := makeTx(nonce, 1, 1, 1, gapper)
blob := encodeForPool(tx)
- id, _ := store.Put(blob)
+ store.Put(blob)
if nonce < 2 {
- valids[id] = struct{}{}
+ valids[tx.Hash()] = struct{}{}
} else {
- gapped[id] = struct{}{}
+ gapped[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions with a gapped starting nonce to verify
// that the entire set will get dropped (case 3).
var (
dangler, _ = crypto.GenerateKey()
- dangling = make(map[uint64]struct{})
+ dangling = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{1, 2, 3} { // first gap at #0, all set dangling
tx := makeTx(nonce, 1, 1, 1, dangler)
blob := encodeForPool(tx)
- id, _ := store.Put(blob)
- dangling[id] = struct{}{}
+ store.Put(blob)
+ dangling[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with already passed nonces to verify
// that the entire set will get dropped (case 4).
var (
filler, _ = crypto.GenerateKey()
- filled = make(map[uint64]struct{})
+ filled = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // account nonce at 3, all set filled
tx := makeTx(nonce, 1, 1, 1, filler)
blob := encodeForPool(tx)
- id, _ := store.Put(blob)
- filled[id] = struct{}{}
+ store.Put(blob)
+ filled[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with partially passed nonces to verify
// that the included part of the set will get dropped (case 4).
var (
overlapper, _ = crypto.GenerateKey()
- overlapped = make(map[uint64]struct{})
+ overlapped = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2, 3} { // account nonce at 2, half filled
tx := makeTx(nonce, 1, 1, 1, overlapper)
blob := encodeForPool(tx)
- id, _ := store.Put(blob)
+ store.Put(blob)
if nonce >= 2 {
- valids[id] = struct{}{}
+ valids[tx.Hash()] = struct{}{}
} else {
- overlapped[id] = struct{}{}
+ overlapped[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions with an underpriced first to verify that
// the entire set will get dropped (case 5).
var (
underpayer, _ = crypto.GenerateKey()
- underpaid = make(map[uint64]struct{})
+ underpaid = make(map[common.Hash]struct{})
)
for i := 0; i < 5; i++ { // make #0 underpriced
var tx *types.Transaction
@@ -577,15 +589,15 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
- id, _ := store.Put(blob)
- underpaid[id] = struct{}{}
+ store.Put(blob)
+ underpaid[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with an underpriced in between to verify
// that it and anything newly gapped will get evicted (case 5).
var (
outpricer, _ = crypto.GenerateKey()
- outpriced = make(map[uint64]struct{})
+ outpriced = make(map[common.Hash]struct{})
)
for i := 0; i < 5; i++ { // make #2 underpriced
var tx *types.Transaction
@@ -596,18 +608,18 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
- id, _ := store.Put(blob)
+ store.Put(blob)
if i < 2 {
- valids[id] = struct{}{}
+ valids[tx.Hash()] = struct{}{}
} else {
- outpriced[id] = struct{}{}
+ outpriced[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions fully overdrafted to verify that the
// entire set will get invalidated (case 6).
var (
exceeder, _ = crypto.GenerateKey()
- exceeded = make(map[uint64]struct{})
+ exceeded = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // nonce 0 overdrafts the account
var tx *types.Transaction
@@ -618,14 +630,14 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
- id, _ := store.Put(blob)
- exceeded[id] = struct{}{}
+ store.Put(blob)
+ exceeded[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions partially overdrafted to verify that part
// of the set will get invalidated (case 6).
var (
overdrafter, _ = crypto.GenerateKey()
- overdrafted = make(map[uint64]struct{})
+ overdrafted = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // nonce 1 overdrafts the account
var tx *types.Transaction
@@ -636,44 +648,46 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
- id, _ := store.Put(blob)
+ store.Put(blob)
if nonce < 1 {
- valids[id] = struct{}{}
+ valids[tx.Hash()] = struct{}{}
} else {
- overdrafted[id] = struct{}{}
+ overdrafted[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions overflowing the account cap to verify
// that part of the set will get invalidated (case 7).
var (
overcapper, _ = crypto.GenerateKey()
- overcapped = make(map[uint64]struct{})
+ overcapped = make(map[common.Hash]struct{})
)
for nonce := uint64(0); nonce < maxTxsPerAccount+3; nonce++ {
- blob := encodeForPool(makeTx(nonce, 1, 1, 1, overcapper))
+ tx := makeTx(nonce, 1, 1, 1, overcapper)
+ blob := encodeForPool(tx)
- id, _ := store.Put(blob)
+ store.Put(blob)
if nonce < maxTxsPerAccount {
- valids[id] = struct{}{}
+ valids[tx.Hash()] = struct{}{}
} else {
- overcapped[id] = struct{}{}
+ overcapped[tx.Hash()] = struct{}{}
}
}
// Insert a batch of duplicated transactions to verify that only one of each
// version will remain (case 8).
var (
duplicater, _ = crypto.GenerateKey()
- duplicated = make(map[uint64]struct{})
+ duplicated = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} {
- blob := encodeForPool(makeTx(nonce, 1, 1, 1, duplicater))
+ tx := makeTx(nonce, 1, 1, 1, duplicater)
+ blob := encodeForPool(tx)
for i := 0; i < int(nonce)+1; i++ {
- id, _ := store.Put(blob)
+ store.Put(blob)
if i == 0 {
- valids[id] = struct{}{}
+ valids[tx.Hash()] = struct{}{}
} else {
- duplicated[id] = struct{}{}
+ duplicated[tx.Hash()] = struct{}{}
}
}
}
@@ -681,17 +695,18 @@ func TestOpenDrops(t *testing.T) {
// remain (case 9).
var (
repeater, _ = crypto.GenerateKey()
- repeated = make(map[uint64]struct{})
+ repeated = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} {
for i := 0; i < int(nonce)+1; i++ {
- blob := encodeForPool(makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater))
+ tx := makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater)
+ blob := encodeForPool(tx)
- id, _ := store.Put(blob)
+ store.Put(blob)
if i == 0 {
- valids[id] = struct{}{}
+ valids[tx.Hash()] = struct{}{}
} else {
- repeated[id] = struct{}{}
+ repeated[tx.Hash()] = struct{}{}
}
}
}
@@ -728,39 +743,41 @@ func TestOpenDrops(t *testing.T) {
// Verify that the malformed (case 1), badly signed (case 2) and gapped (case
// 3) txs have been deleted from the pool
- alive := make(map[uint64]struct{})
+ alive := make(map[common.Hash]struct{})
for _, txs := range pool.index {
for _, tx := range txs {
switch tx.id {
case malformed:
t.Errorf("malformed RLP transaction remained in storage")
- case badsig:
- t.Errorf("invalidly signed transaction remained in storage")
default:
- if _, ok := dangling[tx.id]; ok {
+ if badsig == tx.hash {
+ t.Errorf("invalidly signed transaction remained in storage")
+ }
+ if _, ok := dangling[tx.hash]; ok {
t.Errorf("dangling transaction remained in storage: %d", tx.id)
- } else if _, ok := filled[tx.id]; ok {
+ } else if _, ok := filled[tx.hash]; ok {
t.Errorf("filled transaction remained in storage: %d", tx.id)
- } else if _, ok := overlapped[tx.id]; ok {
+ } else if _, ok := overlapped[tx.hash]; ok {
t.Errorf("overlapped transaction remained in storage: %d", tx.id)
- } else if _, ok := gapped[tx.id]; ok {
+ } else if _, ok := gapped[tx.hash]; ok {
t.Errorf("gapped transaction remained in storage: %d", tx.id)
- } else if _, ok := underpaid[tx.id]; ok {
+ } else if _, ok := underpaid[tx.hash]; ok {
t.Errorf("underpaid transaction remained in storage: %d", tx.id)
- } else if _, ok := outpriced[tx.id]; ok {
+ } else if _, ok := outpriced[tx.hash]; ok {
t.Errorf("outpriced transaction remained in storage: %d", tx.id)
- } else if _, ok := exceeded[tx.id]; ok {
+ } else if _, ok := exceeded[tx.hash]; ok {
t.Errorf("fully overdrafted transaction remained in storage: %d", tx.id)
- } else if _, ok := overdrafted[tx.id]; ok {
+ } else if _, ok := overdrafted[tx.hash]; ok {
t.Errorf("partially overdrafted transaction remained in storage: %d", tx.id)
- } else if _, ok := overcapped[tx.id]; ok {
+ } else if _, ok := overcapped[tx.hash]; ok {
t.Errorf("overcapped transaction remained in storage: %d", tx.id)
- } else if _, ok := duplicated[tx.id]; ok {
- t.Errorf("duplicated transaction remained in storage: %d", tx.id)
- } else if _, ok := repeated[tx.id]; ok {
+ } else if _, ok := repeated[tx.hash]; ok {
t.Errorf("repeated nonce transaction remained in storage: %d", tx.id)
} else {
- alive[tx.id] = struct{}{}
+ if _, ok := alive[tx.hash]; ok {
+ t.Errorf("duplicated transaction remained in storage: %d", tx.id)
+ }
+ alive[tx.hash] = struct{}{}
}
}
}
@@ -769,14 +786,14 @@ func TestOpenDrops(t *testing.T) {
if len(alive) != len(valids) {
t.Errorf("valid transaction count mismatch: have %d, want %d", len(alive), len(valids))
}
- for id := range alive {
- if _, ok := valids[id]; !ok {
- t.Errorf("extra transaction %d", id)
+ for hash := range alive {
+ if _, ok := valids[hash]; !ok {
+ t.Errorf("extra transaction %s", hash)
}
}
- for id := range valids {
- if _, ok := alive[id]; !ok {
- t.Errorf("missing transaction %d", id)
+ for hash := range valids {
+ if _, ok := alive[hash]; !ok {
+ t.Errorf("missing transaction %s", hash)
}
}
// Verify all the calculated pool internals. Interestingly, this is **not**
@@ -995,7 +1012,10 @@ func TestOpenCap(t *testing.T) {
keep = []common.Address{addr1, addr3}
drop = []common.Address{addr2}
- size = 2 * (txAvgSize + blobSize + uint64(txBlobOverhead))
+ // After migration to pooledBlobTx, cells (128 x 2048 = 2*blobSize) replace blobs.
+ // The actual billy slot size for pooledBlobTx is 2*(blobSize+txBlobOverhead)+txAvgSize.
+ pooledSlotSize uint64 = 2*(blobSize+uint64(txBlobOverhead)) + txAvgSize
+ size = 2 * pooledSlotSize
)
store.Put(blob1)
store.Put(blob2)
@@ -1004,7 +1024,7 @@ func TestOpenCap(t *testing.T) {
// Verify pool capping twice: first by reducing the data cap, then restarting
// with a high cap to ensure everything was persisted previously
- for _, datacap := range []uint64{2 * (txAvgSize + blobSize + uint64(txBlobOverhead)), 1000 * (txAvgSize + blobSize + uint64(txBlobOverhead))} {
+ for _, datacap := range []uint64{size, 1000 * pooledSlotSize} {
// Create a blob pool out of the pre-seeded data, but cap it to 2 blob transaction
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
@@ -1310,6 +1330,14 @@ func TestLegacyTxConversion(t *testing.T) {
}
defer pool.Close()
+ deadline := time.Now().Add(30 * time.Second)
+ for pool.Get(tx1.Hash()) == nil || pool.Get(tx2.Hash()) == nil {
+ if time.Now().After(deadline) {
+ t.Fatalf("legacy txs were not converted in time")
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+
// Both transactions should be retrievable.
for _, want := range []*types.Transaction{tx1, tx2} {
got := pool.Get(want.Hash())
@@ -1326,7 +1354,7 @@ func TestLegacyTxConversion(t *testing.T) {
// Legacy formats should not exist on pool.store
pool.store.Iterate(func(id uint64, size uint32, blob []byte) {
- var ptx blobTxForPool
+ var ptx BlobTxForPool
if err := rlp.DecodeBytes(blob, &ptx); err != nil {
t.Errorf("entry %d not in new blobTxForPool format: %v", id, err)
}
@@ -1335,6 +1363,75 @@ func TestLegacyTxConversion(t *testing.T) {
verifyPoolInternals(t, pool)
}
+func TestLegacyLimboConversion(t *testing.T) {
+ storage := t.TempDir()
+ os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700)
+ limbodir := filepath.Join(storage, limboedTransactionStore)
+ os.MkdirAll(limbodir, 0700)
+
+ key, _ := crypto.GenerateKey()
+ tx := makeMultiBlobTx(0, 1, 1000, 100, 2, 0, key)
+
+ store, err := billy.Open(billy.Options{Path: limbodir}, newSlotterEIP7594(testMaxBlobsPerBlock), nil)
+ if err != nil {
+ t.Fatalf("failed to open limbo billy: %v", err)
+ }
+ data, err := rlp.EncodeToBytes(&struct {
+ TxHash common.Hash
+ Block uint64
+ Tx *types.Transaction
+ }{TxHash: tx.Hash(), Block: 42, Tx: tx})
+ if err != nil {
+ t.Fatalf("failed to legacy-encode limbo entry: %v", err)
+ }
+ if _, err := store.Put(data); err != nil {
+ t.Fatalf("failed to put legacy limbo entry: %v", err)
+ }
+ store.Close()
+
+ statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
+ statedb.Commit(0, true, false)
+ chain := &testBlockChain{
+ config: params.MainnetChainConfig,
+ basefee: uint256.NewInt(params.InitialBaseFee),
+ blobfee: uint256.NewInt(params.BlobTxMinBlobGasprice),
+ statedb: statedb,
+ }
+ pool := New(Config{Datadir: storage}, chain, nil)
+ if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil {
+ t.Fatalf("failed to create blob pool: %v", err)
+ }
+ defer pool.Close()
+
+ deadline := time.Now().Add(30 * time.Second)
+ for {
+ pool.lock.RLock()
+ _, ok := pool.limbo.index[tx.Hash()]
+ pool.lock.RUnlock()
+ if ok {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("legacy limbo entry was not converted in time")
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+
+ pool.lock.Lock()
+ ptx, err := pool.limbo.pull(tx.Hash())
+ pool.lock.Unlock()
+ if err != nil {
+ t.Fatalf("failed to pull converted limbo entry: %v", err)
+ }
+ full, err := ptx.toTx()
+ if err != nil {
+ t.Fatalf("failed to reconstruct tx from converted limbo entry: %v", err)
+ }
+ if full.Hash() != tx.Hash() {
+ t.Fatalf("converted limbo tx hash mismatch: have %s, want %s", full.Hash(), tx.Hash())
+ }
+}
+
// TestBlobCountLimit tests the blobpool enforced limits on the max blob count.
func TestBlobCountLimit(t *testing.T) {
var (
@@ -1384,7 +1481,7 @@ func TestBlobCountLimit(t *testing.T) {
// Check that first succeeds second fails.
if errs[0] != nil {
- t.Fatalf("expected tx with 7 blobs to succeed, got %v", errs[0])
+ t.Fatalf("expected tx with 7 blobs to succeed, got: %v", errs[0])
}
if !errors.Is(errs[1], txpool.ErrTxBlobLimitExceeded) {
t.Fatalf("expected tx with 8 blobs to fail, got: %v", errs[1])
@@ -2103,23 +2200,40 @@ func TestGetBlobs(t *testing.T) {
pool.Close()
}
-// TestEncodeForNetwork verifies that encodeForNetwork produces output identical
-// to rlp.EncodeToBytes on the original transaction, for both V0 and V1 sidecars.
+// TestEncodeForNetwork verifies that encodeForNetwork produces the correct wire
+// encoding for each (sidecar version, eth protocol version) combination.
+// - eth/69, eth/70: blobs recovered from cells, output matches the original tx
+// - eth/72: blob payload omitted, output matches removeBlobs(tx)
func TestEncodeForNetwork(t *testing.T) {
- t.Run("v1", func(t *testing.T) { testEncodeForNetwork(t) })
+ cases := []struct {
+ name string
+ ethVer uint
+ }{
+ {"eth70", 70},
+ {"eth72", 72},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ testEncodeForNetwork(t, tc.ethVer)
+ })
+ }
}
-func testEncodeForNetwork(t *testing.T) {
+func testEncodeForNetwork(t *testing.T, ethVer uint) {
key, _ := crypto.GenerateKey()
tx := makeMultiBlobTx(0, 1, 1, 1, 1, 0, key)
- wantRLP, err := rlp.EncodeToBytes(tx)
+ wantTx := tx
+ if ethVer >= 72 {
+ wantTx = removeBlobs(tx)
+ }
+ wantRLP, err := rlp.EncodeToBytes(wantTx)
if err != nil {
t.Fatalf("failed to encode tx: %v", err)
}
storedRLP := encodeForPool(tx)
- gotRLP, err := encodeForNetwork(storedRLP)
+ gotRLP, err := encodeForNetwork(storedRLP, ethVer)
if err != nil {
t.Fatalf("encodeForNetwork failed: %v", err)
}
@@ -2190,7 +2304,8 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) {
b.Fatal(err)
}
statedb.AddBalance(addr, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
- pool.add(tx)
+ pooledTx, _ := newBlobTxForPool(tx)
+ pool.AddPooledTx(pooledTx)
}
statedb.Commit(0, true, false)
defer pool.Close()
@@ -2211,3 +2326,116 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) {
}
}
}
+
+func TestGetCells(t *testing.T) {
+ storage := t.TempDir()
+
+ os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700)
+ store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotterEIP7594(params.BlobTxMaxBlobs), nil)
+
+ var (
+ key1, _ = crypto.GenerateKey()
+
+ addr1 = crypto.PubkeyToAddress(key1.PublicKey)
+
+ tx1 = makeMultiBlobTx(0, 1, 1000, 100, 3, 0, key1) // blobs [0, 3)
+
+ pooledTx1, _ = newBlobTxForPool(tx1)
+
+ blob1, _ = rlp.EncodeToBytes(pooledTx1)
+ )
+
+ store.Put(blob1)
+ store.Close()
+
+ statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
+ statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
+ statedb.Commit(0, true, false)
+
+ chain := &testBlockChain{
+ config: params.MainnetChainConfig,
+ basefee: uint256.NewInt(params.InitialBaseFee),
+ blobfee: uint256.NewInt(params.BlobTxMinBlobGasprice),
+ statedb: statedb,
+ }
+ pool := New(Config{Datadir: storage}, chain, nil)
+ if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil {
+ t.Fatalf("failed to create blob pool: %v", err)
+ }
+ defer pool.Close()
+
+ tests := []struct {
+ name string
+ hash common.Hash
+ mask types.CustodyBitmap
+ expectedLen int
+ shouldFail bool
+ }{
+ {
+ name: "Get cells with single index",
+ hash: tx1.Hash(),
+ mask: types.NewCustodyBitmap([]uint64{5}),
+ expectedLen: 3,
+ shouldFail: false,
+ },
+ {
+ name: "Get cells with all indices",
+ hash: tx1.Hash(),
+ mask: types.CustodyBitmapAll,
+ expectedLen: 384,
+ shouldFail: false,
+ },
+ {
+ name: "Get cells with no indices",
+ hash: tx1.Hash(),
+ mask: types.CustodyBitmap{},
+ expectedLen: 0,
+ shouldFail: false,
+ },
+ {
+ name: "Get cells for non-existent transaction",
+ hash: common.Hash{0x01, 0x02, 0x03},
+ mask: types.NewCustodyBitmap([]uint64{0, 1}),
+ expectedLen: 0,
+ shouldFail: true,
+ },
+ {
+ name: "Get cells with random indices",
+ hash: tx1.Hash(),
+ mask: types.NewRandomCustodyBitmap(8),
+ expectedLen: 24,
+ shouldFail: false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ vhashes := pool.GetBlobHashes(tt.hash)
+ if tt.shouldFail {
+ if vhashes != nil {
+ t.Errorf("expected nil vhashes for non-existent tx")
+ }
+ return
+ }
+ if vhashes == nil {
+ t.Fatalf("expected vhashes, got nil")
+ }
+ blobCells, _, err := pool.GetBlobCells(vhashes, tt.mask)
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ // Count total non-nil cells across all blobs
+ totalCells := 0
+ for _, bc := range blobCells {
+ for _, c := range bc {
+ if c != nil {
+ totalCells++
+ }
+ }
+ }
+ if totalCells != tt.expectedLen {
+ t.Errorf("expected %d cells, got %d", tt.expectedLen, totalCells)
+ }
+ })
+ }
+}
diff --git a/core/txpool/blobpool/buffer.go b/core/txpool/blobpool/buffer.go
new file mode 100644
index 0000000000..d80e793b82
--- /dev/null
+++ b/core/txpool/blobpool/buffer.go
@@ -0,0 +1,318 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package blobpool
+
+import (
+ "cmp"
+ "fmt"
+ "slices"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/txpool"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
+ "github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/metrics"
+)
+
+var (
+ blobBufferTxFirstCounter = metrics.NewRegisteredCounter("blobpool/buffer/txfirst", nil)
+ blobBufferCellsFirstCounter = metrics.NewRegisteredCounter("blobpool/buffer/cellsfirst", nil)
+ blobBufferTotalTx = metrics.NewRegisteredGauge("blobpool/buffer/txcount", nil)
+ blobBufferTotalCells = metrics.NewRegisteredGauge("blobpool/buffer/cellcount", nil)
+)
+
+const (
+ bufferLifetime = 2 * time.Minute
+)
+
+// PeerDelivery holds cells delivered by a single peer, in blob-major order.
+type PeerDelivery struct {
+ Cells []kzg4844.Cell
+ Indices []uint64
+}
+
+type txEntry struct {
+ tx *types.Transaction
+ // Technically it is not required to store peer information to drop properly.
+ // This is mainly for per peer size limit check.
+ peer string
+ added time.Time
+}
+
+type cellEntry struct {
+ deliveries map[string]*PeerDelivery
+ custody types.CustodyBitmap
+ added time.Time
+}
+
+type BlobBuffer struct {
+ mu sync.Mutex
+
+ txs map[common.Hash]*txEntry
+ cells map[common.Hash]*cellEntry
+
+ completed []*BlobTxForPool
+ completedCount atomic.Int32
+ cb BlobBufferFunctions
+}
+
+type BlobBufferFunctions struct {
+ ValidateTx func(*types.Transaction) error
+ AddToPool func(*BlobTxForPool) error
+ DropPeer func(string)
+}
+
+func NewBlobBuffer(cb BlobBufferFunctions) *BlobBuffer {
+ return &BlobBuffer{
+ txs: make(map[common.Hash]*txEntry),
+ cells: make(map[common.Hash]*cellEntry),
+ cb: cb,
+ }
+}
+
+// Flush adds all completed entries to the pool and returns the hashes
+// and corresponding errors (nil on success) for each attempted insert.
+func (b *BlobBuffer) Flush() ([]common.Hash, []error) {
+ // Read the count first and return early if there is nothing to do.
+ // Flush is called very frequently from the blob fetcher so this
+ // optimization is warranted.
+ if b.completedCount.Load() == 0 {
+ return nil, nil
+ }
+
+ b.mu.Lock()
+ defer b.mu.Unlock()
+
+ txs := make([]common.Hash, len(b.completed))
+ errs := make([]error, len(b.completed))
+ for i, ptx := range b.completed {
+ txs[i] = ptx.Tx.Hash()
+ errs[i] = b.cb.AddToPool(ptx)
+ }
+ b.completed = nil
+ b.completedCount.Store(0)
+ return txs, errs
+}
+
+// AddTx buffers a blob transaction (without blobs) from an ETH/72 peer.
+// If cells are already buffered, verification and pool insertion are attempted.
+func (b *BlobBuffer) AddTx(txs []*types.Transaction, peer string) []error {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ defer b.updateMetrics()()
+
+ // First remove any timed-out entries.
+ b.evict()
+
+ errs := make([]error, len(txs))
+ for i, tx := range txs {
+ hash := tx.Hash()
+ sidecar := tx.BlobTxSidecar()
+ if sidecar == nil {
+ errs[i] = fmt.Errorf("blob transaction without sidecar")
+ continue
+ }
+ // tx validation (basic w/o lock)
+ // error will be handled by tx fetcher
+ if err := b.cb.ValidateTx(tx); err != nil {
+ errs[i] = err
+ continue
+ }
+ if entry, ok := b.cells[hash]; ok {
+ b.storeCompleted(hash, tx, entry)
+ continue
+ }
+ blobBufferTxFirstCounter.Inc(1)
+ b.txs[hash] = &txEntry{tx: tx, peer: peer, added: time.Now()}
+ }
+ return errs
+}
+
+// AddCells buffers per-peer cell deliveries from the blob fetcher.
+// If the transaction is already buffered, verification and pool insertion are attempted.
+func (b *BlobBuffer) AddCells(hash common.Hash, deliveries map[string]*PeerDelivery, custody types.CustodyBitmap) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ defer b.updateMetrics()()
+
+ // First remove any timed-out entries.
+ b.evict()
+
+ b.cells[hash] = &cellEntry{
+ deliveries: deliveries,
+ custody: custody,
+ added: time.Now(),
+ }
+ if txe, ok := b.txs[hash]; ok {
+ b.storeCompleted(hash, txe.tx, b.cells[hash])
+ }
+ blobBufferCellsFirstCounter.Inc(1)
+}
+
+// storeCompleted verifies cells per-peer, sorts them, and schedules them for
+// addition into the pool. The actual addition happens in Flush().
+func (b *BlobBuffer) storeCompleted(hash common.Hash, tx *types.Transaction, cells *cellEntry) {
+ sidecar := tx.BlobTxSidecar()
+
+ // Per-peer cell verification
+ if badPeers := b.verifyCells(cells, sidecar); len(badPeers) > 0 {
+ b.dropPeers(badPeers)
+ delete(b.cells, hash)
+ delete(b.txs, hash)
+ return
+ }
+ blobCount := len(tx.BlobHashes())
+ sorted, custody := sortCells(cells, blobCount)
+
+ cellSidecar := types.BlobTxCellSidecar{
+ Version: sidecar.Version,
+ Commitments: sidecar.Commitments,
+ Proofs: sidecar.Proofs,
+ Cells: sorted,
+ Custody: custody,
+ }
+ pooledTx := &BlobTxForPool{
+ Tx: tx.WithoutBlobTxSidecar(),
+ CellSidecar: &cellSidecar,
+ }
+
+ b.completed = append(b.completed, pooledTx)
+ b.completedCount.Add(1)
+ delete(b.cells, hash)
+ delete(b.txs, hash)
+}
+
+func (b *BlobBuffer) HasTx(hash common.Hash) bool {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ _, ok := b.txs[hash]
+ return ok
+}
+
+func (b *BlobBuffer) HasCells(hash common.Hash) bool {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ _, ok := b.cells[hash]
+ return ok
+}
+
+func (b *BlobBuffer) dropPeers(peers []string) {
+ if b.cb.DropPeer == nil {
+ return
+ }
+ for _, p := range peers {
+ b.cb.DropPeer(p)
+ }
+}
+
+func (b *BlobBuffer) evict() {
+ now := time.Now()
+ for hash, entry := range b.txs {
+ if now.Sub(entry.added) > bufferLifetime {
+ delete(b.txs, hash)
+ }
+ }
+ for hash, entry := range b.cells {
+ if now.Sub(entry.added) > bufferLifetime {
+ delete(b.cells, hash)
+ }
+ }
+}
+
+// updateMetrics updates the metrics gauges.
+// This should be called at the start of any operation that changes the buffer
+// content. The returned function is to be called at the end of the operation,
+// usually with defer.
+func (b *BlobBuffer) updateMetrics() func() {
+ preTxCount := len(b.txs)
+ preCellsCount := len(b.cells)
+ return func() {
+ if len(b.txs) != preTxCount {
+ blobBufferTotalTx.Update(int64(len(b.txs)))
+ }
+ if len(b.cells) != preCellsCount {
+ blobBufferTotalCells.Update(int64(len(b.cells)))
+ }
+ }
+}
+
+// verifyCells verifies each peer's cells against the sidecar by treating each
+// per-peer delivery as a mini BlobTxCellSidecar and reusing txpool.ValidateCells.
+// Returns the list of peers whose cells failed verification.
+func (b *BlobBuffer) verifyCells(entry *cellEntry, sidecar *types.BlobTxSidecar) []string {
+ var badPeers []string
+ for peer, delivery := range entry.deliveries {
+ perPeer := &types.BlobTxCellSidecar{
+ Version: sidecar.Version,
+ Cells: delivery.Cells,
+ Commitments: sidecar.Commitments,
+ Proofs: sidecar.Proofs,
+ Custody: types.NewCustodyBitmap(delivery.Indices),
+ }
+ if err := txpool.ValidateCells(perPeer); err != nil {
+ log.Debug("Cell verification failed", "peer", peer, "err", err)
+ badPeers = append(badPeers, peer)
+ }
+ }
+ return badPeers
+}
+
+// sortCells merges all per-peer deliveries into a single flat cell array
+// sorted by custody index.
+//
+// e.g.
+// peer A: cells = [blob0_cell5, blob0_cell3, blob1_cell5, blob1_cell3]
+// peer B: cells = [blob0_cell1, blob0_cell7, blob1_cell1, blob1_cell7]
+// -> [blob0_cell1, blob0_cell3, blob0_cell5, blob0_cell7, blob1_cell1, blob1_cell3, blob1_cell5, blob1_cell7]
+func sortCells(entry *cellEntry, blobCount int) ([]kzg4844.Cell, types.CustodyBitmap) {
+ // indices per delivery
+ var indices []uint64
+
+ // 1. compose per blob cells
+ blob := make([][]kzg4844.Cell, blobCount)
+ for _, d := range entry.deliveries {
+ n := len(d.Indices)
+ indices = append(indices, d.Indices...)
+ for b := range blobCount {
+ blob[b] = append(blob[b], d.Cells[b*n:(b+1)*n]...)
+ }
+ }
+
+ // 2. sort
+ perm := make([]int, len(indices))
+ for i := range perm {
+ perm[i] = i
+ }
+ // perm represents the position of cells in sorted array
+ slices.SortFunc(perm, func(a, b int) int {
+ return cmp.Compare(indices[a], indices[b])
+ })
+ // reorder cells
+ var res []kzg4844.Cell
+ for b := range blobCount {
+ for _, p := range perm {
+ res = append(res, blob[b][p])
+ }
+ }
+
+ custody := types.NewCustodyBitmap(indices)
+ return res, custody
+}
diff --git a/core/txpool/blobpool/buffer_test.go b/core/txpool/blobpool/buffer_test.go
new file mode 100644
index 0000000000..beba17f0a4
--- /dev/null
+++ b/core/txpool/blobpool/buffer_test.go
@@ -0,0 +1,206 @@
+package blobpool
+
+import (
+ "crypto/ecdsa"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
+)
+
+// makeV1Tx creates a V1 blob transaction with cell proofs, then strips blobs
+// (simulating what ETH/72 peers send).
+func makeV1Tx(t *testing.T, nonce uint64, blobCount int, blobOffset int, key *ecdsa.PrivateKey) *types.Transaction {
+ t.Helper()
+ tx := makeMultiBlobTx(nonce, 1, 1, 1, blobCount, blobOffset, key)
+ return removeBlobs(tx)
+}
+
+// makePeerDelivery creates a PeerDelivery for given cell indices from a set of blobs.
+func makePeerDelivery(t *testing.T, blobOffset, blobCount int, indices []uint64) *PeerDelivery {
+ t.Helper()
+ var allCells []kzg4844.Cell
+ for i := 0; i < blobCount; i++ {
+ cells, err := kzg4844.ComputeCells([]kzg4844.Blob{*testBlobs[blobOffset+i]})
+ if err != nil {
+ t.Fatal(err)
+ }
+ allCells = append(allCells, cells...)
+ }
+ var deliveryCells []kzg4844.Cell
+ for b := 0; b < blobCount; b++ {
+ for _, idx := range indices {
+ deliveryCells = append(deliveryCells, allCells[b*kzg4844.CellsPerBlob+int(idx)])
+ }
+ }
+ return &PeerDelivery{Cells: deliveryCells, Indices: indices}
+}
+
+func newTestBuffer(t *testing.T) *BlobBuffer {
+ t.Helper()
+ return NewBlobBuffer(BlobBufferFunctions{
+ ValidateTx: func(tx *types.Transaction) error { return nil },
+ AddToPool: func(ptx *BlobTxForPool) error { return nil },
+ DropPeer: func(peer string) {},
+ })
+}
+
+func TestSortCells(t *testing.T) {
+ blobCount := 2
+ blobOffset := 0
+
+ peerA := makePeerDelivery(t, blobOffset, blobCount, []uint64{5, 3})
+ peerB := makePeerDelivery(t, blobOffset, blobCount, []uint64{1, 7})
+
+ custody := types.NewCustodyBitmap([]uint64{1, 3, 5, 7})
+ entry := &cellEntry{
+ deliveries: map[string]*PeerDelivery{
+ "peerA": peerA,
+ "peerB": peerB,
+ },
+ custody: custody,
+ }
+ sorted, resultCustody := sortCells(entry, blobCount)
+
+ resultIndices := resultCustody.Indices()
+ if len(resultIndices) != 4 {
+ t.Fatalf("expected 4 indices, got %d", len(resultIndices))
+ }
+ for i, expected := range []uint64{1, 3, 5, 7} {
+ if resultIndices[i] != expected {
+ t.Errorf("index %d: expected %d, got %d", i, expected, resultIndices[i])
+ }
+ }
+
+ expected := makePeerDelivery(t, blobOffset, blobCount, []uint64{1, 3, 5, 7})
+ if len(sorted) != len(expected.Cells) {
+ t.Fatalf("sorted length %d != expected %d", len(sorted), len(expected.Cells))
+ }
+ for i := range sorted {
+ if sorted[i] != expected.Cells[i] {
+ t.Errorf("cell %d mismatch", i)
+ }
+ }
+}
+
+func TestAddTxThenCells(t *testing.T) {
+ key, _ := crypto.GenerateKey()
+ blobCount := 2
+ buf := newTestBuffer(t)
+
+ tx := makeV1Tx(t, 0, blobCount, 0, key)
+ hash := tx.Hash()
+
+ if err := buf.AddTx([]*types.Transaction{tx}, "peerA")[0]; err != nil {
+ t.Fatal(err)
+ }
+ if !buf.HasTx(hash) {
+ t.Fatal("tx should be buffered")
+ }
+
+ dataIndices := make([]uint64, kzg4844.DataPerBlob)
+ for i := range dataIndices {
+ dataIndices[i] = uint64(i)
+ }
+ delivery := makePeerDelivery(t, 0, blobCount, dataIndices)
+ custody := types.NewCustodyBitmap(dataIndices)
+
+ buf.AddCells(hash, map[string]*PeerDelivery{"peerB": delivery}, custody)
+ if buf.HasTx(hash) || buf.HasCells(hash) {
+ t.Fatal("buffer should be empty after add")
+ }
+}
+
+func TestAddCellsThenTx(t *testing.T) {
+ key, _ := crypto.GenerateKey()
+ blobCount := 2
+ buf := newTestBuffer(t)
+
+ tx := makeV1Tx(t, 0, blobCount, 0, key)
+ hash := tx.Hash()
+
+ dataIndices := make([]uint64, kzg4844.DataPerBlob)
+ for i := range dataIndices {
+ dataIndices[i] = uint64(i)
+ }
+ delivery := makePeerDelivery(t, 0, blobCount, dataIndices)
+ custody := types.NewCustodyBitmap(dataIndices)
+
+ buf.AddCells(hash, map[string]*PeerDelivery{"peerB": delivery}, custody)
+ if !buf.HasCells(hash) {
+ t.Fatal("cells should be buffered")
+ }
+
+ if err := buf.AddTx([]*types.Transaction{tx}, "peerA")[0]; err != nil {
+ t.Fatal(err)
+ }
+ if buf.HasTx(hash) || buf.HasCells(hash) {
+ t.Fatal("buffer should be empty after add")
+ }
+}
+
+func TestMultiPeerDelivery(t *testing.T) {
+ key, _ := crypto.GenerateKey()
+ blobCount := 2
+ buf := newTestBuffer(t)
+
+ tx := makeV1Tx(t, 0, blobCount, 0, key)
+ hash := tx.Hash()
+ buf.AddTx([]*types.Transaction{tx}, "peerA")
+
+ indicesA := []uint64{0, 2, 4, 6}
+ indicesB := []uint64{1, 3, 5, 7}
+ deliveryA := makePeerDelivery(t, 0, blobCount, indicesA)
+ deliveryB := makePeerDelivery(t, 0, blobCount, indicesB)
+
+ allIndices := append(indicesA, indicesB...)
+ custody := types.NewCustodyBitmap(allIndices)
+
+ buf.AddCells(hash, map[string]*PeerDelivery{
+ "peerB": deliveryA,
+ "peerC": deliveryB,
+ }, custody)
+ if buf.HasTx(hash) || buf.HasCells(hash) {
+ t.Fatal("buffer should be empty after add")
+ }
+}
+
+func TestBadCell(t *testing.T) {
+ key, _ := crypto.GenerateKey()
+ blobCount := 1
+
+ var dropped []string
+ buf := NewBlobBuffer(BlobBufferFunctions{
+ ValidateTx: func(tx *types.Transaction) error { return nil },
+ AddToPool: func(ptx *BlobTxForPool) error { return nil },
+ DropPeer: func(peer string) { dropped = append(dropped, peer) },
+ })
+
+ tx := makeV1Tx(t, 0, blobCount, 0, key)
+ hash := tx.Hash()
+ buf.AddTx([]*types.Transaction{tx}, "peerA")
+
+ goodDelivery := makePeerDelivery(t, 0, blobCount, []uint64{0, 1, 2, 3})
+ badDelivery := makePeerDelivery(t, 0, blobCount, []uint64{4, 5, 6, 7})
+ for i := range badDelivery.Cells {
+ for j := range badDelivery.Cells[i] {
+ badDelivery.Cells[i][j] ^= 0xFF
+ }
+ }
+
+ allIndices := []uint64{0, 1, 2, 3, 4, 5, 6, 7}
+ custody := types.NewCustodyBitmap(allIndices)
+
+ buf.AddCells(hash, map[string]*PeerDelivery{
+ "peerB": goodDelivery,
+ "peerC": badDelivery,
+ }, custody)
+
+ if len(dropped) != 1 || dropped[0] != "peerC" {
+ t.Fatalf("only peerC should have been dropped, got: %v", dropped)
+ }
+ if buf.HasTx(hash) || buf.HasCells(hash) {
+ t.Fatal("buffer should be empty after bad cell drop")
+ }
+}
diff --git a/core/txpool/blobpool/cache.go b/core/txpool/blobpool/cache.go
index 12f9802775..2292ce1277 100644
--- a/core/txpool/blobpool/cache.go
+++ b/core/txpool/blobpool/cache.go
@@ -36,7 +36,7 @@ import (
)
const (
- topKTimeout = 4 * time.Second
+ topKTimeout = 1 * time.Second
hasBlobsTimeout = 1 * time.Second
)
@@ -46,12 +46,14 @@ var (
// missing on the lower level (in this case, the blobpool). The amount that
// we failed to predict) can be calculated with the telemetry span
// (blobs.filled - cache.hit).
- cacheHitMeter = metrics.NewRegisteredMeter("blobpool/cache/hit", nil)
- cacheMissMeter = metrics.NewRegisteredMeter("blobpool/cache/miss", nil)
- cacheBlobsGauge = metrics.NewRegisteredGauge("blobpool/cache/blobs", nil)
+ cacheHitMeter = metrics.NewRegisteredMeter("blobpool/cache/hit", nil)
+ cacheMissMeter = metrics.NewRegisteredMeter("blobpool/cache/miss", nil)
+ cacheEntriesGauge = metrics.NewRegisteredGauge("blobpool/cache/entries", nil)
)
type cachedBlob struct {
+ cell []kzg4844.Cell
+ custody types.CustodyBitmap
blob *kzg4844.Blob
commitment kzg4844.Commitment
proofs []kzg4844.Proof
@@ -74,11 +76,16 @@ type Cache struct {
mu sync.Mutex
entries map[common.Hash]*cachedBlob
+ // needCell is owned by the loop goroutine; it is only read and written
+ // there, so it needs no synchronization. It is flipped on via enableCellCh.
+ needCell bool
+
// channels into loop
quit chan struct{}
topkRequest chan struct{}
topkTimer mclock.Timer
hasBlobsCh chan []common.Hash // list of tx hashes that should be pinned
+ cellModeCh chan bool // signals the loop to switch cell mode on/offo
step func() // test hook fired after each loop iteration
@@ -103,6 +110,7 @@ func newCache(p *BlobPool, clock mclock.Clock, step func()) *Cache {
step: step,
quit: make(chan struct{}),
topkRequest: make(chan struct{}, 1),
+ cellModeCh: make(chan bool, 1),
}
c.wg.Add(1)
@@ -197,13 +205,10 @@ func (c *Cache) GetBlobs(ctx context.Context, vhashes []common.Hash, version byt
c.mu.Lock()
for vhash, idxs := range indices {
n := len(idxs)
-
- cached := c.entries[vhash]
- if cached == nil || cached.version != version {
+ cached, ok := c.entries[vhash]
+ if !ok || cached.version != version || cached.blob == nil {
cacheMiss += n
- if cached == nil {
- misses = append(misses, vhash)
- }
+ misses = append(misses, vhash)
continue
}
cacheHits += n
@@ -241,6 +246,82 @@ func (c *Cache) GetBlobs(ctx context.Context, vhashes []common.Hash, version byt
return blobs, commitments, proofs, nil
}
+// GetCells returns cells for the given versioned blob hashes, filtered
+// by the requested cell indices (mask). Each entry in the result
+// corresponds to one vhash. Nil entries mean the cell was not available.
+// If the cell is not available in cache, it falls back to the blobpool.
+func (c *Cache) GetCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error) {
+ var (
+ cells = make([][]*kzg4844.Cell, len(vhashes))
+ proofs = make([][]*kzg4844.Proof, len(vhashes))
+ indices = make(map[common.Hash][]int)
+ misses []common.Hash
+ )
+ for i, h := range vhashes {
+ indices[h] = append(indices[h], i)
+ }
+ requested := mask.Indices()
+
+ c.mu.Lock()
+ for vhash, idxs := range indices {
+ cached, ok := c.entries[vhash]
+ if !ok {
+ misses = append(misses, vhash)
+ continue
+ }
+ stored := cached.custody.Indices()
+ blobCells := make([]*kzg4844.Cell, len(requested))
+ blobProofs := make([]*kzg4844.Proof, len(requested))
+ for i, cellIdx := range requested {
+ pos := -1
+ for k, s := range stored {
+ if s == cellIdx {
+ pos = k
+ break
+ }
+ }
+ if pos >= 0 && pos < len(cached.cell) {
+ cell := cached.cell[pos]
+ blobCells[i] = &cell
+ if int(cellIdx) < len(cached.proofs) {
+ pf := cached.proofs[cellIdx]
+ blobProofs[i] = &pf
+ }
+ }
+ }
+ for _, idx := range idxs {
+ cells[idx] = blobCells
+ proofs[idx] = blobProofs
+ }
+ }
+ c.mu.Unlock()
+
+ if len(misses) > 0 {
+ mc, mp, err := c.blobpool.GetBlobCells(misses, mask)
+ if err != nil {
+ return nil, nil, err
+ }
+ for j, vhash := range misses {
+ for _, idx := range indices[vhash] {
+ cells[idx] = mc[j]
+ proofs[idx] = mp[j]
+ }
+ }
+ }
+ return cells, proofs, nil
+}
+
+// EnableCell allows the cache to store only cells without recovering
+// blobs. This means we can also cache cells that lack enough blobs to
+// recover. It signals the loop to switch to cell mode and re-select
+// transactions from this wider pool.
+func (c *Cache) SetCellMode(on bool) {
+ select {
+ case c.cellModeCh <- on:
+ case <-c.quit:
+ }
+}
+
func (c *Cache) loop() {
defer c.wg.Done()
@@ -258,6 +339,14 @@ func (c *Cache) loop() {
c.update(want)
c.triggerTopKAfter(topKTimeout)
+ case on := <-c.cellModeCh:
+ // This runs when the CL signals (non-)support for cell proofs. Enable/disable
+ // cell mode and re-select immediately to force an update.
+ if c.needCell != on {
+ c.needCell = on
+ c.triggerTopK()
+ }
+
case <-c.quit:
c.cancelUpdate()
if c.topkTimer != nil {
@@ -285,6 +374,8 @@ func (c *Cache) cancelUpdate() {
// are no longer wanted and loads the missing ones from the blobpool in the
// background.
func (c *Cache) update(want []common.Hash) {
+ cellMode := c.needCell
+
wantSet := make(map[common.Hash]struct{}, len(want))
for _, vh := range want {
wantSet[vh] = struct{}{}
@@ -298,7 +389,13 @@ func (c *Cache) update(want []common.Hash) {
c.mu.Lock()
var missing []common.Hash
for vh := range wantSet {
- if _, ok := c.entries[vh]; !ok {
+ e, ok := c.entries[vh]
+ if ok && ((cellMode && e.cell == nil) || (!cellMode && e.blob == nil)) {
+ delete(c.entries, vh)
+ cacheEntriesGauge.Dec(1)
+ ok = false
+ }
+ if !ok {
missing = append(missing, vh)
}
}
@@ -307,7 +404,7 @@ func (c *Cache) update(want []common.Hash) {
continue
}
delete(c.entries, vh)
- cacheBlobsGauge.Dec(1)
+ cacheEntriesGauge.Dec(1)
}
c.mu.Unlock()
@@ -330,37 +427,92 @@ func (c *Cache) update(want []common.Hash) {
if ptx == nil {
continue
}
- sidecar := ptx.Sidecar()
- if sidecar == nil {
- continue
+ if cellMode {
+ c.loadCells(ptx, wantSet)
+ } else {
+ c.loadBlobs(ptx, wantSet)
}
-
- c.mu.Lock()
- for i, v := range sidecar.BlobHashes() {
- if _, ok := wantSet[v]; !ok {
- continue
- }
- if _, exists := c.entries[v]; exists {
- continue // recompute only new entries
- }
- cellProofs, err := sidecar.CellProofsAt(i)
- if err != nil {
- log.Error("Failed to get cell proofs", "txhash", ptx.Tx.Hash(), "err", err)
- continue
- }
- c.entries[v] = &cachedBlob{
- blob: &sidecar.Blobs[i],
- commitment: sidecar.Commitments[i],
- proofs: cellProofs,
- version: sidecar.Version,
- }
- cacheBlobsGauge.Inc(1)
- }
- c.mu.Unlock()
}
}()
}
+// loadCells loads the cells whose vhash is in wantSet from ptx.
+func (c *Cache) loadCells(ptx *BlobTxForPool, wantSet map[common.Hash]struct{}) {
+ cs := ptx.CellSidecar
+ cellsPerBlob := cs.Custody.OneCount()
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ for i, v := range ptx.Tx.BlobHashes() {
+ if _, ok := wantSet[v]; !ok {
+ continue
+ }
+ if _, exists := c.entries[v]; exists {
+ continue
+ }
+ cellStart := i * cellsPerBlob
+ if cellStart+cellsPerBlob > len(cs.Cells) {
+ continue
+ }
+ blobCells := make([]kzg4844.Cell, cellsPerBlob)
+ copy(blobCells, cs.Cells[cellStart:cellStart+cellsPerBlob])
+
+ var pf []kzg4844.Proof
+ if ps := i * kzg4844.CellProofsPerBlob; ps+kzg4844.CellProofsPerBlob <= len(cs.Proofs) {
+ pf = make([]kzg4844.Proof, kzg4844.CellProofsPerBlob)
+ copy(pf, cs.Proofs[ps:ps+kzg4844.CellProofsPerBlob])
+ }
+ c.entries[v] = &cachedBlob{
+ cell: blobCells,
+ custody: cs.Custody,
+ commitment: cs.Commitments[i],
+ proofs: pf,
+ version: cs.Version,
+ }
+ cacheEntriesGauge.Inc(1)
+ }
+}
+
+// loadBlobs loads the blobs whose vhash is in wantSet from ptx.
+func (c *Cache) loadBlobs(ptx *BlobTxForPool, wantSet map[common.Hash]struct{}) {
+ if ptx.CellSidecar.Custody.OneCount() < kzg4844.DataPerBlob {
+ return
+ }
+ // blobs will be computed inside of sidecar()
+ sidecar, err := ptx.sidecar()
+ if err != nil || sidecar == nil {
+ return
+ }
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ for i, v := range sidecar.BlobHashes() {
+ if _, ok := wantSet[v]; !ok {
+ continue
+ }
+ if _, exists := c.entries[v]; exists {
+ continue
+ }
+ var pf []kzg4844.Proof
+ switch sidecar.Version {
+ case types.BlobSidecarVersion0:
+ pf = []kzg4844.Proof{sidecar.Proofs[i]}
+ case types.BlobSidecarVersion1:
+ cellProofs, err := sidecar.CellProofsAt(i)
+ if err != nil {
+ log.Error("Failed to get cell proofs", "txhash", ptx.Tx.Hash(), "err", err)
+ continue
+ }
+ pf = cellProofs
+ }
+ c.entries[v] = &cachedBlob{
+ blob: &sidecar.Blobs[i],
+ commitment: sidecar.Commitments[i],
+ proofs: pf,
+ version: sidecar.Version,
+ }
+ cacheEntriesGauge.Inc(1)
+ }
+}
+
// selectTopTxs returns the vhashes of the top K most profitable pending blob
// transactions, up to the active fork's maxBlobsPerBlock.
func (c *Cache) selectTopTxs() []common.Hash {
@@ -373,8 +525,9 @@ func (c *Cache) selectTopTxs() []common.Hash {
baseFee := eip1559.CalcBaseFee(config, head)
filter := txpool.PendingFilter{
- BlobTxs: true,
- BaseFee: uint256.MustFromBig(baseFee),
+ BlobTxs: true,
+ BaseFee: uint256.MustFromBig(baseFee),
+ PartialCells: c.needCell,
}
if head.ExcessBlobGas != nil {
filter.BlobFee = uint256.MustFromBig(eip4844.CalcBlobFee(config, head))
diff --git a/core/txpool/blobpool/cache_test.go b/core/txpool/blobpool/cache_test.go
index 3a0436ab85..1902f9b353 100644
--- a/core/txpool/blobpool/cache_test.go
+++ b/core/txpool/blobpool/cache_test.go
@@ -146,7 +146,10 @@ func (tc *testCache) inject(t *testing.T, spec txSpec) []common.Hash {
tx := makeMultiBlobTx(0, spec.tip, 1_000_000, 1_000_000, spec.blobs, tc.offset, key)
tc.offset += spec.blobs
- ptx := newBlobTxForPool(tx)
+ ptx, err := newBlobTxForPool(tx)
+ if err != nil {
+ t.Fatalf("new blob tx for pool: %v", err)
+ }
tc.blobpool.lock.Lock()
defer tc.blobpool.lock.Unlock()
@@ -155,7 +158,7 @@ func (tc *testCache) inject(t *testing.T, spec txSpec) []common.Hash {
if err != nil {
t.Fatalf("store put: %v", err)
}
- meta := newBlobTxMeta(id, ptx.TxSize(), tc.blobpool.store.Size(id), ptx)
+ meta := newBlobTxMeta(id, tc.blobpool.store.Size(id), ptx)
addr := crypto.PubkeyToAddress(key.PublicKey)
tc.blobpool.index[addr] = append(tc.blobpool.index[addr], meta)
tc.blobpool.lookup.track(meta)
diff --git a/core/txpool/blobpool/config.go b/core/txpool/blobpool/config.go
index 1d180739cd..d4c7d1c0f6 100644
--- a/core/txpool/blobpool/config.go
+++ b/core/txpool/blobpool/config.go
@@ -25,6 +25,8 @@ type Config struct {
Datadir string // Data directory containing the currently executable blobs
Datacap uint64 // Soft-cap of database storage (hard cap is larger due to overhead)
PriceBump uint64 // Minimum price bump percentage to replace an already existing nonce
+
+ FetchProbability uint64 // EIP-8070: full blob fetch probability for sparse blobpool
}
// DefaultConfig contains the default configurations for the transaction pool.
diff --git a/core/txpool/blobpool/conversion.go b/core/txpool/blobpool/conversion.go
new file mode 100644
index 0000000000..714975cce3
--- /dev/null
+++ b/core/txpool/blobpool/conversion.go
@@ -0,0 +1,207 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package blobpool
+
+import (
+ "errors"
+ "slices"
+ "sync/atomic"
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/log"
+)
+
+// maxPendingConversionTasks caps the number of pending conversion tasks. This
+// prevents excessive memory usage; the worst-case scenario (2k transactions
+// with 6 blobs each) would consume approximately 1.5GB of memory.
+const maxPendingConversionTasks = 2048
+
+type convertResult struct {
+ ptx *BlobTxForPool
+ err error
+}
+
+// txConvert represents a conversion task with an attached legacy blob transaction.
+type txConvert struct {
+ tx *types.Transaction // Legacy blob transaction
+ done chan convertResult // Channel for signaling back if the conversion succeeds
+}
+
+// conversionQueue is a dedicated queue for converting legacy blob transactions
+// received from the network after the Osaka fork. Since conversion is expensive,
+// it is performed in the background by a single thread, ensuring the main Geth
+// process is not overloaded.
+type conversionQueue struct {
+ tasks chan *txConvert
+ startConversion chan func()
+ quit chan struct{}
+ closed chan struct{}
+
+ queue []func()
+ taskDone chan struct{}
+}
+
+// newConversionQueue constructs the conversion queue.
+func newConversionQueue() *conversionQueue {
+ q := &conversionQueue{
+ tasks: make(chan *txConvert),
+ startConversion: make(chan func()),
+ quit: make(chan struct{}),
+ closed: make(chan struct{}),
+ }
+ go q.loop()
+ return q
+}
+
+// convert accepts a legacy blob transaction with version-0 blobs and queues it
+// for conversion.
+//
+// This function may block for a long time until the transaction is processed.
+func (q *conversionQueue) convert(tx *types.Transaction) (*BlobTxForPool, error) {
+ done := make(chan convertResult, 1)
+ select {
+ case q.tasks <- &txConvert{tx: tx, done: done}:
+ res := <-done
+ return res.ptx, res.err
+ case <-q.closed:
+ return nil, errors.New("conversion queue closed")
+ }
+}
+
+// launchConversion starts a conversion task in the background.
+func (q *conversionQueue) launchConversion(fn func()) error {
+ select {
+ case q.startConversion <- fn:
+ return nil
+ case <-q.closed:
+ return errors.New("conversion queue closed")
+ }
+}
+
+// close terminates the conversion queue.
+func (q *conversionQueue) close() {
+ select {
+ case <-q.closed:
+ return
+ default:
+ close(q.quit)
+ <-q.closed
+ }
+}
+
+// run converts a batch of legacy blob txs to the new cell proof format.
+func (q *conversionQueue) run(tasks []*txConvert, done chan struct{}, interrupt *atomic.Int32) {
+ defer close(done)
+
+ for _, t := range tasks {
+ if interrupt != nil && interrupt.Load() != 0 {
+ t.done <- convertResult{err: errors.New("conversion is interrupted")}
+ continue
+ }
+ // Run the conversion, the original sidecar will be mutated in place
+ start := time.Now()
+ ptx, err := newBlobTxForPool(t.tx)
+ t.done <- convertResult{ptx: ptx, err: err}
+ log.Trace("Converted legacy blob tx", "hash", t.tx.Hash(), "err", err, "elapsed", common.PrettyDuration(time.Since(start)))
+ }
+}
+
+func (q *conversionQueue) loop() {
+ defer close(q.closed)
+
+ var (
+ done chan struct{} // Non-nil if background routine is active
+ interrupt *atomic.Int32 // Flag to signal conversion interruption
+
+ // The pending tasks for sidecar conversion. We assume the number of legacy
+ // blob transactions requiring conversion will not be excessive. However,
+ // a hard cap is applied as a protective measure.
+ txTasks []*txConvert
+ )
+
+ for {
+ select {
+ case t := <-q.tasks:
+ if len(txTasks) >= maxPendingConversionTasks {
+ t.done <- convertResult{err: errors.New("conversion queue is overloaded")}
+ continue
+ }
+ txTasks = append(txTasks, t)
+
+ // Launch the background conversion thread if it's idle
+ if done == nil {
+ done, interrupt = make(chan struct{}), new(atomic.Int32)
+
+ tasks := slices.Clone(txTasks)
+ txTasks = txTasks[:0]
+ go q.run(tasks, done, interrupt)
+ }
+
+ case <-done:
+ done, interrupt = nil, nil
+ if len(txTasks) > 0 {
+ done, interrupt = make(chan struct{}), new(atomic.Int32)
+ tasks := slices.Clone(txTasks)
+ txTasks = txTasks[:0]
+ go q.run(tasks, done, interrupt)
+ }
+
+ case fn := <-q.startConversion:
+ q.queue = append(q.queue, fn)
+ q.runNextTask()
+
+ case <-q.taskDone:
+ q.runNextTask()
+
+ case <-q.quit:
+ if done != nil {
+ log.Debug("Waiting for blob proof conversion to exit")
+ interrupt.Store(1)
+ <-done
+ }
+ if q.taskDone != nil {
+ log.Debug("Waiting for blobpool billy conversion to exit")
+ <-q.taskDone
+ }
+ // Signal any tasks that were queued for the next batch but never started
+ // so callers blocked in convert() receive an error instead of hanging.
+ for _, t := range txTasks {
+ // Best-effort notify; t.done is a buffered channel of size 1
+ // created by convert(), and we send exactly once per task.
+ t.done <- convertResult{err: errors.New("conversion queue closed")}
+ }
+ // Drop references to allow GC of the backing array.
+ txTasks = txTasks[:0]
+ return
+ }
+ }
+}
+
+func (q *conversionQueue) runNextTask() {
+ if len(q.queue) == 0 {
+ q.taskDone = nil
+ return
+ }
+ fn := q.queue[0]
+ q.queue = append(q.queue[:0], q.queue[1:]...)
+
+ done := make(chan struct{})
+ go func() { defer close(done); fn() }()
+ q.taskDone = done
+}
diff --git a/core/txpool/blobpool/conversion_test.go b/core/txpool/blobpool/conversion_test.go
new file mode 100644
index 0000000000..b52239641e
--- /dev/null
+++ b/core/txpool/blobpool/conversion_test.go
@@ -0,0 +1,114 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package blobpool
+
+import (
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/params"
+)
+
+func TestConversionQueueBasic(t *testing.T) {
+ queue := newConversionQueue()
+ defer queue.close()
+
+ key, _ := crypto.GenerateKey()
+ tx := makeTx(0, 1, 1, 1, key)
+
+ ptx, err := queue.convert(tx)
+ if err != nil {
+ t.Fatalf("Expected successful conversion, got error: %v", err)
+ }
+ if ptx == nil {
+ t.Fatal("Expected a converted transaction, got nil")
+ }
+ if ptx.Tx.Hash() != tx.Hash() {
+ t.Errorf("Converted tx hash mismatch: have %s, want %s", ptx.Tx.Hash(), tx.Hash())
+ }
+ if len(ptx.CellSidecar.Cells) == 0 {
+ t.Error("Expected cells to be computed during conversion")
+ }
+}
+
+func TestConversionQueueClosed(t *testing.T) {
+ queue := newConversionQueue()
+ queue.close()
+
+ key, _ := crypto.GenerateKey()
+ tx := makeTx(0, 1, 1, 1, key)
+
+ if _, err := queue.convert(tx); err == nil {
+ t.Fatal("Expected error when converting on closed queue, got nil")
+ }
+}
+
+func TestConversionQueueDoubleClose(t *testing.T) {
+ queue := newConversionQueue()
+ queue.close()
+ queue.close() // Should not panic
+}
+
+func TestConversionQueueAutoRestartBatch(t *testing.T) {
+ queue := newConversionQueue()
+ defer queue.close()
+
+ key, _ := crypto.GenerateKey()
+
+ // Create a heavy transaction to ensure the first batch runs long enough
+ // for subsequent tasks to be queued while it is active.
+ heavy := makeMultiBlobTx(0, 1, 1, 1, int(params.BlobTxMaxBlobs), 0, key)
+
+ var wg sync.WaitGroup
+ wg.Add(1)
+ heavyDone := make(chan error, 1)
+ go func() {
+ defer wg.Done()
+ _, err := queue.convert(heavy)
+ heavyDone <- err
+ }()
+
+ // Give the conversion worker a head start so that the following tasks are
+ // enqueued while the first batch is running.
+ time.Sleep(200 * time.Millisecond)
+
+ tx1 := makeTx(1, 1, 1, 1, key)
+ tx2 := makeTx(2, 1, 1, 1, key)
+
+ wg.Add(2)
+ done1 := make(chan error, 1)
+ done2 := make(chan error, 1)
+ go func() { defer wg.Done(); _, err := queue.convert(tx1); done1 <- err }()
+ go func() { defer wg.Done(); _, err := queue.convert(tx2); done2 <- err }()
+
+ for _, c := range []struct {
+ name string
+ done chan error
+ }{{"tx1", done1}, {"tx2", done2}, {"heavy", heavyDone}} {
+ select {
+ case err := <-c.done:
+ if err != nil {
+ t.Fatalf("%s conversion error: %v", c.name, err)
+ }
+ case <-time.After(30 * time.Second):
+ t.Fatalf("timeout waiting for %s conversion", c.name)
+ }
+ }
+ wg.Wait()
+}
diff --git a/core/txpool/blobpool/limbo.go b/core/txpool/blobpool/limbo.go
index f941f06ec4..e599877f2a 100644
--- a/core/txpool/blobpool/limbo.go
+++ b/core/txpool/blobpool/limbo.go
@@ -33,7 +33,7 @@ import (
type limboBlob struct {
TxHash common.Hash // Owner transaction's hash to support resurrecting reorged txs
Block uint64 // Block in which the blob transaction was included
- Ptx *blobTxForPool
+ Ptx *BlobTxForPool
}
// limbo is a light, indexed database to temporarily store recently included
@@ -49,7 +49,7 @@ type limbo struct {
}
// newLimbo opens and indexes a set of limboed blob transactions.
-func newLimbo(config *params.ChainConfig, datadir string) (*limbo, error) {
+func newLimbo(config *params.ChainConfig, datadir string) (*limbo, []uint64, error) {
l := &limbo{
index: make(map[common.Hash]uint64),
groups: make(map[uint64]map[uint64]common.Hash),
@@ -61,7 +61,7 @@ func newLimbo(config *params.ChainConfig, datadir string) (*limbo, error) {
// See if we need to migrate the limbo after fusaka.
slotter, err := tryMigrate(config, slotter, datadir)
if err != nil {
- return nil, err
+ return nil, nil, err
}
// Index all limboed blobs on disk and delete anything unprocessable
@@ -71,62 +71,30 @@ func newLimbo(config *params.ChainConfig, datadir string) (*limbo, error) {
)
index := func(id uint64, size uint32, data []byte) {
err := l.parseBlob(id, data)
- if err != nil {
- fails = append(fails, id)
- }
if errors.Is(err, errLegacyTx) {
convert = append(convert, id)
+ return
+ }
+ if err != nil {
+ fails = append(fails, id)
}
}
store, err := billy.Open(billy.Options{Path: datadir, Repair: true}, slotter, index)
if err != nil {
- return nil, err
+ return nil, nil, err
}
l.store = store
- // Migrate legacy limbo entries to blobTxForPool. Note all ids in `converted` are also
- // in `fails`, and the converted entries will be deleted by the loop that handles
- // `fails`.
- for _, id := range convert {
- data, err := l.store.Get(id)
- if err != nil {
- continue
- }
- var legacy struct {
- TxHash common.Hash
- Block uint64
- Tx *types.Transaction
- }
- if err := rlp.DecodeBytes(data, &legacy); err != nil {
- continue
- }
- blob, err := rlp.EncodeToBytes(&limboBlob{
- TxHash: legacy.TxHash,
- Block: legacy.Block,
- Ptx: newBlobTxForPool(legacy.Tx),
- })
- if err != nil {
- continue
- }
- newID, err := l.store.Put(blob)
- if err != nil {
- continue
- }
- if err := l.parseBlob(newID, blob); err != nil {
- fails = append(fails, newID)
- }
- }
-
if len(fails) > 0 {
log.Warn("Dropping invalidated limboed blobs", "ids", fails)
for _, id := range fails {
if err := l.store.Delete(id); err != nil {
l.Close()
- return nil, err
+ return nil, nil, err
}
}
}
- return l, nil
+ return l, convert, nil
}
// Close closes down the underlying persistent store.
@@ -201,7 +169,9 @@ func (l *limbo) finalize(final *types.Header) {
// push stores a new blob transaction into the limbo, waiting until finality for
// it to be automatically evicted.
-func (l *limbo) push(ptx *blobTxForPool, block uint64) error {
+func (l *limbo) push(ptx *BlobTxForPool, block uint64) error {
+ // If the blobs are already tracked by the limbo, consider it a programming
+ // error. There's not much to do against it, but be loud.
hash := ptx.Tx.Hash()
if _, ok := l.index[hash]; ok {
log.Error("Limbo cannot push already tracked blobs", "tx", hash)
@@ -217,7 +187,7 @@ func (l *limbo) push(ptx *blobTxForPool, block uint64) error {
// pull retrieves a previously pushed set of blobs back from the limbo, removing
// it at the same time. This method should be used when a previously included blob
// transaction gets reorged out.
-func (l *limbo) pull(tx common.Hash) (*blobTxForPool, error) {
+func (l *limbo) pull(tx common.Hash) (*BlobTxForPool, error) {
// If the blobs are not tracked by the limbo, there's not much to do. This
// can happen for example if a blob transaction is mined without pushing it
// into the network first.
@@ -294,7 +264,7 @@ func (l *limbo) getAndDrop(id uint64) (*limboBlob, error) {
// setAndIndex assembles a limbo blob database entry and stores it, also updating
// the in-memory indices.
-func (l *limbo) setAndIndex(ptx *blobTxForPool, block uint64) error {
+func (l *limbo) setAndIndex(ptx *BlobTxForPool, block uint64) error {
txhash := ptx.Tx.Hash()
item := &limboBlob{
TxHash: txhash,
diff --git a/core/txpool/blobpool/limbo_test.go b/core/txpool/blobpool/limbo_test.go
index 722587b5c1..04cb5f5c20 100644
--- a/core/txpool/blobpool/limbo_test.go
+++ b/core/txpool/blobpool/limbo_test.go
@@ -1,4 +1,4 @@
-// Copyright 2026 The go-ethereum Authors
+// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@@ -27,8 +27,9 @@ import (
"github.com/holiman/billy"
)
-// TestLimboLegacyMigration checks whether limbo entries in the legacy limboBlob type
-// are migrated to the blobTxForPool layout on startup instead of being dropped.
+// TestLimboLegacyMigration checks that a limbo entry in the legacy limboBlob
+// layout is flagged for conversion by newLimbo instead of being dropped, so the
+// conversion queue can migrate it to the blobTxForPool layout.
func TestLimboLegacyMigration(t *testing.T) {
key, _ := crypto.GenerateKey()
tx := makeMultiBlobTx(0, 10, 100, 100, 2, 0, key)
@@ -49,24 +50,20 @@ func TestLimboLegacyMigration(t *testing.T) {
if err != nil {
t.Fatalf("failed to encode legacy entry: %v", err)
}
- if _, err := store.Put(data); err != nil {
+ id, err := store.Put(data)
+ if err != nil {
t.Fatalf("failed to store legacy entry: %v", err)
}
store.Close()
- // Open the limbo, which should migrate the legacy entry.
- l, err := newLimbo(new(params.ChainConfig), dir)
+ // Open the limbo, which should flag the legacy entry for conversion.
+ l, convert, err := newLimbo(new(params.ChainConfig), dir)
if err != nil {
t.Fatalf("failed to open limbo: %v", err)
}
defer l.Close()
- // The migrated transaction must be tracked and reconstruct the original.
- ptx, err := l.pull(tx.Hash())
- if err != nil {
- t.Fatalf("failed to pull migrated tx: %v", err)
- }
- if got := ptx.ToTx().Hash(); got != tx.Hash() {
- t.Fatalf("migrated tx hash mismatch: got %x, want %x", got, tx.Hash())
+ if len(convert) != 1 || convert[0] != id {
+ t.Fatalf("legacy entry not flagged for conversion: got %v, want [%d]", convert, id)
}
}
diff --git a/core/txpool/blobpool/lookup.go b/core/txpool/blobpool/lookup.go
index 2911d696d2..0b916859b9 100644
--- a/core/txpool/blobpool/lookup.go
+++ b/core/txpool/blobpool/lookup.go
@@ -18,11 +18,15 @@ package blobpool
import (
"github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/types"
)
type txMetadata struct {
- id uint64 // the billy id of transaction
- size uint64 // the RLP encoded size of transaction (blobs are included)
+ id uint64 // the billy id of transaction
+ size uint64 // the RLP encoded size of transaction (blobs are included)
+ sizeWithoutBlob uint64 // the RLP encoded size without blob data (for ETH/72 announcements)
+ custody types.CustodyBitmap
+ vhashes []common.Hash // blob versioned hashes for the transaction
}
// lookup maps blob versioned hashes to transaction hashes that include them,
@@ -56,6 +60,15 @@ func (l *lookup) storeidOfTx(txhash common.Hash) (uint64, bool) {
return meta.id, true
}
+// blobHashesOfTx returns the blob versioned hashes for a transaction.
+func (l *lookup) blobHashesOfTx(txhash common.Hash) ([]common.Hash, bool) {
+ meta, ok := l.txIndex[txhash]
+ if !ok {
+ return nil, false
+ }
+ return meta.vhashes, true
+}
+
// storeidOfBlob returns the datastore storage item id of a blob.
func (l *lookup) storeidOfBlob(vhash common.Hash) (uint64, bool) {
// If the blob is unknown, return a miss
@@ -91,8 +104,11 @@ func (l *lookup) track(tx *blobTxMeta) {
}
// Map the transaction hash to the datastore id and RLP-encoded transaction size
l.txIndex[tx.hash] = &txMetadata{
- id: tx.id,
- size: tx.size,
+ id: tx.id,
+ size: tx.size,
+ sizeWithoutBlob: tx.sizeWithoutBlob,
+ custody: *tx.custody,
+ vhashes: tx.vhashes,
}
}
diff --git a/core/txpool/errors.go b/core/txpool/errors.go
index 8285cbf10e..b1c8b2006e 100644
--- a/core/txpool/errors.go
+++ b/core/txpool/errors.go
@@ -74,4 +74,7 @@ var (
// ErrKZGVerificationError is returned when a KZG proof was not verified correctly.
ErrKZGVerificationError = errors.New("KZG verification error")
+
+ // ErrSidecarFormatError is returned when sidecar is malformed
+ ErrSidecarFormatError = errors.New("Wrong sidecar format")
)
diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go
index 3d66803fd7..edb5fccbfe 100644
--- a/core/txpool/legacypool/legacypool.go
+++ b/core/txpool/legacypool/legacypool.go
@@ -1010,7 +1010,7 @@ func (pool *LegacyPool) get(hash common.Hash) *types.Transaction {
}
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
-func (pool *LegacyPool) GetRLP(hash common.Hash) []byte {
+func (pool *LegacyPool) GetRLP(hash common.Hash, _ uint) []byte {
tx := pool.all.Get(hash)
if tx == nil {
return nil
diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go
index 4cc1b193d6..70a2d9300f 100644
--- a/core/txpool/subpool.go
+++ b/core/txpool/subpool.go
@@ -80,14 +80,16 @@ type PendingFilter struct {
// When BlobTxs true, return only blob transactions (block blob-space filling)
// when false, return only non-blob txs (peer-join announces, block space filling)
- BlobTxs bool
- BlobVersion byte // Blob tx version to include. 0 means pre-Osaka, 1 means Osaka and later
+ BlobTxs bool
+ PartialCells bool
+ BlobVersion byte // Blob tx version to include. 0 means pre-Osaka, 1 means Osaka and later
}
// TxMetadata denotes the metadata of a transaction.
type TxMetadata struct {
- Type uint8 // The type of the transaction
- Size uint64 // The length of the 'rlp encoding' of a transaction
+ Type uint8 // The type of the transaction
+ Size uint64 // The length of the 'rlp encoding' of a transaction (including blobs)
+ SizeWithoutBlob uint64 // The length without blob data (for ETH/72 announcements)
}
// SubPool represents a specialized transaction pool that lives on its own (e.g.
@@ -132,7 +134,7 @@ type SubPool interface {
Get(hash common.Hash) *types.Transaction
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
- GetRLP(hash common.Hash) []byte
+ GetRLP(hash common.Hash, version uint) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go
index cb425e5809..a0d934723d 100644
--- a/core/txpool/txpool.go
+++ b/core/txpool/txpool.go
@@ -288,9 +288,9 @@ func (p *TxPool) Get(hash common.Hash) *types.Transaction {
}
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
-func (p *TxPool) GetRLP(hash common.Hash) []byte {
+func (p *TxPool) GetRLP(hash common.Hash, version uint) []byte {
for _, subpool := range p.subpools {
- encoded := subpool.GetRLP(hash)
+ encoded := subpool.GetRLP(hash, version)
if len(encoded) != 0 {
return encoded
}
diff --git a/core/txpool/validation.go b/core/txpool/validation.go
index 27cbb84ad1..0e75ae7718 100644
--- a/core/txpool/validation.go
+++ b/core/txpool/validation.go
@@ -65,9 +65,6 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
if opts.Accept&(1< opts.MaxBlobCount {
- return fmt.Errorf("%w: blob count %v, limit %v", ErrTxBlobLimitExceeded, blobCount, opts.MaxBlobCount)
- }
// Before performing any expensive validations, sanity check that the tx is
// smaller than the maximum limit the pool can meaningfully handle
if tx.Size() > opts.MaxSize {
@@ -161,7 +158,7 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.GasTipCap(), opts.MinTip)
}
if tx.Type() == types.BlobTxType {
- return validateBlobTx(tx)
+ return validateBlobSidecar(tx, head, opts)
}
if tx.Type() == types.SetCodeTxType {
if len(tx.SetCodeAuthorizations()) == 0 {
@@ -171,42 +168,67 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
return nil
}
-// validateBlobTx implements the blob-transaction specific validations.
-func validateBlobTx(tx *types.Transaction) error {
+// validateBlobSidecar implements the blob sidecar validation.
+// Note that this doesn't verify the consistency between blobs(cells) and
+// proofs. For proof verification, use validateCells.
+func validateBlobSidecar(tx *types.Transaction, head *types.Header, opts *ValidationOptions) error {
+ if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
+ return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
+ }
sidecar := tx.BlobTxSidecar()
if sidecar == nil {
return errors.New("missing sidecar in blob transaction")
}
- // Ensure the sidecar is constructed with the correct version
- if sidecar.Version != types.BlobSidecarVersion1 {
- return fmt.Errorf("unexpected sidecar version, want: %d, got: %d", types.BlobSidecarVersion1, sidecar.Version)
- }
- // Ensure the blob fee cap satisfies the minimum blob gas price
- if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
- return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
- }
- // Ensure the number of items in the blob transaction and various side
- // data match up before doing any expensive validations
hashes := tx.BlobHashes()
- if len(hashes) == 0 {
- return errors.New("blobless blob transaction")
- }
- if len(hashes) > params.BlobTxMaxBlobs {
- return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.BlobTxMaxBlobs)
- }
- if len(sidecar.Blobs) != len(hashes) {
- return fmt.Errorf("invalid number of %d blobs compared to %d blob hashes", len(sidecar.Blobs), len(hashes))
- }
if err := sidecar.ValidateBlobCommitmentHashes(hashes); err != nil {
return err
}
- return validateBlobSidecarOsaka(sidecar, hashes)
-}
-func validateBlobSidecarOsaka(sidecar *types.BlobTxSidecar, hashes []common.Hash) error {
- if len(sidecar.Proofs) != len(hashes)*kzg4844.CellProofsPerBlob {
- return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes)*kzg4844.CellProofsPerBlob)
+ if len(hashes) > opts.MaxBlobCount {
+ return fmt.Errorf("%w: blob count %v, limit %v", ErrTxBlobLimitExceeded, len(hashes), opts.MaxBlobCount)
}
- if err := kzg4844.VerifyCellProofs(sidecar.Blobs, sidecar.Commitments, sidecar.Proofs); err != nil {
+ if sidecar.Version != types.BlobSidecarVersion1 {
+ return fmt.Errorf("%w: unexpected sidecar version, want: %d, got: %d", ErrSidecarFormatError, types.BlobSidecarVersion1, sidecar.Version)
+ }
+ if len(sidecar.Proofs) != len(sidecar.Commitments)*kzg4844.CellProofsPerBlob {
+ return fmt.Errorf("%w: invalid number of %d blob proofs expected %d", ErrSidecarFormatError, len(sidecar.Proofs), len(sidecar.Commitments)*kzg4844.CellProofsPerBlob)
+ }
+ return nil
+}
+
+func ValidateCells(sidecar *types.BlobTxCellSidecar) error {
+ // Two checks here (custody count check and blobCount check) is duplicated in buffer.go
+ // However it is required to 1) serve eth71 peer and direct submission 2) catch any bug in
+ // merging cell delivery.
+ if sidecar.Custody.OneCount() == 0 {
+ return errors.New("blobless blob transaction")
+ }
+ // Verify whether the blob count is consistent with other parts of the sidecar and the transaction
+ blobCount := len(sidecar.Cells) / sidecar.Custody.OneCount()
+ if blobCount == 0 {
+ return errors.New("blobless blob transaction")
+ }
+ if blobCount != len(sidecar.Commitments) {
+ return fmt.Errorf("invalid number of %d blobs compared to %d commitments", blobCount, len(sidecar.Commitments))
+ }
+ if len(sidecar.Proofs) != len(sidecar.Commitments)*kzg4844.CellProofsPerBlob {
+ return fmt.Errorf("invalid number of %d proofs compared to %d commitments", len(sidecar.Proofs), len(sidecar.Commitments))
+ }
+ if sidecar.Version != types.BlobSidecarVersion1 {
+ return fmt.Errorf("unexpected sidecar version, want: %d, got: %d", types.BlobSidecarVersion1, sidecar.Version)
+ }
+ return validateCellsOsaka(sidecar)
+}
+
+func validateCellsOsaka(sidecar *types.BlobTxCellSidecar) error {
+ indices := sidecar.Custody.Indices()
+ cellProofs := make([]kzg4844.Proof, 0)
+ for blobIdx := range len(sidecar.Commitments) {
+ for _, proofIdx := range indices {
+ idx := blobIdx*kzg4844.CellProofsPerBlob + int(proofIdx)
+ cellProofs = append(cellProofs, sidecar.Proofs[idx])
+ }
+ }
+ if err := kzg4844.VerifyCells(sidecar.Cells, sidecar.Commitments, cellProofs, sidecar.Custody.Indices()); err != nil {
return fmt.Errorf("%w: %v", ErrKZGVerificationError, err)
}
return nil
diff --git a/core/types/custody_bitmap.go b/core/types/custody_bitmap.go
new file mode 100644
index 0000000000..ba3740d147
--- /dev/null
+++ b/core/types/custody_bitmap.go
@@ -0,0 +1,152 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package types
+
+import (
+ "fmt"
+ "math/bits"
+ "math/rand"
+
+ "github.com/ethereum/go-ethereum/common/hexutil"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
+)
+
+// CustodyBitmap is a bitmap to represent which custody index to store (little endian).
+type CustodyBitmap [16]byte
+
+// MarshalText implements encoding.TextMarshaler.
+func (b CustodyBitmap) MarshalText() ([]byte, error) {
+ return []byte(hexutil.Encode(b[:])), nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (b *CustodyBitmap) UnmarshalText(input []byte) error {
+ decoded, err := hexutil.Decode(string(input))
+ if err != nil {
+ return fmt.Errorf("custody bitmap: %v", err)
+ }
+ if len(decoded) != len(b) {
+ return fmt.Errorf("custody bitmap: invalid length %d, want %d", len(decoded), len(b))
+ }
+ copy(b[:], decoded)
+ return nil
+}
+
+var (
+ CustodyBitmapAll = func() CustodyBitmap {
+ var result CustodyBitmap
+ for i := range result {
+ result[i] = 0xFF
+ }
+ return result
+ }()
+
+ CustodyBitmapData = func() CustodyBitmap {
+ var result CustodyBitmap
+ for i := 0; i < kzg4844.DataPerBlob/8; i++ {
+ result[i] = 0xFF
+ }
+ return result
+ }()
+)
+
+func NewCustodyBitmap(indices []uint64) CustodyBitmap {
+ var result CustodyBitmap
+ for _, i := range indices {
+ if i >= uint64(kzg4844.CellsPerBlob) {
+ panic("CustodyBitmap: bit index out of range")
+ }
+ result[i/8] |= 1 << (i % 8)
+ }
+ return result
+}
+
+// NewRandomCustodyBitmap creates a CustodyBitmap with n randomly selected indices.
+// This should be used only for tests.
+func NewRandomCustodyBitmap(n int) CustodyBitmap {
+ if n <= 0 || n > kzg4844.CellsPerBlob {
+ panic("CustodyBitmap: invalid number of indices")
+ }
+ indices := make([]uint64, 0, n)
+ used := make(map[uint64]bool)
+ for len(indices) < n {
+ idx := uint64(rand.Intn(kzg4844.CellsPerBlob))
+ if !used[idx] {
+ used[idx] = true
+ indices = append(indices, idx)
+ }
+ }
+ return NewCustodyBitmap(indices)
+}
+
+// IsSet returns whether bit i is set.
+func (b CustodyBitmap) IsSet(i uint64) bool {
+ if i >= uint64(kzg4844.CellsPerBlob) {
+ return false
+ }
+ return (b[i/8]>>(i%8))&1 == 1
+}
+
+// OneCount returns the number of bits set to 1.
+func (b CustodyBitmap) OneCount() int {
+ total := 0
+ for _, v := range b {
+ total += bits.OnesCount8(v)
+ }
+ return total
+}
+
+// Indices returns the bit positions set to 1, in ascending order.
+func (b CustodyBitmap) Indices() []uint64 {
+ out := make([]uint64, 0, b.OneCount())
+ for byteIdx, val := range b {
+ v := val
+ for v != 0 {
+ tz := bits.TrailingZeros8(v)
+ out = append(out, uint64(byteIdx*8+tz))
+ v &^= 1 << tz
+ }
+ }
+ return out
+}
+
+// Difference returns b AND NOT set (bits in b but not in set).
+func (b CustodyBitmap) Difference(set CustodyBitmap) CustodyBitmap {
+ var out CustodyBitmap
+ for i := range b {
+ out[i] = b[i] &^ set[i]
+ }
+ return out
+}
+
+// Intersection returns b AND set.
+func (b CustodyBitmap) Intersection(set CustodyBitmap) CustodyBitmap {
+ var out CustodyBitmap
+ for i := range b {
+ out[i] = b[i] & set[i]
+ }
+ return out
+}
+
+// Union returns b OR set.
+func (b CustodyBitmap) Union(set CustodyBitmap) CustodyBitmap {
+ var out CustodyBitmap
+ for i := range b {
+ out[i] = b[i] | set[i]
+ }
+ return out
+}
diff --git a/core/types/tx_blob.go b/core/types/tx_blob.go
index 90603341e6..b5789eea32 100644
--- a/core/types/tx_blob.go
+++ b/core/types/tx_blob.go
@@ -176,6 +176,16 @@ func (sc *BlobTxSidecar) Copy() *BlobTxSidecar {
}
}
+// BlobTxCellSidecar is a sidecar that carries cells instead of blobs.
+// The Custody field represents which cells of each blob this sidecar contains.
+type BlobTxCellSidecar struct {
+ Version byte
+ Cells []kzg4844.Cell
+ Commitments []kzg4844.Commitment
+ Proofs []kzg4844.Proof
+ Custody CustodyBitmap
+}
+
// blobTxWithBlobs represents blob tx with its corresponding sidecar.
// This is an interface because sidecars are versioned.
type blobTxWithBlobs interface {
diff --git a/eth/backend.go b/eth/backend.go
index c880e75f27..ba9b4d4459 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -44,6 +44,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig"
+ "github.com/ethereum/go-ethereum/eth/fetcher"
"github.com/ethereum/go-ethereum/eth/gasprice"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/eth/protocols/snap"
@@ -349,15 +350,17 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
// Permit the downloader to use the trie cache allowance during fast sync
cacheLimit := options.TrieCleanLimit + options.TrieDirtyLimit + options.SnapshotLimit
if eth.handler, err = newHandler(&handlerConfig{
- NodeID: eth.p2pServer.Self().ID(),
- Database: chainDb,
- Chain: eth.blockchain,
- TxPool: eth.txPool,
- Network: networkID,
- Sync: config.SyncMode,
- BloomCache: uint64(cacheLimit),
- RequiredBlocks: config.RequiredBlocks,
- SnapV2: config.SnapV2,
+ NodeID: eth.p2pServer.Self().ID(),
+ Database: chainDb,
+ Chain: eth.blockchain,
+ TxPool: eth.txPool,
+ BlobPool: eth.blobTxPool,
+ Network: networkID,
+ Sync: config.SyncMode,
+ BloomCache: uint64(cacheLimit),
+ RequiredBlocks: config.RequiredBlocks,
+ SnapV2: config.SnapV2,
+ FetchProbability: config.BlobPool.FetchProbability,
}); err != nil {
return nil, err
}
@@ -441,6 +444,7 @@ func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager
func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain }
func (s *Ethereum) TxPool() *txpool.TxPool { return s.txPool }
func (s *Ethereum) BlobTxPool() *blobpool.BlobPool { return s.blobTxPool }
+func (s *Ethereum) BlobFetcher() *fetcher.BlobFetcher { return s.handler.blobFetcher }
func (s *Ethereum) BlobCache() *blobpool.Cache { return s.blobCache }
func (s *Ethereum) Engine() consensus.Engine { return s.engine }
func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb }
diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go
index d66f96148d..46f8cc98fc 100644
--- a/eth/catalyst/api.go
+++ b/eth/catalyst/api.go
@@ -22,6 +22,7 @@ import (
"errors"
"fmt"
"reflect"
+ "slices"
"strconv"
"sync"
"sync/atomic"
@@ -219,7 +220,7 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV3(ctx context.Context, update engine.
// ForkchoiceUpdatedV4 is equivalent to V3 with the addition of slot number
// in the payload attributes. It supports only PayloadAttributesV4.
-func (api *ConsensusAPI) ForkchoiceUpdatedV4(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) {
+func (api *ConsensusAPI) ForkchoiceUpdatedV4(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes, custodyColumns *types.CustodyBitmap) (engine.ForkChoiceResponse, error) {
if params != nil {
switch {
case params.Withdrawals == nil:
@@ -232,6 +233,9 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV4(ctx context.Context, update engine.
return engine.STATUS_INVALID, unsupportedForkErr("fcuV4 must only be called for amsterdam payloads")
}
}
+ if custodyColumns != nil {
+ api.eth.BlobFetcher().UpdateCustody(*custodyColumns)
+ }
// TODO(matt): the spec requires that fcu is applied when called on a valid
// hash, even if params are wrong. To do this we need to split up
// forkchoiceUpdate into a function that only updates the head and then a
@@ -710,6 +714,63 @@ func (api *ConsensusAPI) getBlobs(ctx context.Context, hashes []common.Hash, v2
return res, nil
}
+// GetBlobsV4 returns cell-level blob data from the transaction pool.
+// V4 returns only the requested cells as specified by the indices_bitarray.
+func (api *ConsensusAPI) GetBlobsV4(hashes []common.Hash, indicesBitarray types.CustodyBitmap) ([]*engine.BlobCellsAndProofsV1, error) {
+ head := api.eth.BlockChain().CurrentHeader()
+ // Sparse blobpool is not necessarily coupled with the Amsterdam fork and
+ // can technically be supported after the Osaka fork
+ // (where cell proofs are introduced).
+ if api.config().LatestFork(head.Time) < forks.Osaka {
+ return nil, nil
+ }
+ if len(hashes) > 128 {
+ return nil, engine.TooLargeRequest.With(fmt.Errorf("requested blob count too large: %v", len(hashes)))
+ }
+ cells, proofs, err := api.eth.BlobCache().GetCells(hashes, indicesBitarray)
+ if err != nil {
+ return nil, engine.InvalidParams.With(err)
+ }
+ var (
+ res = make([]*engine.BlobCellsAndProofsV1, len(hashes))
+ hitCount int
+ )
+ getBlobsRequestedCounter.Inc(int64(len(hashes)))
+ for i := range hashes {
+ if cells[i] == nil || proofs[i] == nil {
+ continue
+ }
+ hitCount++
+ blobCells := make([]*hexutil.Bytes, len(cells[i]))
+ for j, cell := range cells[i] {
+ if cell != nil {
+ b := hexutil.Bytes(cell[:])
+ blobCells[j] = &b
+ }
+ }
+ blobProofs := make([]*hexutil.Bytes, len(proofs[i]))
+ for j, proof := range proofs[i] {
+ if proof != nil {
+ b := hexutil.Bytes(proof[:])
+ blobProofs[j] = &b
+ }
+ }
+ res[i] = &engine.BlobCellsAndProofsV1{
+ BlobCells: blobCells,
+ Proofs: blobProofs,
+ }
+ }
+ getBlobsAvailableCounter.Inc(int64(hitCount))
+ if hitCount == len(hashes) {
+ getBlobsRequestCompleteHit.Inc(1)
+ } else if hitCount > 0 {
+ getBlobsRequestPartialHit.Inc(1)
+ } else {
+ getBlobsRequestMiss.Inc(1)
+ }
+ return res, nil
+}
+
// HasBlobs reports availability for the requested blob-versioned-hashes.
func (api *ConsensusAPI) HasBlobs(hashes []common.Hash) []bool {
return api.eth.BlobCache().HasBlobs(context.Background(), hashes)
@@ -1139,17 +1200,26 @@ func (api *ConsensusAPI) checkFork(timestamp uint64, forks ...forks.Fork) bool {
}
// ExchangeCapabilities returns the current methods provided by this node.
-func (api *ConsensusAPI) ExchangeCapabilities([]string) []string {
+func (api *ConsensusAPI) ExchangeCapabilities(caps []string) []string {
valueT := reflect.TypeOf(api)
- caps := make([]string, 0, valueT.NumMethod())
+
+ // If the CL supports getBlobsV4, we call EnableCell() on the
+ // blob cache to skip the blob recovery process. This is a
+ // one-directional toggle, which assumes that once the CL
+ // supports getBlobsV4, it will not fall back to getBlobsV3
+ // again.
+ cellmode := slices.Contains(caps, "engine_getBlobsV4")
+ api.eth.BlobCache().SetCellMode(cellmode)
+
+ ourCaps := make([]string, 0, valueT.NumMethod())
for i := 0; i < valueT.NumMethod(); i++ {
name := []rune(valueT.Method(i).Name)
if string(name) == "ExchangeCapabilities" {
continue
}
- caps = append(caps, "engine_"+string(unicode.ToLower(name[0]))+string(name[1:]))
+ ourCaps = append(ourCaps, "engine_"+string(unicode.ToLower(name[0]))+string(name[1:]))
}
- return caps
+ return ourCaps
}
// GetClientVersionV1 exchanges client version data of this node.
diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go
index 2d3c4929ea..c4298b25ab 100644
--- a/eth/catalyst/api_test.go
+++ b/eth/catalyst/api_test.go
@@ -38,6 +38,7 @@ import (
"github.com/ethereum/go-ethereum/consensus/beacon"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/types/bal"
"github.com/ethereum/go-ethereum/crypto"
@@ -1969,23 +1970,27 @@ func init() {
// makeMultiBlobTx is a utility method to construct a random blob tx with
// certain number of blobs in its sidecar.
-func makeMultiBlobTx(chainConfig *params.ChainConfig, nonce uint64, blobCount int, blobOffset int, key *ecdsa.PrivateKey, version byte) *types.Transaction {
+func makeMultiBlobTx(chainConfig *params.ChainConfig, nonce uint64, blobCount int, blobOffset int, key *ecdsa.PrivateKey, version byte, custody types.CustodyBitmap) *blobpool.BlobTxForPool {
+ indices := custody.Indices()
var (
- blobs []kzg4844.Blob
blobHashes []common.Hash
commitments []kzg4844.Commitment
proofs []kzg4844.Proof
+ cells []kzg4844.Cell
)
for i := 0; i < blobCount; i++ {
- blobs = append(blobs, *testBlobs[blobOffset+i])
- commitments = append(commitments, testBlobCommits[blobOffset+i])
+ j := blobOffset + i
+ blobHashes = append(blobHashes, testBlobVHashes[j])
+ commitments = append(commitments, testBlobCommits[j])
if version == types.BlobSidecarVersion0 {
- proofs = append(proofs, testBlobProofs[blobOffset+i])
+ proofs = append(proofs, testBlobProofs[j])
} else {
- cellProofs, _ := kzg4844.ComputeCellProofs(testBlobs[blobOffset+i])
- proofs = append(proofs, cellProofs...)
+ proofs = append(proofs, testBlobCellProofs[j]...)
+ }
+ full, _ := kzg4844.ComputeCells([]kzg4844.Blob{*testBlobs[j]})
+ for _, idx := range indices {
+ cells = append(cells, full[idx])
}
- blobHashes = append(blobHashes, testBlobVHashes[blobOffset+i])
}
blobtx := &types.BlobTx{
ChainID: uint256.MustFromBig(chainConfig.ChainID),
@@ -1996,12 +2001,20 @@ func makeMultiBlobTx(chainConfig *params.ChainConfig, nonce uint64, blobCount in
BlobFeeCap: uint256.NewInt(1000),
BlobHashes: blobHashes,
Value: uint256.NewInt(100),
- Sidecar: types.NewBlobTxSidecar(version, blobs, commitments, proofs),
}
- return types.MustSignNewTx(key, types.LatestSigner(chainConfig), blobtx)
+ return &blobpool.BlobTxForPool{
+ Tx: types.MustSignNewTx(key, types.LatestSigner(chainConfig), blobtx),
+ CellSidecar: &types.BlobTxCellSidecar{
+ Version: version,
+ Cells: cells,
+ Commitments: commitments,
+ Proofs: proofs,
+ Custody: custody,
+ },
+ }
}
-func newGetBlobEnv(t testing.TB, version byte) (*node.Node, *ConsensusAPI) {
+func newGetBlobEnv(t testing.TB, version byte, custody types.CustodyBitmap) (*node.Node, *ConsensusAPI) {
var (
// Create a database pre-initialize with a genesis block
config = *params.MergedTestChainConfig
@@ -2030,17 +2043,23 @@ func newGetBlobEnv(t testing.TB, version byte) (*node.Node, *ConsensusAPI) {
}
n, ethServ := startEthService(t, gspec, nil)
- // fill blob txs into the pool
- tx1 := makeMultiBlobTx(&config, 0, 2, 0, key1, version) // blob[0, 2)
- tx2 := makeMultiBlobTx(&config, 0, 2, 2, key2, version) // blob[2, 4)
- tx3 := makeMultiBlobTx(&config, 0, 2, 4, key3, version) // blob[4, 6)
- ethServ.TxPool().Add([]*types.Transaction{tx1, tx2, tx3}, true)
+ // fill blob txs into the pool, each holding only the given custody cells
+ txs := []*blobpool.BlobTxForPool{
+ makeMultiBlobTx(&config, 0, 2, 0, key1, version, custody), // blob[0, 2)
+ makeMultiBlobTx(&config, 0, 2, 2, key2, version, custody), // blob[2, 4)
+ makeMultiBlobTx(&config, 0, 2, 4, key3, version, custody), // blob[4, 6)
+ }
+ for _, ptx := range txs {
+ if err := ethServ.BlobTxPool().AddPooledTx(ptx); err != nil {
+ t.Fatalf("failed to add blob tx: %v", err)
+ }
+ }
api := newConsensusAPIWithoutHeartbeat(ethServ)
return n, api
}
func TestGetBlobsV2And3(t *testing.T) {
- n, api := newGetBlobEnv(t, 1)
+ n, api := newGetBlobEnv(t, 1, types.CustodyBitmapAll)
defer n.Close()
suites := []struct {
@@ -2076,7 +2095,7 @@ func TestGetBlobsV2And3(t *testing.T) {
// Benchmark GetBlobsV2 internals
// Note that this is not an RPC-level benchmark, so JSON-RPC overhead is not included.
func BenchmarkGetBlobsV2(b *testing.B) {
- n, api := newGetBlobEnv(b, 1)
+ n, api := newGetBlobEnv(b, 1, types.CustodyBitmapAll)
defer n.Close()
// for blobs in [1, 2, 4, 6], print string and run benchmark
@@ -2136,3 +2155,112 @@ func runGetBlobs(t testing.TB, getBlobs getBlobsFn, start, limit int, fillRandom
t.Fatalf("Unexpected result for case %s", name)
}
}
+
+func TestGetBlobsV4(t *testing.T) {
+ // The pool holds only this set of custody cells.
+ custody := types.NewCustodyBitmap([]uint64{0, 1, 2, 3, 4})
+ n, api := newGetBlobEnv(t, 1, custody)
+ defer n.Close()
+
+ masks := []struct {
+ name string
+ mask types.CustodyBitmap
+ }{
+ {"missing", types.NewCustodyBitmap([]uint64{5})},
+ {"overlap", types.NewCustodyBitmap([]uint64{0, 2, 5, 127})},
+ {"aligned", custody},
+ }
+ suites := []struct {
+ start int
+ limit int
+ missingBlob bool
+ }{
+ {start: 0, limit: 1},
+ {start: 0, limit: 2},
+ {start: 1, limit: 3},
+ {start: 0, limit: 6},
+ {start: 1, limit: 5},
+ {start: 0, limit: 6, missingBlob: true},
+ }
+ for _, m := range masks {
+ for i, suite := range suites {
+ runGetBlobsV4(t, api, custody, m.mask, suite.start, suite.limit, suite.missingBlob, fmt.Sprintf("GetBlobsV4 mask=%s suite=%d", m.name, i))
+ }
+ }
+}
+
+func runGetBlobsV4(t testing.TB, api *ConsensusAPI, custody, mask types.CustodyBitmap, start, limit int, missingBlob bool, name string) {
+ // Fill the request for retrieving cells and build the expected response.
+ var (
+ vhashes []common.Hash
+ expect []*engine.BlobCellsAndProofsV1
+ indices = mask.Indices()
+ )
+ for j := start; j < limit; j++ {
+ vhashes = append(vhashes, testBlobVHashes[j])
+
+ cells, err := kzg4844.ComputeCells([]kzg4844.Blob{*testBlobs[j]})
+ if err != nil {
+ t.Fatalf("Failed to compute cells for case %s: %v", name, err)
+ }
+ blobCells := make([]*hexutil.Bytes, len(indices))
+ blobProofs := make([]*hexutil.Bytes, len(indices))
+ for i, idx := range indices {
+ if !custody.IsSet(idx) {
+ continue
+ }
+ cell := hexutil.Bytes(cells[idx][:])
+ blobCells[i] = &cell
+ proof := hexutil.Bytes(testBlobCellProofs[j][idx][:])
+ blobProofs[i] = &proof
+ }
+ expect = append(expect, &engine.BlobCellsAndProofsV1{
+ BlobCells: blobCells,
+ Proofs: blobProofs,
+ })
+ }
+ // put random missing blob
+ if missingBlob {
+ vhashes = append(vhashes, testrand.Hash())
+ expect = append(expect, nil)
+ }
+ result, err := api.GetBlobsV4(vhashes, mask)
+ if err != nil {
+ t.Errorf("Unexpected error for case %s, %v", name, err)
+ }
+ if !reflect.DeepEqual(result, expect) {
+ t.Fatalf("Unexpected result for case %s", name)
+ }
+}
+
+// TestForkchoiceUpdatedV4 tests the custody bitmap argument added in V4.
+func TestForkchoiceUpdatedV4(t *testing.T) {
+ n, api := newGetBlobEnv(t, 1, types.CustodyBitmapAll)
+ defer n.Close()
+
+ head := api.eth.BlockChain().CurrentHeader().Hash()
+ fcState := engine.ForkchoiceStateV1{
+ HeadBlockHash: head,
+ SafeBlockHash: head,
+ FinalizedBlockHash: head,
+ }
+
+ // Non nil custody bitmap case.
+ custody := types.NewCustodyBitmap([]uint64{0, 1, 2, 127})
+ resp, err := api.ForkchoiceUpdatedV4(context.Background(), fcState, nil, &custody)
+ if err != nil {
+ t.Fatalf("Unexpected error with custody bitmap: %v", err)
+ }
+ if resp.PayloadStatus.Status != engine.VALID {
+ t.Fatalf("Unexpected status with custody bitmap: got %s, want %s", resp.PayloadStatus.Status, engine.VALID)
+ }
+
+ // Nil custody bitmap case.
+ resp, err = api.ForkchoiceUpdatedV4(context.Background(), fcState, nil, nil)
+ if err != nil {
+ t.Fatalf("Unexpected error with nil custody bitmap: %v", err)
+ }
+ if resp.PayloadStatus.Status != engine.VALID {
+ t.Fatalf("Unexpected status with nil custody bitmap: got %s, want %s", resp.PayloadStatus.Status, engine.VALID)
+ }
+}
diff --git a/eth/fetcher/blob_fetcher.go b/eth/fetcher/blob_fetcher.go
new file mode 100644
index 0000000000..8a97e1c567
--- /dev/null
+++ b/eth/fetcher/blob_fetcher.go
@@ -0,0 +1,959 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package fetcher
+
+import (
+ "fmt"
+ "iter"
+ "math/rand"
+ "slices"
+ "sort"
+ "time"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/common/mclock"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
+)
+
+type random interface {
+ Intn(n int) int
+}
+
+// BlobFetcher fetches blobs of new type-3 transactions with probability p,
+// and for the remaining (1-p) transactions, it performs availability checks.
+// For availability checks, it fetches cells from each blob in the transaction
+// according to the custody cell indices provided by the consensus client
+// connected to this execution client.
+var blobFetchTimeout = 5 * time.Second
+var blobAvailabilityTimeout = 2 * time.Second
+
+// DefaultFetchProbability is the default probability of fetching the full blob
+// payload for the sparse blobpool.
+const DefaultFetchProbability = 15
+
+const (
+ availabilityThreshold = 2
+ maxPayloadRetrievals = 128
+ maxPayloadAnnounces = 4096
+
+ // maxCellRequests caps the burst of cell requests we can issue at once
+ // to a single peer. Worst case 256 * 6 = 1536 cells (~3 MB)
+ maxCellRequests = 256
+ // refillInterval is the gap between token refill. Combined with
+ // maxCellRequests and 2-minute buffer timeout, a peer's worst case
+ // buffered cells is about 16 MB
+ refillInterval = time.Second / 9
+)
+
+type blobTxAnnounce struct {
+ origin string // Identifier of the peer that sent the announcement
+ txs []common.Hash // Hashes of transactions announced
+ cells types.CustodyBitmap // Custody information of transactions being announced
+}
+
+type cellRequest struct {
+ txs []common.Hash // Transactions that have been requested for their cells
+ cells types.CustodyBitmap // Requested cell indices
+ time mclock.AbsTime // Timestamp when the request was made
+}
+
+type payloadDelivery struct {
+ origin string // Peer from which the payloads were delivered
+ txs []common.Hash // Hashes of transactions that were delivered
+ cells [][]kzg4844.Cell
+ cellBitmap types.CustodyBitmap
+}
+
+type cellWithSeq struct {
+ seq uint64
+ cells types.CustodyBitmap
+}
+
+// PeerCellDelivery holds cells delivered by a single peer.
+type PeerCellDelivery struct {
+ Cells []kzg4844.Cell // blob-major order as received
+ Indices []uint64 // custody indices provided by this peer
+}
+
+type fetchStatus struct {
+ fetching types.CustodyBitmap // To avoid fetching cells which had already been fetched / currently being fetched
+ fetched []uint64 // Custody indices that have been fetched (per-blob, same for all blobs)
+ deliveries map[string]*PeerCellDelivery // Per-peer cell deliveries
+}
+
+type BlobFetcherFunctions struct {
+ HasPayload func(common.Hash) bool
+ AddCells func(common.Hash, map[string]*PeerCellDelivery, types.CustodyBitmap)
+ FetchPayloads func(string, []common.Hash, types.CustodyBitmap) error
+ DropPeer func(string)
+}
+
+// BlobFetcher is responsible for managing type 3 transactions based on peer announcements.
+//
+// BlobFetcher manages three buffers:
+// - Transactions not to be fetched are moved to "waitlist"
+// if a payload(blob) seems to be possessed by D(threshold) other peers, request custody cells for that.
+// Accept it when the cells are received. Otherwise, it is dropped.
+// - Transactions queued to be fetched are moved to "announces"
+// if a payload is received, it is added to the blob pool. Otherwise, the transaction is dropped.
+// - Transactions to be fetched are moved to "fetching"
+// if a payload/cell announcement is received during fetch, the peer is recorded as an alternate source.
+type BlobFetcher struct {
+ notify chan *blobTxAnnounce
+ cleanup chan *payloadDelivery
+ drop chan *txDrop
+ custodyCh chan types.CustodyBitmap
+ quit chan struct{}
+ custody types.CustodyBitmap
+
+ txSeq uint64 // To make transactions fetched in arrival order
+
+ full map[common.Hash]struct{}
+ partial map[common.Hash]struct{}
+
+ // Buffer 1: Set of blob txs whose blob data is waiting for availability confirmation (partial fetch)
+ waitlist map[common.Hash]map[string]struct{} // Peer set that announced blob availability
+ waittime map[common.Hash]mclock.AbsTime // Timestamp when added to waitlist
+ waitslots map[string]map[common.Hash]struct{} // Waiting announcements grouped by peer (DoS protection)
+ // waitSlots should also include announcements with partial cells
+
+ // Buffer 2: Transactions queued for fetching (full fetch + partial fetch)
+ // "announces" is shared with stage 3, for DoS protection
+ announces map[string]map[common.Hash]*cellWithSeq // Set of announced transactions, grouped by origin peer
+
+ // Buffer 2
+ // Stage 3: Transactions whose payloads/cells are currently being fetched (full fetch + partial fetch)
+ fetches map[common.Hash]*fetchStatus // Hash -> Bitmap, in-flight transaction cells
+ requests map[string][]*cellRequest // In-flight transaction retrievals
+ alternates map[common.Hash]map[string]types.CustodyBitmap // In-flight transaction alternate origins (in case the peer is dropped)
+
+ fn BlobFetcherFunctions // callbacks
+ fetchProbability uint64
+
+ // peerTokens tracks each peer's remaining cell request token.
+ peerTokens map[string]*token
+
+ step chan struct{} // Notification channel when the fetcher loop iterates
+ clock mclock.Clock // Monotonic clock or simulated clock for tests
+ realTime func() time.Time // Real system time or simulated time for tests
+ rand random // Randomizer
+}
+
+// token is a per peer token bucket for outgoing cell requests.
+type token struct {
+ amount int64
+ last mclock.AbsTime
+}
+
+func NewBlobFetcher(fn BlobFetcherFunctions, custody types.CustodyBitmap, rand random, fetchProbability uint64) *BlobFetcher {
+ if fetchProbability < DefaultFetchProbability {
+ fetchProbability = DefaultFetchProbability
+ }
+ return &BlobFetcher{
+ notify: make(chan *blobTxAnnounce),
+ cleanup: make(chan *payloadDelivery),
+ drop: make(chan *txDrop),
+ custodyCh: make(chan types.CustodyBitmap),
+ quit: make(chan struct{}),
+ full: make(map[common.Hash]struct{}),
+ partial: make(map[common.Hash]struct{}),
+ waitlist: make(map[common.Hash]map[string]struct{}),
+ waittime: make(map[common.Hash]mclock.AbsTime),
+ waitslots: make(map[string]map[common.Hash]struct{}),
+ announces: make(map[string]map[common.Hash]*cellWithSeq),
+ fetches: make(map[common.Hash]*fetchStatus),
+ requests: make(map[string][]*cellRequest),
+ alternates: make(map[common.Hash]map[string]types.CustodyBitmap),
+ peerTokens: make(map[string]*token),
+ fn: fn,
+ fetchProbability: fetchProbability,
+ custody: custody,
+ clock: mclock.System{},
+ realTime: time.Now,
+ rand: rand,
+ }
+}
+
+// Notify is called when a Type 3 transaction is observed on the network. (TransactionPacket / NewPooledTransactionHashesPacket)
+func (f *BlobFetcher) Notify(peer string, txs []common.Hash, cells types.CustodyBitmap) error {
+ blobAnnounceInMeter.Mark(int64(len(txs)))
+ anns := make([]common.Hash, 0)
+ for _, tx := range txs {
+ if f.fn.HasPayload(tx) {
+ continue
+ }
+ anns = append(anns, tx)
+ }
+
+ blobAnnounce := &blobTxAnnounce{origin: peer, txs: anns, cells: cells}
+ select {
+ case f.notify <- blobAnnounce:
+ return nil
+ case <-f.quit:
+ return errTerminated
+ }
+}
+
+// Enqueue inserts a batch of received blob payloads into the blob pool.
+// This is triggered by ethHandler upon receiving direct request responses.
+func (f *BlobFetcher) Enqueue(peer string, hashes []common.Hash, cells [][]kzg4844.Cell, cellBitmap types.CustodyBitmap) error {
+ blobReplyInMeter.Mark(int64(len(hashes)))
+
+ if len(hashes) != len(cells) {
+ return fmt.Errorf("cells reply length mismatch: %d hashes, %d cells", len(hashes), len(cells))
+ }
+ // An empty custody mask means "no cells"; a peer delivering cells alongside it
+ // is malformed and should be dropped.
+ if cellBitmap.OneCount() == 0 {
+ for _, c := range cells {
+ if len(c) != 0 {
+ return fmt.Errorf("cells delivered with empty custody mask (%d cells)", len(c))
+ }
+ }
+ }
+
+ select {
+ case f.cleanup <- &payloadDelivery{origin: peer, txs: hashes, cells: cells, cellBitmap: cellBitmap}:
+ case <-f.quit:
+ return errTerminated
+ }
+ return nil
+}
+
+func (f *BlobFetcher) Drop(peer string) error {
+ select {
+ case f.drop <- &txDrop{peer: peer}:
+ return nil
+ case <-f.quit:
+ return errTerminated
+ }
+}
+
+// UpdateCustody hands a new custody bitmap to the fetcher loop. The actual
+// swap happens inside the loop so f.custody is never read and written
+// concurrently.
+func (f *BlobFetcher) UpdateCustody(cells types.CustodyBitmap) {
+ select {
+ case f.custodyCh <- cells:
+ case <-f.quit:
+ }
+}
+
+func (f *BlobFetcher) Start() {
+ go f.loop()
+}
+
+func (f *BlobFetcher) Stop() {
+ close(f.quit)
+}
+
+func (f *BlobFetcher) loop() {
+ var (
+ waitTimer = new(mclock.Timer) // Timer for waitlist (availability)
+ waitTrigger = make(chan struct{}, 1)
+ timeoutTimer = new(mclock.Timer) // Timer for payload fetch request
+ timeoutTrigger = make(chan struct{}, 1)
+ )
+ for {
+ select {
+ case ann := <-f.notify:
+ // Drop part of the announcements if too many have accumulated from that peer
+ // This prevents a peer from dominating the queue with txs without responding to the request
+ used := len(f.waitslots[ann.origin]) + len(f.announces[ann.origin])
+ if used >= maxPayloadAnnounces {
+ blobAnnounceDOSMeter.Mark(int64(len(ann.txs)))
+ break
+ }
+
+ want := used + len(ann.txs)
+ if want >= maxPayloadAnnounces {
+ blobAnnounceDOSMeter.Mark(int64(want - maxPayloadAnnounces))
+ ann.txs = ann.txs[:maxPayloadAnnounces-used]
+ }
+
+ var (
+ idleWait = len(f.waittime) == 0
+ _, oldPeer = f.announces[ann.origin]
+ nextSeq = func() uint64 {
+ seq := f.txSeq
+ f.txSeq++
+ return seq
+ }
+ reschedule = make(map[string]struct{})
+ )
+ for _, hash := range ann.txs {
+ if oldPeer && f.announces[ann.origin][hash] != nil {
+ // Ignore already announced information
+ // We also have to prevent reannouncement by changing cells field.
+ // Considering cell custody transition is notified in advance of its finalization by consensus client,
+ // there is no reason to reannounce cells, and it has to be prevented.
+ continue
+ }
+ // Decide full or partial request
+ if _, ok := f.full[hash]; !ok {
+ if _, ok := f.partial[hash]; !ok {
+ // Not decided yet
+ var randomValue int
+ if f.rand == nil {
+ randomValue = rand.Intn(100)
+ } else {
+ randomValue = f.rand.Intn(100)
+ }
+ // For eager mode, always fetch immediately
+ if uint64(randomValue) < f.fetchProbability || f.custody.OneCount() >= kzg4844.DataPerBlob {
+ f.full[hash] = struct{}{}
+ } else {
+ f.partial[hash] = struct{}{}
+ // Register for availability check
+ f.waitlist[hash] = make(map[string]struct{})
+ f.waittime[hash] = f.clock.Now()
+ }
+ }
+ }
+ if _, ok := f.full[hash]; ok {
+ // 1) Decided to send full request of the tx
+ // if the cells are not set to all bitmap, ignore.
+ if ann.cells != types.CustodyBitmapAll {
+ continue
+ }
+ if f.announces[ann.origin] == nil {
+ f.announces[ann.origin] = make(map[common.Hash]*cellWithSeq)
+ }
+ f.announces[ann.origin][hash] = &cellWithSeq{
+ cells: types.CustodyBitmapData,
+ seq: nextSeq(),
+ }
+ reschedule[ann.origin] = struct{}{}
+ continue
+ }
+ if _, ok := f.partial[hash]; ok {
+ // 2) Decided to send partial request of the tx
+ if f.waitlist[hash] != nil {
+ // it is already waiting for availability check
+ if ann.cells != types.CustodyBitmapAll {
+ // Availability check is only meaningful with full availability announcements
+ continue
+ }
+ // Transaction is at the stage of availability check
+ // Add the peer to the peer list with full availability (waitlist)
+ f.waitlist[hash][ann.origin] = struct{}{}
+ if waitslots := f.waitslots[ann.origin]; waitslots != nil {
+ waitslots[hash] = struct{}{}
+ } else {
+ f.waitslots[ann.origin] = map[common.Hash]struct{}{
+ hash: {},
+ }
+ }
+ if len(f.waitlist[hash]) >= availabilityThreshold {
+ // Passed availability check, move to fetching stage
+ blobFetcherWaitTime.Update(int64(time.Duration(f.clock.Now() - f.waittime[hash])))
+ for peer := range f.waitlist[hash] {
+ if f.announces[peer] == nil {
+ f.announces[peer] = make(map[common.Hash]*cellWithSeq)
+ }
+ f.announces[peer][hash] = &cellWithSeq{
+ cells: f.custody,
+ seq: nextSeq(),
+ }
+ delete(f.waitslots[peer], hash)
+ if len(f.waitslots[peer]) == 0 {
+ delete(f.waitslots, peer)
+ }
+ reschedule[peer] = struct{}{}
+ }
+ delete(f.waitlist, hash)
+ delete(f.waittime, hash)
+ }
+ continue
+ }
+ if ann.cells.Intersection(f.custody).OneCount() == 0 {
+ // If there's no custody overlapping in ann, it can be ignored
+ continue
+ }
+ // Add this peer as a possible fetch source
+ if f.announces[ann.origin] == nil {
+ f.announces[ann.origin] = make(map[common.Hash]*cellWithSeq)
+ }
+ f.announces[ann.origin][hash] = &cellWithSeq{
+ cells: ann.cells.Intersection(f.custody),
+ seq: nextSeq(),
+ }
+ reschedule[ann.origin] = struct{}{}
+ }
+ }
+
+ // If a new item was added to the waitlist, schedule its timeout
+ if idleWait && len(f.waittime) > 0 {
+ f.rescheduleWait(waitTimer, waitTrigger)
+ }
+
+ // If this is a new peer and that peer sent transaction with payload flag,
+ // schedule transaction fetches from it
+ if !oldPeer && len(f.announces[ann.origin]) > 0 {
+ f.scheduleFetches(timeoutTimer, timeoutTrigger, reschedule)
+ }
+
+ case <-waitTrigger:
+ // At least one transaction's waiting time ran out. Instead of dropping,
+ // convert timed-out partial fetches to full fetches so we don't lose
+ // the transaction. All peers in the waitlist announced full custody
+ // (that was the entry condition), so they can serve as full fetch sources.
+ reschedule := make(map[string]struct{})
+ for hash, instance := range f.waittime {
+ if time.Duration(f.clock.Now()-instance)+txGatherSlack > blobAvailabilityTimeout {
+ // partial -> full conversion
+ delete(f.partial, hash)
+ f.full[hash] = struct{}{}
+ blobAnnounceTimeoutMeter.Mark(1)
+
+ for peer := range f.waitlist[hash] {
+ if f.announces[peer] == nil {
+ f.announces[peer] = make(map[common.Hash]*cellWithSeq)
+ }
+ f.announces[peer][hash] = &cellWithSeq{
+ cells: types.CustodyBitmapData,
+ seq: f.txSeq,
+ }
+ f.txSeq++
+ delete(f.waitslots[peer], hash)
+ if len(f.waitslots[peer]) == 0 {
+ delete(f.waitslots, peer)
+ }
+ reschedule[peer] = struct{}{}
+ }
+ delete(f.waittime, hash)
+ delete(f.waitlist, hash)
+ }
+ }
+ if len(reschedule) > 0 {
+ f.scheduleFetches(timeoutTimer, timeoutTrigger, reschedule)
+ }
+ // If transactions are still waiting for availability, reschedule the wait timer
+ if len(f.waittime) > 0 {
+ f.rescheduleWait(waitTimer, waitTrigger)
+ }
+
+ case <-timeoutTrigger:
+ // Clean up any expired retrievals and avoid re-requesting them from the
+ // same peer (either overloaded or malicious, useless in both cases).
+ // Update blobpool according to availability result.
+ for peer, requests := range f.requests {
+ newRequests := make([]*cellRequest, 0)
+ for _, req := range requests {
+ if time.Duration(f.clock.Now()-req.time)+txGatherSlack > blobFetchTimeout {
+ blobRequestTimeoutMeter.Mark(int64(len(req.txs)))
+ for _, hash := range req.txs {
+ // Do not request the same tx from this peer
+ delete(f.announces[peer], hash)
+ delete(f.alternates[hash], peer)
+ if f.fetches[hash] == nil {
+ // already completed or discarded via another peer
+ continue
+ }
+ // Allow other candidates to be requested these cells
+ f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(req.cells)
+
+ // Drop the tx entirely if the remaining sources can no longer complete it
+ if !f.recoverable(hash) {
+ f.discard(hash)
+ }
+ }
+ if len(f.announces[peer]) == 0 {
+ delete(f.announces, peer)
+ }
+ } else {
+ newRequests = append(newRequests, req)
+ }
+ }
+ f.requests[peer] = newRequests
+ if len(f.requests[peer]) == 0 {
+ delete(f.requests, peer)
+ }
+ }
+
+ // Schedule a new transaction retrieval
+ f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
+
+ // Trigger timeout for new schedule
+ f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
+
+ case delivery := <-f.cleanup:
+ // Remove from announce
+ var requestId int
+ var request *cellRequest
+ for _, hash := range delivery.txs {
+ // Find the request
+ for i, req := range f.requests[delivery.origin] {
+ if slices.Contains(req.txs, hash) && req.cells == delivery.cellBitmap {
+ request = req
+ requestId = i
+ break
+ }
+ }
+ if request != nil {
+ break
+ }
+ }
+ if request == nil {
+ // peer sent cells not requested. ignore
+ break
+ }
+
+ for i, hash := range delivery.txs {
+ if !slices.Contains(request.txs, hash) || f.fetches[hash] == nil {
+ // Unexpected or canceled hash, ignore
+ continue
+ }
+ indices := delivery.cellBitmap.Indices()
+ if len(indices) > 0 {
+ status := f.fetches[hash]
+ status.deliveries[delivery.origin] = &PeerCellDelivery{
+ Cells: delivery.cells[i],
+ Indices: indices,
+ }
+ status.fetched = append(status.fetched, indices...)
+ }
+
+ // Update announces of this peer
+ delete(f.announces[delivery.origin], hash)
+ if len(f.announces[delivery.origin]) == 0 {
+ delete(f.announces, delivery.origin)
+ }
+ delete(f.alternates[hash], delivery.origin)
+ if len(f.alternates[hash]) == 0 {
+ delete(f.alternates, hash)
+ }
+
+ // Check whether the all required cells are fetched
+ completed := false
+ if _, ok := f.full[hash]; ok && len(f.fetches[hash].fetched) >= kzg4844.DataPerBlob {
+ completed = true
+ } else if _, ok := f.partial[hash]; ok {
+ fetched := make([]uint64, len(f.fetches[hash].fetched))
+ copy(fetched, f.fetches[hash].fetched)
+ slices.Sort(fetched)
+
+ custodyIndices := f.custody.Indices()
+
+ completed = slices.Equal(fetched, custodyIndices)
+ }
+
+ if completed {
+ blobFetcherFetchTime.Update(int64(time.Duration(f.clock.Now() - request.time)))
+ status := f.fetches[hash]
+ collectedCustody := types.NewCustodyBitmap(status.fetched)
+ f.fn.AddCells(hash, status.deliveries, collectedCustody)
+
+ f.discard(hash)
+ }
+ }
+ blobRequestDoneMeter.Mark(int64(len(delivery.txs)))
+
+ // Remove the request
+ f.requests[delivery.origin][requestId] = f.requests[delivery.origin][len(f.requests[delivery.origin])-1]
+ f.requests[delivery.origin] = f.requests[delivery.origin][:len(f.requests[delivery.origin])-1]
+ if len(f.requests[delivery.origin]) == 0 {
+ delete(f.requests, delivery.origin)
+ }
+
+ // Reschedule missing transactions in the request
+ // Anything not delivered should be re-scheduled (with or without
+ // this peer, depending on the response cutoff)
+ delivered := make(map[common.Hash]struct{})
+ for _, hash := range delivery.txs {
+ delivered[hash] = struct{}{}
+ }
+ cutoff := len(request.txs)
+ for i, hash := range request.txs {
+ if _, ok := delivered[hash]; ok {
+ cutoff = i
+ continue
+ }
+ }
+ // Reschedule missing hashes from alternates, not-fulfilled from alt+self
+ for i, hash := range request.txs {
+ if _, ok := delivered[hash]; !ok {
+ // Not delivered
+ if i < cutoff {
+ // Remove origin from candidate sources for partial responses
+ delete(f.alternates[hash], delivery.origin)
+ delete(f.announces[delivery.origin], hash)
+ if len(f.announces[delivery.origin]) == 0 {
+ delete(f.announces, delivery.origin)
+ }
+ }
+ // Mark cells deliverable by other peers
+ if f.fetches[hash] != nil {
+ f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(delivery.cellBitmap)
+ }
+ }
+ }
+ // Something was delivered, try to reschedule requests
+ f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too
+
+ case drop := <-f.drop:
+ // A peer was dropped, remove all traces of it
+ delete(f.peerTokens, drop.peer)
+ if _, ok := f.waitslots[drop.peer]; ok {
+ for hash := range f.waitslots[drop.peer] {
+ delete(f.waitlist[hash], drop.peer)
+ if len(f.waitlist[hash]) == 0 {
+ delete(f.waitlist, hash)
+ delete(f.waittime, hash)
+ }
+ }
+ delete(f.waitslots, drop.peer)
+ if len(f.waitlist) > 0 {
+ f.rescheduleWait(waitTimer, waitTrigger)
+ }
+ }
+ // Clean up general announcement tracking
+ if _, ok := f.announces[drop.peer]; ok {
+ for hash := range f.announces[drop.peer] {
+ delete(f.alternates[hash], drop.peer)
+ if len(f.alternates[hash]) == 0 {
+ delete(f.alternates, hash)
+ }
+ }
+ delete(f.announces, drop.peer)
+ }
+ delete(f.announces, drop.peer)
+
+ // Clean up any active requests
+ if request, ok := f.requests[drop.peer]; ok && len(request) != 0 {
+ for _, req := range request {
+ for _, hash := range req.txs {
+ if f.fetches[hash] == nil {
+ // already canceled
+ continue
+ }
+ // Undelivered hash, reschedule if there's an alternative origin available
+ f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(req.cells)
+ delete(f.alternates[hash], drop.peer)
+ if !f.recoverable(hash) {
+ f.discard(hash)
+ }
+ }
+ }
+ delete(f.requests, drop.peer)
+ // If a request was cancelled, check if anything needs to be rescheduled
+ f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
+ f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
+ }
+
+ case cells := <-f.custodyCh:
+ f.custody = cells
+
+ case <-f.quit:
+ return
+ }
+ // Update metrics gauges
+ blobFetcherWaitingPeers.Update(int64(len(f.waitslots)))
+ blobFetcherWaitingHashes.Update(int64(len(f.waitlist)))
+ blobFetcherQueueingPeers.Update(int64(len(f.announces) - len(f.requests)))
+ blobFetcherQueueingHashes.Update(int64(len(f.announces)))
+ blobFetcherFetchingPeers.Update(int64(len(f.requests)))
+ blobFetcherFetchingHashes.Update(int64(len(f.fetches)))
+
+ // Loop did something, ping the step notifier if needed (tests)
+ if f.step != nil {
+ f.step <- struct{}{}
+ }
+ }
+}
+
+func (f *BlobFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) {
+ if *timer != nil {
+ (*timer).Stop()
+ }
+ now := f.clock.Now()
+
+ earliest := now
+ for _, instance := range f.waittime {
+ if earliest > instance {
+ earliest = instance
+ if blobAvailabilityTimeout-time.Duration(now-earliest) < txGatherSlack {
+ break
+ }
+ }
+ }
+ *timer = f.clock.AfterFunc(blobAvailabilityTimeout-time.Duration(now-earliest), func() {
+ trigger <- struct{}{}
+ })
+}
+
+// Exactly same as the one in TxFetcher
+func (f *BlobFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) {
+ if *timer != nil {
+ (*timer).Stop()
+ }
+ now := f.clock.Now()
+
+ earliest := now
+ for _, requests := range f.requests {
+ for _, req := range requests {
+ // If this request already timed out, skip it altogether
+ if req.txs == nil {
+ continue
+ }
+ if earliest > req.time {
+ earliest = req.time
+ if blobFetchTimeout-time.Duration(now-earliest) < txGatherSlack {
+ break
+ }
+ }
+ }
+ }
+ *timer = f.clock.AfterFunc(blobFetchTimeout-time.Duration(now-earliest), func() {
+ trigger <- struct{}{}
+ })
+}
+
+// consumeToken consumes n tokens from peer's cell-request budget.
+// It returns false if the remaining tokens cannot cover n.
+func (f *BlobFetcher) consumeToken(peer string, n int) bool {
+ now := f.clock.Now()
+ b, ok := f.peerTokens[peer]
+ if !ok {
+ b = &token{amount: maxCellRequests, last: now}
+ f.peerTokens[peer] = b
+ } else {
+ // Here, fractional remaining elapsed time is left in b.last
+ // so that it can be carried over to the next call
+ elapsed := time.Duration(now - b.last)
+ if add := int64(elapsed / refillInterval); add > 0 {
+ b.amount += add
+ if b.amount > maxCellRequests {
+ b.amount = maxCellRequests
+ }
+ b.last += mclock.AbsTime(time.Duration(add) * refillInterval)
+ }
+ }
+ if b.amount < int64(n) {
+ return false
+ }
+ b.amount -= int64(n)
+ return true
+}
+
+// recoverable checks whether the missing cells of a tx can be fully
+// received from the alternative peers.
+func (f *BlobFetcher) recoverable(hash common.Hash) bool {
+ // union of already fetched cells and the cells still announced by peers
+ var covered types.CustodyBitmap
+ if status := f.fetches[hash]; status != nil {
+ covered = types.NewCustodyBitmap(status.fetched)
+ }
+ for _, anns := range f.announces {
+ if cs, ok := anns[hash]; ok {
+ covered = covered.Union(cs.cells)
+ }
+ }
+ if _, ok := f.full[hash]; ok {
+ // Full case: peer coverage is enough to recover the blob
+ return covered.OneCount() >= kzg4844.DataPerBlob
+ }
+ // Partial case:
+ // return true if our custody can be covered
+ return f.custody.Difference(covered).OneCount() == 0
+}
+
+// discard removes every trace of a transaction from the fetcher: its fetch
+// state and all pending announcements/alternates across peers.
+func (f *BlobFetcher) discard(hash common.Hash) {
+ for peer, anns := range f.announces {
+ if _, ok := anns[hash]; ok {
+ delete(anns, hash)
+ if len(anns) == 0 {
+ delete(f.announces, peer)
+ }
+ }
+ }
+ delete(f.alternates, hash)
+ delete(f.fetches, hash)
+ delete(f.full, hash)
+ delete(f.partial, hash)
+}
+
+func (f *BlobFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, whitelist map[string]struct{}) {
+ // Gather the set of peers we want to retrieve from (default to all)
+ actives := whitelist
+ if actives == nil {
+ actives = make(map[string]struct{})
+ for peer := range f.announces {
+ actives[peer] = struct{}{}
+ }
+ }
+ if len(actives) == 0 {
+ return
+ }
+
+ wasIdle := len(f.requests) == 0
+
+ // For each active peer, try to schedule some payload fetches.
+ for peer := range f.peers(actives) {
+ if len(f.announces[peer]) == 0 || len(f.requests[peer]) != 0 {
+ continue
+ }
+ var (
+ hashes []common.Hash
+ custodies []types.CustodyBitmap
+ )
+ for hash, cells := range f.announcesByArrival(f.announces[peer]) {
+ if !f.recoverable(hash) {
+ continue
+ }
+ var unfetched types.CustodyBitmap
+ if f.fetches[hash] == nil {
+ // tx is not being fetched
+ unfetched = cells
+ } else {
+ unfetched = cells.Difference(f.fetches[hash].fetching)
+ }
+
+ // Mark fetching for unfetched cells if the peer has enough token.
+ // Otherwise, the next peer who announced the hash and has token will be selected
+ // in the next loop
+ if unfetched.OneCount() > 0 && f.consumeToken(peer, unfetched.OneCount()) {
+ if f.fetches[hash] == nil {
+ f.fetches[hash] = &fetchStatus{
+ fetching: unfetched,
+ fetched: make([]uint64, 0),
+ deliveries: make(map[string]*PeerCellDelivery),
+ }
+ } else {
+ f.fetches[hash].fetching = f.fetches[hash].fetching.Union(unfetched)
+ }
+ // Accumulate the hash and stop if the limit was reached
+ hashes = append(hashes, hash)
+ custodies = append(custodies, unfetched)
+ }
+
+ // Mark alternatives
+ if f.alternates[hash] == nil {
+ f.alternates[hash] = map[string]types.CustodyBitmap{
+ peer: cells,
+ }
+ } else {
+ f.alternates[hash][peer] = cells
+ }
+
+ // Stop once we've accumulated enough hashes for this peer
+ if len(hashes) >= maxPayloadRetrievals {
+ break
+ }
+ }
+
+ // If any hashes were allocated, request them from the peer
+ if len(hashes) > 0 {
+ // Group hashes by custody bitmap
+ requestByCustody := make(map[types.CustodyBitmap]*cellRequest)
+
+ for i, hash := range hashes {
+ key := custodies[i]
+ if _, ok := requestByCustody[key]; !ok {
+ requestByCustody[key] = &cellRequest{
+ txs: []common.Hash{},
+ cells: custodies[i],
+ time: f.clock.Now(),
+ }
+ }
+ requestByCustody[key].txs = append(requestByCustody[key].txs, hash)
+ }
+ // construct request
+ var request []*cellRequest
+ for _, cr := range requestByCustody {
+ request = append(request, cr)
+ }
+ f.requests[peer] = request
+ // Iterate over a copy: the loop may swap-truncate f.requests[peer]
+ // concurrently when a delivery for this peer arrives.
+ reqs := slices.Clone(request)
+ go func() {
+ for _, req := range reqs {
+ blobRequestOutMeter.Mark(int64(len(req.txs)))
+ if err := f.fn.FetchPayloads(peer, req.txs, req.cells); err != nil {
+ blobRequestFailMeter.Mark(int64(len(req.txs)))
+ f.Drop(peer)
+ break
+ }
+ }
+ }()
+ }
+ }
+
+ // If a new request was fired, schedule a timeout timer
+ if wasIdle && len(f.requests) > 0 {
+ f.rescheduleTimeout(timer, timeout)
+ }
+}
+
+// announcesByArrival returns an iterator over the given announcements
+// in arrival order. We enforce an arrival ordering to minimize
+// the chances of transaction nonce-gaps, which result in
+// transactions being rejected by the txpool.
+
+func (f *BlobFetcher) announcesByArrival(announces map[common.Hash]*cellWithSeq) iter.Seq2[common.Hash, types.CustodyBitmap] {
+ return func(yield func(hash common.Hash, cells types.CustodyBitmap) bool) {
+ type announcement struct {
+ hash common.Hash
+ cells types.CustodyBitmap
+ seq uint64
+ }
+ // Process announcements by their arrival order
+ list := make([]announcement, 0, len(announces))
+ for hash, entry := range announces {
+ list = append(list, announcement{hash: hash, cells: entry.cells, seq: entry.seq})
+ }
+ sort.Slice(list, func(i, j int) bool {
+ return list[i].seq < list[j].seq
+ })
+ for i := range list {
+ if !yield(list[i].hash, list[i].cells) {
+ return
+ }
+ }
+ }
+}
+
+// peers returns an iterator over a map of peers in production, but during
+// testing it does a deterministic sorted random to allow reproducing issues.
+func (f *BlobFetcher) peers(peers map[string]struct{}) iter.Seq[string] {
+ return func(yield func(peer string) bool) {
+ // If we're running production(step == nil), use whatever Go's map gives us
+ if f.step == nil {
+ for peer := range peers {
+ if !yield(peer) {
+ return
+ }
+ }
+ return
+ }
+ // We're running the test suite, make iteration deterministic (sorted by peer id)
+ list := make([]string, 0, len(peers))
+ for peer := range peers {
+ list = append(list, peer)
+ }
+ sort.Strings(list)
+ for _, peer := range list {
+ if !yield(peer) {
+ return
+ }
+ }
+ }
+}
diff --git a/eth/fetcher/blob_fetcher_test.go b/eth/fetcher/blob_fetcher_test.go
new file mode 100644
index 0000000000..bdcc4ade81
--- /dev/null
+++ b/eth/fetcher/blob_fetcher_test.go
@@ -0,0 +1,1152 @@
+// Copyright 2026 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package fetcher
+
+import (
+ "slices"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/common/mclock"
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
+)
+
+// makeTestBlobSidecar is a helper method to create random blob sidecar
+// with certain number of blobs.
+func makeTestCellSidecar(blobCount int) *types.BlobTxCellSidecar {
+ var (
+ blobs []kzg4844.Blob
+ commitments []kzg4844.Commitment
+ proofs []kzg4844.Proof
+ )
+
+ for i := 0; i < blobCount; i++ {
+ blob := &kzg4844.Blob{}
+ blob[0] = byte(i)
+ blobs = append(blobs, *blob)
+
+ commit, _ := kzg4844.BlobToCommitment(blob)
+ commitments = append(commitments, commit)
+
+ cellProofs, _ := kzg4844.ComputeCellProofs(blob)
+ proofs = append(proofs, cellProofs...)
+ }
+
+ cells, _ := kzg4844.ComputeCells(blobs)
+ return &types.BlobTxCellSidecar{
+ Version: types.BlobSidecarVersion1,
+ Cells: cells,
+ Commitments: commitments,
+ Proofs: proofs,
+ Custody: types.CustodyBitmapAll,
+ }
+}
+
+func selectCells(cells []kzg4844.Cell, custody types.CustodyBitmap) []kzg4844.Cell {
+ custodyIndices := custody.Indices()
+ result := make([]kzg4844.Cell, 0)
+
+ for _, idx := range custodyIndices {
+ result = append(result, cells[idx])
+ }
+
+ return result
+}
+
+var (
+ testBlobTxHashes = []common.Hash{
+ {0x01}, {0x02}, {0x03}, {0x04}, {0x05}, {0x06}, {0x07}, {0x08},
+ }
+
+ testBlobSidecars = []*types.BlobTxCellSidecar{
+ makeTestCellSidecar(1),
+ makeTestCellSidecar(2),
+ makeTestCellSidecar(3),
+ makeTestCellSidecar(4),
+ }
+
+ custody = types.NewCustodyBitmap([]uint64{0, 1, 2, 3, 4, 5, 6, 7})
+
+ fullCustody = types.CustodyBitmapAll
+ halfCustody = types.CustodyBitmapData
+ frontCustody = types.NewCustodyBitmap([]uint64{0, 1, 2, 3, 8, 9, 10, 11})
+ backCustody = types.NewCustodyBitmap([]uint64{4, 5, 6, 7, 8, 9, 10, 11})
+ differentCustody = types.NewCustodyBitmap([]uint64{8, 9, 10, 11, 12, 13, 14, 15})
+)
+
+type doBlobNotify struct {
+ peer string
+ hashes []common.Hash
+ custody types.CustodyBitmap
+}
+
+type doBlobEnqueue struct {
+ peer string
+ hashes []common.Hash
+ cells [][]kzg4844.Cell
+ custody types.CustodyBitmap
+}
+
+type blobDoFunc func(*BlobFetcher)
+
+type isWaitingAvailability map[common.Hash]map[string]struct{}
+
+type isDecidedFull map[common.Hash]struct{}
+type isDecidedPartial map[common.Hash]struct{}
+
+type blobAnnounce struct {
+ hash common.Hash
+ custody types.CustodyBitmap
+}
+
+type isBlobScheduled struct {
+ announces map[string][]blobAnnounce // announces에 있는 것들 (peer -> hash+custody)
+ fetching map[string][]blobAnnounce // requests에 있는 것들 (peer -> hash+custody)
+}
+
+type isCompleted []common.Hash
+type isDropped []string
+
+type isFetching struct {
+ hashes map[common.Hash]fetchInfo
+}
+
+type fetchInfo struct {
+ fetching *types.CustodyBitmap
+ fetched []uint64
+}
+
+type blobFetcherTest struct {
+ init func() *BlobFetcher
+ steps []interface{}
+}
+
+type mockRand struct {
+ value int
+}
+
+func (r *mockRand) Intn(n int) int {
+ return r.value
+}
+
+// TestBlobFetcherFullSchedule tests scheduling full payload decision
+// Blob should be fetched immediately when its availability is announced
+// by idle peer, if the client decided to pull the full payload
+// Additional announcements should be recorded as alternates during the fetch
+func TestBlobFetcherFullFetch(t *testing.T) {
+ testBlobFetcher(t, blobFetcherTest{
+ init: func() *BlobFetcher {
+ return NewBlobFetcher(
+ BlobFetcherFunctions{
+ HasPayload: func(common.Hash) bool { return false },
+ AddCells: func(common.Hash, map[string]*PeerCellDelivery, types.CustodyBitmap) {},
+ FetchPayloads: func(string, []common.Hash, types.CustodyBitmap) error {
+ return nil
+ },
+ DropPeer: func(string) {},
+ },
+ custody,
+ &mockRand{value: 5}, // Force full requests (5 < fetchProbability)
+ 15,
+ )
+ },
+ steps: []interface{}{
+ // A announced full custody blob (should make full decision & start fetching)
+ doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ isDecidedFull{testBlobTxHashes[0]: struct{}{}},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+ isFetching{
+ hashes: map[common.Hash]fetchInfo{
+ testBlobTxHashes[0]: {
+ fetching: &halfCustody,
+ fetched: []uint64{},
+ },
+ },
+ },
+
+ // Same hash announced by another peer(B) -> should be added to alternatives
+ doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ "B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+
+ // Announce partial custody by C -> should be ignored
+ doBlobNotify{peer: "C", hashes: []common.Hash{testBlobTxHashes[1]}, custody: custody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ "B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+
+ // Additional hashes announced by A -> should not be fetched
+ doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[1]}, custody: fullCustody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}, {hash: testBlobTxHashes[1], custody: halfCustody}},
+ "B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+
+ // Announce of multiple transactions
+ doBlobNotify{peer: "D", hashes: []common.Hash{testBlobTxHashes[2], testBlobTxHashes[3]}, custody: fullCustody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}, {hash: testBlobTxHashes[1], custody: halfCustody}},
+ "B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ "D": {{hash: testBlobTxHashes[2], custody: halfCustody}, {hash: testBlobTxHashes[3], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ "D": {{hash: testBlobTxHashes[2], custody: halfCustody}, {hash: testBlobTxHashes[3], custody: halfCustody}},
+ },
+ },
+ },
+ })
+}
+
+// TestBlobFetcherPartialFetching tests partial request decision and availability check flow
+func TestBlobFetcherPartialFetch(t *testing.T) {
+ testBlobFetcher(t, blobFetcherTest{
+ init: func() *BlobFetcher {
+ return NewBlobFetcher(
+ BlobFetcherFunctions{
+ HasPayload: func(common.Hash) bool { return false }, AddCells: func(common.Hash, map[string]*PeerCellDelivery, types.CustodyBitmap) {},
+
+ FetchPayloads: func(string, []common.Hash, types.CustodyBitmap) error {
+ return nil
+ },
+ DropPeer: func(string) {},
+ },
+ custody,
+ &mockRand{value: 60}, // Force partial requests (20 >= 15)
+ 15,
+ )
+ },
+ steps: []interface{}{
+ // First full announce for tx 0, 1 -> should make partial decision and go to waitlist
+ doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0], testBlobTxHashes[1]}, custody: fullCustody},
+ isDecidedPartial{testBlobTxHashes[0]: struct{}{}, testBlobTxHashes[1]: struct{}{}},
+ isWaitingAvailability{testBlobTxHashes[0]: map[string]struct{}{"A": {}}, testBlobTxHashes[1]: map[string]struct{}{"A": {}}},
+ isBlobScheduled{announces: nil, fetching: nil},
+
+ // Partial announce for tx 0 (waitlist) -> should be dropped
+ doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: custody},
+ isWaitingAvailability{testBlobTxHashes[0]: map[string]struct{}{"A": {}}, testBlobTxHashes[1]: map[string]struct{}{"A": {}}},
+ isBlobScheduled{announces: nil, fetching: nil},
+
+ // Second full announce for tx 0 -> should make tx 0 available & fetched
+ doBlobNotify{peer: "C", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ isWaitingAvailability{testBlobTxHashes[1]: map[string]struct{}{"A": {}}},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ "C": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ },
+ isFetching{
+ hashes: map[common.Hash]fetchInfo{
+ testBlobTxHashes[0]: {
+ fetching: &custody,
+ fetched: []uint64{},
+ },
+ },
+ },
+
+ // Partial announce for tx 0, overlapped custody -> overlapping part should be accepted
+ doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: frontCustody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ "B": {{hash: testBlobTxHashes[0], custody: frontCustody.Intersection(custody)}},
+ "C": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ },
+
+ // Partial announce for tx 0, with additional custody -> don't update
+ doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: custody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ "B": {{hash: testBlobTxHashes[0], custody: frontCustody.Intersection(custody)}},
+ "C": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ },
+
+ // Partial announce for tx 0, without any overlapped custody -> should be dropped
+ doBlobNotify{peer: "D", hashes: []common.Hash{testBlobTxHashes[0]}, custody: differentCustody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ "B": {{hash: testBlobTxHashes[0], custody: frontCustody.Intersection(custody)}},
+ "C": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ },
+ },
+ })
+}
+
+// TestBlobFetcherFullDelivery tests cell delivery and fetch completion logic (full fetch)
+func TestBlobFetcherFullDelivery(t *testing.T) {
+ testBlobFetcher(t, blobFetcherTest{
+ init: func() *BlobFetcher {
+ return NewBlobFetcher(
+ BlobFetcherFunctions{
+ HasPayload: func(common.Hash) bool { return false }, AddCells: func(common.Hash, map[string]*PeerCellDelivery, types.CustodyBitmap) {},
+
+ FetchPayloads: func(string, []common.Hash, types.CustodyBitmap) error {
+ return nil
+ },
+ DropPeer: func(string) {},
+ },
+ custody,
+ &mockRand{value: 5}, // Force full requests for simplicity
+ 15,
+ )
+ },
+ steps: []interface{}{
+ // Full announce by two peers (A, B) -> schedule fetch
+ doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ "B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+ isFetching{
+ hashes: map[common.Hash]fetchInfo{
+ testBlobTxHashes[0]: {
+ fetching: &halfCustody,
+ fetched: []uint64{},
+ },
+ },
+ },
+
+ // All alternates should be clean up on delivery
+ doBlobEnqueue{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, cells: [][]kzg4844.Cell{selectCells(testBlobSidecars[0].Cells, halfCustody)}, custody: halfCustody},
+ isBlobScheduled{announces: nil, fetching: nil},
+ isFetching{hashes: nil}, // fetches should be empty after completion
+ isCompleted{testBlobTxHashes[0]},
+ },
+ })
+}
+
+// TestBlobFetcherPartialDelivery tests cell delivery and fetch completion logic (partial fetch)
+func TestBlobFetcherPartialDelivery(t *testing.T) {
+ testBlobFetcher(t, blobFetcherTest{
+ init: func() *BlobFetcher {
+ return NewBlobFetcher(
+ BlobFetcherFunctions{
+ HasPayload: func(common.Hash) bool { return false }, AddCells: func(common.Hash, map[string]*PeerCellDelivery, types.CustodyBitmap) {},
+
+ FetchPayloads: func(string, []common.Hash, types.CustodyBitmap) error {
+ return nil
+ },
+ DropPeer: func(string) {},
+ },
+ custody,
+ &mockRand{value: 60},
+ 15,
+ )
+ },
+ steps: []interface{}{
+ // Full announce by two peers (A, B) -> schedule fetch
+ doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ isWaitingAvailability(nil),
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ "B": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ },
+ isFetching{
+ hashes: map[common.Hash]fetchInfo{
+ testBlobTxHashes[0]: {
+ fetching: &custody,
+ fetched: []uint64{},
+ },
+ },
+ },
+
+ // Partial announce by C, D -> alternates
+ doBlobNotify{peer: "C", hashes: []common.Hash{testBlobTxHashes[0]}, custody: frontCustody},
+ doBlobNotify{peer: "D", hashes: []common.Hash{testBlobTxHashes[0]}, custody: backCustody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ "B": {{hash: testBlobTxHashes[0], custody: custody}},
+ "C": {{hash: testBlobTxHashes[0], custody: frontCustody.Intersection(custody)}},
+ "D": {{hash: testBlobTxHashes[0], custody: backCustody.Intersection(custody)}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ },
+
+ // Drop A, B -> schedule fetch from C, D
+ doDrop("A"),
+ doDrop("B"),
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "C": {{hash: testBlobTxHashes[0], custody: frontCustody.Intersection(custody)}},
+ "D": {{hash: testBlobTxHashes[0], custody: backCustody.Intersection(custody)}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "C": {{hash: testBlobTxHashes[0], custody: frontCustody.Intersection(custody)}},
+ "D": {{hash: testBlobTxHashes[0], custody: backCustody.Intersection(custody)}},
+ },
+ },
+
+ // Delivery from C -> wait for D
+ doBlobEnqueue{
+ peer: "C",
+ hashes: []common.Hash{testBlobTxHashes[0]},
+ cells: [][]kzg4844.Cell{selectCells(testBlobSidecars[0].Cells, frontCustody.Intersection(custody))},
+ custody: frontCustody.Intersection(custody),
+ },
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "D": {{hash: testBlobTxHashes[0], custody: backCustody.Intersection(custody)}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "D": {{hash: testBlobTxHashes[0], custody: backCustody.Intersection(custody)}},
+ },
+ },
+ isFetching{
+ hashes: map[common.Hash]fetchInfo{
+ testBlobTxHashes[0]: {
+ fetching: &custody,
+ fetched: frontCustody.Intersection(custody).Indices(),
+ },
+ },
+ },
+
+ // Announce already delivered cells + fetching cells -> leave fetching cells only
+ doBlobNotify{peer: "E", hashes: []common.Hash{testBlobTxHashes[0]}, custody: custody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "D": {{hash: testBlobTxHashes[0], custody: backCustody.Intersection(custody)}},
+ "E": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "D": {{hash: testBlobTxHashes[0], custody: backCustody.Intersection(custody)}},
+ },
+ },
+
+ // Not delivered -> reschedule to E
+ doWait{time: blobFetchTimeout, step: true},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "E": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "E": {{hash: testBlobTxHashes[0], custody: backCustody.Intersection(custody)}},
+ },
+ },
+ isFetching{
+ hashes: map[common.Hash]fetchInfo{
+ testBlobTxHashes[0]: {
+ fetching: &custody,
+ fetched: frontCustody.Intersection(custody).Indices(),
+ },
+ },
+ },
+ // Delivery from E -> complete
+ doWait{time: blobFetchTimeout / 2, step: true},
+ doBlobEnqueue{
+ peer: "E",
+ hashes: []common.Hash{testBlobTxHashes[0]},
+ cells: [][]kzg4844.Cell{selectCells(testBlobSidecars[0].Cells, backCustody.Intersection(custody))},
+ custody: backCustody.Intersection(custody),
+ },
+ isCompleted{testBlobTxHashes[0]},
+ },
+ })
+}
+
+// TestBlobFetcherAvailabilityTimeout tests availability timeout for partial requests
+func TestBlobFetcherAvailabilityTimeout(t *testing.T) {
+ testBlobFetcher(t, blobFetcherTest{
+ init: func() *BlobFetcher {
+ return NewBlobFetcher(
+ BlobFetcherFunctions{
+ HasPayload: func(common.Hash) bool { return false }, AddCells: func(common.Hash, map[string]*PeerCellDelivery, types.CustodyBitmap) {},
+
+ FetchPayloads: func(string, []common.Hash, types.CustodyBitmap) error {
+ return nil
+ },
+ DropPeer: func(string) {},
+ },
+ custody,
+ &mockRand{value: 60},
+ 15,
+ )
+ },
+ steps: []interface{}{
+ // First full announce for tx 0 -> should make partial decision and go to waitlist
+ doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ isDecidedPartial{testBlobTxHashes[0]: struct{}{}},
+ isWaitingAvailability{testBlobTxHashes[0]: map[string]struct{}{"A": {}}},
+ isBlobScheduled{announces: nil, fetching: nil},
+
+ // Run clock for timeout → partial converts to full, peer A moves to announces
+ doWait{time: blobAvailabilityTimeout, step: true},
+
+ // After timeout, waitlist should be empty but tx promoted to full fetch
+ isWaitingAvailability{},
+ isDecidedFull{testBlobTxHashes[0]: struct{}{}},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+ },
+ })
+}
+
+// TestBlobFetcherPeerDrop tests peer drop scenarios
+func TestBlobFetcherPeerDrop(t *testing.T) {
+ testBlobFetcher(t, blobFetcherTest{
+ init: func() *BlobFetcher {
+ return NewBlobFetcher(
+ BlobFetcherFunctions{
+ HasPayload: func(common.Hash) bool { return false }, AddCells: func(common.Hash, map[string]*PeerCellDelivery, types.CustodyBitmap) {},
+
+ FetchPayloads: func(string, []common.Hash, types.CustodyBitmap) error {
+ return nil
+ },
+ DropPeer: func(string) {},
+ },
+ custody,
+ &mockRand{value: 5},
+ 15,
+ )
+ },
+ steps: []interface{}{
+ // Full announce by peer A -> should schedule fetch
+ doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ isDecidedFull{testBlobTxHashes[0]: struct{}{}},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+ isFetching{
+ hashes: map[common.Hash]fetchInfo{
+ testBlobTxHashes[0]: {
+ fetching: &halfCustody,
+ fetched: []uint64{},
+ },
+ },
+ },
+
+ // Another peer B announces same hash -> should be added to alternates
+ doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ "B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+
+ // Drop peer A -> should reschedule fetch to peer B
+ doDrop("A"),
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+ isFetching{
+ hashes: map[common.Hash]fetchInfo{
+ testBlobTxHashes[0]: {
+ fetching: &halfCustody,
+ fetched: []uint64{},
+ },
+ },
+ },
+
+ // Drop peer B -> should drop the transaction, remove all traces
+ doDrop("B"),
+ isBlobScheduled{announces: nil, fetching: nil},
+ isFetching{hashes: nil},
+ },
+ })
+}
+
+// TestBlobFetcherPartialUnrecoverable checks that a partial fetch is discarded
+// once the remaining announcing peers can no longer cover the required custody
+func TestBlobFetcherPartialUnrecoverable(t *testing.T) {
+ testBlobFetcher(t, blobFetcherTest{
+ init: func() *BlobFetcher {
+ return NewBlobFetcher(
+ BlobFetcherFunctions{
+ HasPayload: func(common.Hash) bool { return false },
+ AddCells: func(common.Hash, map[string]*PeerCellDelivery, types.CustodyBitmap) {},
+ FetchPayloads: func(string, []common.Hash, types.CustodyBitmap) error { return nil },
+ DropPeer: func(string) {},
+ },
+ custody,
+ &mockRand{value: 60}, // Force partial requests
+ 15,
+ )
+ },
+ steps: []interface{}{
+ // Two full announces pass the availability check and start a partial fetch (A requested).
+ doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ doBlobNotify{peer: "C", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ isFetching{hashes: map[common.Hash]fetchInfo{
+ testBlobTxHashes[0]: {fetching: &custody, fetched: []uint64{}},
+ }},
+
+ // Drop A: C (full) takes over the whole in-flight custody.
+ doDrop("A"),
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "C": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "C": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ },
+
+ // A partial peer announces part of the custody. C already has all of it in
+ // flight, so B is only recorded as an alternate (no request of its own).
+ doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: frontCustody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "C": {{hash: testBlobTxHashes[0], custody: custody}},
+ "B": {{hash: testBlobTxHashes[0], custody: frontCustody.Intersection(custody)}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "C": {{hash: testBlobTxHashes[0], custody: custody}},
+ },
+ },
+
+ // Drop C: only B's partial custody remains, which cannot cover the
+ // required custody. The tx is unrecoverable, so every trace of it is
+ // discarded, including B's lingering announce.
+ doDrop("C"),
+ isFetching{hashes: nil},
+ isBlobScheduled{
+ announces: nil,
+ fetching: nil,
+ },
+ },
+ })
+}
+
+// TestBlobFetcherFetchTimeout tests fetch timeout and rescheduling, full request case
+func TestBlobFetcherFetchTimeout(t *testing.T) {
+ testBlobFetcher(t, blobFetcherTest{
+ init: func() *BlobFetcher {
+ return NewBlobFetcher(
+ BlobFetcherFunctions{
+ HasPayload: func(common.Hash) bool { return false }, AddCells: func(common.Hash, map[string]*PeerCellDelivery, types.CustodyBitmap) {},
+
+ FetchPayloads: func(string, []common.Hash, types.CustodyBitmap) error {
+ return nil
+ },
+ DropPeer: func(string) {},
+ },
+ custody,
+ &mockRand{value: 5},
+ 15,
+ )
+ },
+ steps: []interface{}{
+ // Full announce by peer A -> schedule fetch
+ doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ isDecidedFull{testBlobTxHashes[0]: struct{}{}},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+ isFetching{
+ hashes: map[common.Hash]fetchInfo{
+ testBlobTxHashes[0]: {
+ fetching: &halfCustody,
+ fetched: []uint64{},
+ },
+ },
+ },
+
+ // Another peer announces same hash -> should be added to alternates
+ doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ "B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+
+ // Wait for fetch timeout -> should reschedule to peer B
+ doWait{time: blobFetchTimeout, step: true},
+ isBlobScheduled{
+ announces: map[string][]blobAnnounce{
+ "B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ fetching: map[string][]blobAnnounce{
+ "B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
+ },
+ },
+ isFetching{
+ hashes: map[common.Hash]fetchInfo{
+ testBlobTxHashes[0]: {
+ fetching: &halfCustody,
+ fetched: []uint64{},
+ },
+ },
+ },
+
+ // Wait for timeout -> should drop transaction
+ doWait{time: blobFetchTimeout, step: true},
+ isBlobScheduled{announces: nil, fetching: nil},
+ isFetching{hashes: nil},
+ },
+ })
+}
+
+// testBlobFetcher is the generic test runner for blob fetcher tests
+func testBlobFetcher(t *testing.T, tt blobFetcherTest) {
+ clock := new(mclock.Simulated)
+ wait := make(chan struct{})
+
+ // Create a fetcher and boot it up
+ fetcher := tt.init()
+ fetcher.clock = clock
+ fetcher.step = wait
+
+ fetcher.Start()
+ defer fetcher.Stop()
+
+ defer func() {
+ for {
+ select {
+ case <-wait:
+ default:
+ return
+ }
+ }
+ }()
+
+ // Iterate through all the test steps and execute them
+ for i, step := range tt.steps {
+ // Clear the channel if anything is left over
+ for len(wait) > 0 {
+ <-wait
+ }
+ // Process the next step of the test
+ switch step := step.(type) {
+ case doBlobNotify:
+ if err := fetcher.Notify(step.peer, step.hashes, step.custody); err != nil {
+ t.Errorf("step %d: failed to notify fetcher: %v", i, err)
+ return
+ }
+ <-wait
+
+ case doBlobEnqueue:
+ if err := fetcher.Enqueue(step.peer, step.hashes, step.cells, step.custody); err != nil {
+ t.Errorf("step %d: failed to enqueue blobs: %v", i, err)
+ return
+ }
+ <-wait
+
+ case blobDoFunc:
+ step(fetcher)
+
+ case isWaitingAvailability:
+ // Check expected hashes and peers are present
+ for hash, peers := range step {
+ if waitPeers, ok := fetcher.waitlist[hash]; !ok {
+ t.Errorf("step %d: hash %x not in waitlist", i, hash)
+ return
+ } else {
+ // Check expected peers are present
+ for peer := range peers {
+ if _, ok := waitPeers[peer]; !ok {
+ t.Errorf("step %d: peer %s not waiting for hash %x", i, peer, hash)
+ return
+ }
+ }
+ // Check no unexpected peers are present
+ for peer := range waitPeers {
+ if _, ok := peers[peer]; !ok {
+ t.Errorf("step %d: unexpected peer %s waiting for hash %x", i, peer, hash)
+ return
+ }
+ }
+ }
+ }
+ // Check no unexpected hashes in waitlist
+ for hash := range fetcher.waitlist {
+ if _, ok := step[hash]; !ok {
+ t.Errorf("step %d: unexpected hash %x in waitlist", i, hash)
+ return
+ }
+ }
+
+ case isDecidedFull:
+ for hash := range step {
+ if _, ok := fetcher.full[hash]; !ok {
+ t.Errorf("step %d: hash %x not decided for full request", i, hash)
+ return
+ }
+ }
+
+ case isDecidedPartial:
+ for hash := range step {
+ if _, ok := fetcher.partial[hash]; !ok {
+ t.Errorf("step %d: hash %x not decided for partial request", i, hash)
+ return
+ }
+ }
+
+ case isBlobScheduled:
+ // Check tracking (announces) - bidirectional verification
+ for peer, announces := range step.announces {
+ peerAnnounces := fetcher.announces[peer]
+ if peerAnnounces == nil {
+ t.Errorf("step %d: peer %s missing from announces", i, peer)
+ continue
+ }
+ // Check expected announces are present
+ for _, ann := range announces {
+ if cellWithSeq, ok := peerAnnounces[ann.hash]; !ok {
+ t.Errorf("step %d, peer %s: hash %x missing from announces", i, peer, ann.hash)
+ } else if cellWithSeq.cells != ann.custody {
+ t.Errorf("step %d, peer %s, hash %x: custody mismatch in announces", i, peer, ann.hash)
+ }
+ }
+ // Check no unexpected announces are present
+ for hash := range peerAnnounces {
+ found := false
+ for _, ann := range announces {
+ if ann.hash == hash {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Errorf("step %d, peer %s: unexpected hash %x in announces", i, peer, hash)
+ }
+ }
+ }
+ // Check no unexpected peers in announces
+ for peer := range fetcher.announces {
+ if _, ok := step.announces[peer]; !ok {
+ t.Errorf("step %d: unexpected peer %s in announces", i, peer)
+ }
+ }
+
+ // Check fetching (requests)
+ for peer, requests := range step.fetching {
+ peerRequests := fetcher.requests[peer]
+ if peerRequests == nil {
+ t.Errorf("step %d: peer %s missing from requests", i, peer)
+ continue
+ }
+ // Check expected requests are present
+ for _, req := range requests {
+ found := false
+ for _, cellReq := range peerRequests {
+ for _, hash := range cellReq.txs {
+ if hash == req.hash && cellReq.cells == req.custody {
+ found = true
+ break
+ }
+ }
+ if found {
+ break
+ }
+ }
+ if !found {
+ t.Errorf("step %d, peer %s: hash %x with custody not found in requests", i, peer, req.hash)
+ }
+ }
+ // (bidirectional) Check no unexpected requests are present
+ for _, cellReq := range peerRequests {
+ for _, hash := range cellReq.txs {
+ found := false
+ for _, req := range requests {
+ if req.hash == hash && cellReq.cells == req.custody {
+ found = true
+ break
+ }
+ }
+ if !found {
+ t.Errorf("step %d, peer %s: unexpected hash %x in requests", i, peer, hash)
+ }
+ }
+ }
+ }
+ // Check no unexpected peers in requests
+ for peer := range fetcher.requests {
+ if _, ok := step.fetching[peer]; !ok {
+ t.Errorf("step %d: unexpected peer %s in requests", i, peer)
+ }
+ }
+
+ // Check internal consistency: alternates should match announces
+ // For every hash being fetched, alternates should contain all peers who announced it
+ for _, announces := range step.fetching {
+ for _, announce := range announces {
+ hash := announce.hash
+ alternates := fetcher.alternates[hash]
+ if alternates == nil {
+ t.Errorf("step %d: hash %x missing from alternates", i, hash)
+ continue
+ }
+
+ // Check that all peers with this hash in announces are in alternates with matching custody
+ for peer, peerAnnounces := range fetcher.announces {
+ if cellWithSeq := peerAnnounces[hash]; cellWithSeq != nil {
+ if altCustody, ok := alternates[peer]; !ok {
+ t.Errorf("step %d, hash %x: peer %s missing from alternates", i, hash, peer)
+ } else if altCustody != cellWithSeq.cells {
+ t.Errorf("step %d, hash %x, peer %s: custody bitmap mismatch in alternates", i, hash, peer)
+ }
+ }
+ }
+
+ // Check that all peers in alternates actually have this hash announced with matching custody
+ for peer, altCustody := range alternates {
+ if fetcher.announces[peer] == nil || fetcher.announces[peer][hash] == nil {
+ t.Errorf("step %d, hash %x: peer %s extra in alternates", i, hash, peer)
+ } else if cellWithSeq := fetcher.announces[peer][hash]; cellWithSeq.cells != altCustody {
+ t.Errorf("step %d, hash %x, peer %s: custody bitmap mismatch between announces and alternates", i, hash, peer)
+ }
+ }
+ }
+ }
+
+ case isFetching:
+ // Check expected hashes are present in fetches
+ for hash, expected := range step.hashes {
+ if fetchStatus, ok := fetcher.fetches[hash]; !ok {
+ t.Errorf("step %d: hash %x missing from fetches", i, hash)
+ } else {
+ // Check fetching bitmap
+ if expected.fetching != nil {
+ if fetchStatus.fetching != *expected.fetching {
+ t.Errorf("step %d, hash %x: fetching bitmap mismatch", i, hash)
+ }
+ }
+
+ // Check fetched indices
+ if expected.fetched != nil {
+ if len(fetchStatus.fetched) != len(expected.fetched) {
+ t.Errorf("step %d, hash %x: fetched length mismatch, got %d, want %d", i, hash, len(fetchStatus.fetched), len(expected.fetched))
+ } else {
+ // Sort both slices before comparing
+ gotFetched := make([]uint64, len(fetchStatus.fetched))
+ copy(gotFetched, fetchStatus.fetched)
+ slices.Sort(gotFetched)
+
+ expectedFetched := make([]uint64, len(expected.fetched))
+ copy(expectedFetched, expected.fetched)
+ slices.Sort(expectedFetched)
+
+ if !slices.Equal(gotFetched, expectedFetched) {
+ t.Errorf("step %d, hash %x: fetched indices mismatch", i, hash)
+ }
+ }
+ }
+ }
+ }
+ // Check no unexpected hashes in fetches
+ for hash := range fetcher.fetches {
+ if _, ok := step.hashes[hash]; !ok {
+ t.Errorf("step %d: unexpected hash %x in fetches", i, hash)
+ }
+ }
+
+ case isCompleted:
+ for _, hash := range step {
+ if _, ok := fetcher.fetches[hash]; ok {
+ t.Errorf("step %d: hash %x still in fetches (should be completed)", i, hash)
+ return
+ }
+ }
+
+ case isDropped:
+ for _, peer := range step {
+ if _, ok := fetcher.announces[peer]; ok {
+ t.Errorf("step %d: peer %s still has announces (should be dropped)", i, peer)
+ return
+ }
+ }
+
+ case doWait:
+ clock.Run(step.time)
+ if step.step {
+ <-wait
+ }
+
+ case doDrop:
+ if err := fetcher.Drop(string(step)); err != nil {
+ t.Errorf("step %d: %v", i, err)
+ }
+ <-wait
+
+ default:
+ t.Errorf("step %d: unknown step type %T", i, step)
+ return
+ }
+ }
+}
+
+// selectMultiBlobCells extracts cells from a multi-blob sidecar for a given
+// custody mask, returning them in blob-major order.
+func selectMultiBlobCells(sc *types.BlobTxCellSidecar, mask types.CustodyBitmap) []kzg4844.Cell {
+ var result []kzg4844.Cell
+ cellsPerBlob := sc.Custody.OneCount()
+ blobCount := len(sc.Cells) / cellsPerBlob
+ for b := 0; b < blobCount; b++ {
+ for _, idx := range mask.Indices() {
+ result = append(result, sc.Cells[b*cellsPerBlob+int(idx)])
+ }
+ }
+ return result
+}
+
+// TestMultiBlobDeliveryVerification tests that cells delivered in two partial
+// deliveries for a multi-blob tx are correctly assembled and pass KZG cell
+// proof verification via the addPayload callback.
+func TestMultiBlobDeliveryVerification(t *testing.T) {
+ sidecar := testBlobSidecars[2] // 3 blobs
+
+ var verifyErr error
+ testBlobFetcher(t, blobFetcherTest{
+ init: func() *BlobFetcher {
+ return NewBlobFetcher(
+ BlobFetcherFunctions{
+ HasPayload: func(common.Hash) bool { return false },
+ AddCells: func(h common.Hash, deliveries map[string]*PeerCellDelivery, custody types.CustodyBitmap) {
+ // Verify each peer's delivered cells pass KZG cell proof verification
+ for _, d := range deliveries {
+ var cellProofs []kzg4844.Proof
+ for blobIdx := 0; blobIdx < len(sidecar.Commitments); blobIdx++ {
+ for _, idx := range d.Indices {
+ cellProofs = append(cellProofs, sidecar.Proofs[blobIdx*kzg4844.CellProofsPerBlob+int(idx)])
+ }
+ }
+ verifyErr = kzg4844.VerifyCells(d.Cells, sidecar.Commitments, cellProofs, d.Indices)
+ }
+ },
+ FetchPayloads: func(string, []common.Hash, types.CustodyBitmap) error {
+ return nil
+ },
+ DropPeer: func(string) {},
+ },
+ custody,
+ &mockRand{value: 60}, // Force partial requests (60 >= fetchProbability)
+ 15,
+ )
+ },
+ steps: []interface{}{
+ // Two full-custody peers → passes availability, promotes to announces
+ doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+ doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
+
+ // Two partial peers with front/back custody
+ doBlobNotify{peer: "D", hashes: []common.Hash{testBlobTxHashes[0]}, custody: backCustody},
+ doBlobNotify{peer: "C", hashes: []common.Hash{testBlobTxHashes[0]}, custody: frontCustody},
+
+ // Drop A and B so C and D get scheduled for fetch
+ doDrop("A"),
+ doDrop("B"),
+
+ // Deliver back cells from D → completes fetch and triggers addPayload
+ doBlobEnqueue{
+ peer: "D",
+ hashes: []common.Hash{testBlobTxHashes[0]},
+ cells: [][]kzg4844.Cell{selectMultiBlobCells(sidecar, backCustody.Intersection(custody))},
+ custody: backCustody.Intersection(custody),
+ },
+ // Deliver front cells from C
+ doBlobEnqueue{
+ peer: "C",
+ hashes: []common.Hash{testBlobTxHashes[0]},
+ cells: [][]kzg4844.Cell{selectMultiBlobCells(sidecar, frontCustody.Intersection(custody))},
+ custody: frontCustody.Intersection(custody),
+ },
+ isCompleted{testBlobTxHashes[0]},
+ },
+ })
+ if verifyErr != nil {
+ t.Fatalf("KZG cell verification failed after multi-blob delivery: %v", verifyErr)
+ }
+}
diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go
index 3c0d6a8fd8..1126fdc382 100644
--- a/eth/fetcher/metrics.go
+++ b/eth/fetcher/metrics.go
@@ -57,4 +57,26 @@ var (
// to become "unfrozen", either by eventually replying to the request
// or by being dropped, measuring from the moment the request was sent.
txFetcherSlowWait = metrics.NewRegisteredHistogram("eth/fetcher/transaction/slow/wait", nil, metrics.NewExpDecaySample(1028, 0.015))
+
+ blobAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/in", nil)
+ blobAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/dos", nil)
+ // This metric tracks partial→full conversions due to availability timeout
+ blobAnnounceTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/timeout", nil)
+
+ blobRequestOutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/out", nil)
+ blobRequestFailMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/fail", nil)
+ blobRequestDoneMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/done", nil)
+ blobRequestTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/timeout", nil)
+
+ blobReplyInMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/replies/in", nil)
+
+ blobFetcherWaitingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/waiting/peers", nil)
+ blobFetcherWaitingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/waiting/hashes", nil)
+ blobFetcherQueueingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/queueing/peers", nil)
+ blobFetcherQueueingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/queueing/hashes", nil)
+ blobFetcherFetchingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/fetching/peers", nil)
+ blobFetcherFetchingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/fetching/hashes", nil)
+
+ blobFetcherWaitTime = metrics.NewRegisteredHistogram("eth/fetcher/blob/wait/time", nil, metrics.NewExpDecaySample(1028, 0.015))
+ blobFetcherFetchTime = metrics.NewRegisteredHistogram("eth/fetcher/blob/fetch/time", nil, metrics.NewExpDecaySample(1028, 0.015))
)
diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go
index 8b917413ec..63efdfab60 100644
--- a/eth/fetcher/tx_fetcher.go
+++ b/eth/fetcher/tx_fetcher.go
@@ -29,8 +29,11 @@ import (
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/txpool"
+ "github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/log"
+ "github.com/ethereum/go-ethereum/metrics"
)
const (
@@ -186,6 +189,8 @@ type TxFetcher struct {
dropPeer func(string) // Drops a peer in case of announcement violation
onAccepted func(peer string, hashes []common.Hash) // Optional: notified with accepted tx hashes per peer
+ buffer *blobpool.BlobBuffer
+
step chan struct{} // Notification channel when the fetcher loop iterates
clock mclock.Clock // Monotonic clock or simulated clock for tests
realTime func() time.Time // Real system time or simulated time for tests
@@ -195,8 +200,9 @@ type TxFetcher struct {
// NewTxFetcher creates a transaction fetcher to retrieve transaction
// based on hash announcements.
// Chain can be nil to disable on-chain checks.
-func NewTxFetcher(chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), onAccepted func(string, []common.Hash)) *TxFetcher {
- return NewTxFetcherForTests(chain, validateMeta, addTxs, fetchTxs, dropPeer, onAccepted, mclock.System{}, time.Now, nil)
+func NewTxFetcher(chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error,
+ dropPeer func(string), onAccepted func(string, []common.Hash), buffer *blobpool.BlobBuffer) *TxFetcher {
+ return NewTxFetcherForTests(chain, validateMeta, addTxs, fetchTxs, dropPeer, onAccepted, buffer, mclock.System{}, time.Now, nil)
}
// NewTxFetcherForTests is a testing method to mock out the realtime clock with
@@ -204,7 +210,7 @@ func NewTxFetcher(chain *core.BlockChain, validateMeta func(common.Hash, byte) e
// Chain can be nil to disable on-chain checks.
func NewTxFetcherForTests(
chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), onAccepted func(string, []common.Hash),
- clock mclock.Clock, realTime func() time.Time, rand *mrand.Rand) *TxFetcher {
+ buffer *blobpool.BlobBuffer, clock mclock.Clock, realTime func() time.Time, rand *mrand.Rand) *TxFetcher {
return &TxFetcher{
notify: make(chan *txAnnounce),
cleanup: make(chan *txDelivery),
@@ -225,6 +231,7 @@ func NewTxFetcherForTests(
addTxs: addTxs,
fetchTxs: fetchTxs,
dropPeer: dropPeer,
+ buffer: buffer,
onAccepted: onAccepted,
clock: clock,
realTime: realTime,
@@ -233,8 +240,8 @@ func NewTxFetcherForTests(
}
// Notify announces the fetcher of the potential availability of a new batch of
-// transactions in the network.
-func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []common.Hash) error {
+// transactions in the network. It returns array of hashes decided to be fetched.
+func (f *TxFetcher) Notify(peer string, kinds []byte, sizes []uint32, hashes []common.Hash) ([]common.Hash, error) {
// Keep track of all the announced transactions
txAnnounceInMeter.Mark(int64(len(hashes)))
@@ -247,13 +254,18 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
unknownHashes = make([]common.Hash, 0, len(hashes))
unknownMetas = make([]txMetadata, 0, len(hashes))
+ blobFetchHashes = make([]common.Hash, 0, len(hashes))
+
duplicate int64
onchain int64
underpriced int64
)
for i, hash := range hashes {
- err := f.validateMeta(hash, types[i])
+ err := f.validateMeta(hash, kinds[i])
if errors.Is(err, txpool.ErrAlreadyKnown) {
+ if kinds[i] == types.BlobTxType {
+ blobFetchHashes = append(blobFetchHashes, hash)
+ }
duplicate++
continue
}
@@ -273,11 +285,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
}
unknownHashes = append(unknownHashes, hash)
+ if kinds[i] == types.BlobTxType {
+ blobFetchHashes = append(blobFetchHashes, hash)
+ }
// Transaction metadata has been available since eth68, and all
// legacy eth protocols (prior to eth68) have been deprecated.
// Therefore, metadata is always expected in the announcement.
- unknownMetas = append(unknownMetas, txMetadata{kind: types[i], size: sizes[i]})
+ unknownMetas = append(unknownMetas, txMetadata{kind: kinds[i], size: sizes[i]})
}
txAnnounceKnownMeter.Mark(duplicate)
txAnnounceUnderpricedMeter.Mark(underpriced)
@@ -285,14 +300,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
// If anything's left to announce, push it into the internal loop
if len(unknownHashes) == 0 {
- return nil
+ return blobFetchHashes, nil
}
announce := &txAnnounce{origin: peer, hashes: unknownHashes, metas: unknownMetas}
select {
case f.notify <- announce:
- return nil
+ return blobFetchHashes, nil
case <-f.quit:
- return errTerminated
+ return nil, errTerminated
}
}
@@ -306,26 +321,36 @@ func (f *TxFetcher) isKnownUnderpriced(hash common.Hash) bool {
return ok
}
+type deliveryMetrics struct {
+ inMeter *metrics.Meter
+ knownMeter *metrics.Meter
+ underpricedMeter *metrics.Meter
+ otherRejectMeter *metrics.Meter
+}
+
// Enqueue imports a batch of received transaction into the transaction pool
// and the fetcher. This method may be called by both transaction broadcasts and
// direct request replies. The differentiation is important so the fetcher can
// re-schedule missing transactions as soon as possible.
-func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error {
- var (
- inMeter = txReplyInMeter
- knownMeter = txReplyKnownMeter
- underpricedMeter = txReplyUnderpricedMeter
- otherRejectMeter = txReplyOtherRejectMeter
- violation error
- )
+func (f *TxFetcher) Enqueue(peer string, version uint, txs []*types.Transaction, direct bool) error {
+ var violation error
+
+ metrics := deliveryMetrics{
+ inMeter: txReplyInMeter,
+ knownMeter: txReplyKnownMeter,
+ underpricedMeter: txReplyUnderpricedMeter,
+ otherRejectMeter: txReplyOtherRejectMeter,
+ }
if !direct {
- inMeter = txBroadcastInMeter
- knownMeter = txBroadcastKnownMeter
- underpricedMeter = txBroadcastUnderpricedMeter
- otherRejectMeter = txBroadcastOtherRejectMeter
+ metrics = deliveryMetrics{
+ inMeter: txBroadcastInMeter,
+ knownMeter: txBroadcastKnownMeter,
+ underpricedMeter: txBroadcastUnderpricedMeter,
+ otherRejectMeter: txBroadcastOtherRejectMeter,
+ }
}
// Keep track of all the propagated transactions
- inMeter.Mark(int64(len(txs)))
+ metrics.inMeter.Mark(int64(len(txs)))
// Push all the transactions into the pool, tracking underpriced ones to avoid
// re-requesting them and dropping the peer in case of malicious transfers.
@@ -339,41 +364,41 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool)
if end > len(txs) {
end = len(txs)
}
- var (
- duplicate int64
- underpriced int64
- otherreject int64
- )
batch := txs[i:end]
-
- var accepted []common.Hash
-
- for j, err := range f.addTxs(batch) {
- // Track the transaction hash if the price is too low for us.
- // Avoid re-request this transaction when we receive another
- // announcement.
- if errors.Is(err, txpool.ErrUnderpriced) || errors.Is(err, txpool.ErrReplaceUnderpriced) || errors.Is(err, txpool.ErrTxGasPriceTooLow) {
- f.underpriced.Add(batch[j].Hash(), batch[j].Time())
+ var (
+ poolTxs []*types.Transaction
+ blobTxs []*types.Transaction
+ )
+ if version >= eth.ETH72 {
+ for _, tx := range batch {
+ if tx.Type() == types.BlobTxType {
+ blobTxs = append(blobTxs, tx)
+ } else {
+ poolTxs = append(poolTxs, tx)
+ }
}
- // Track a few interesting failure types
- switch {
- case err == nil:
+ } else {
+ poolTxs = batch
+ }
+ batch = append(poolTxs, blobTxs...)
+
+ // Add regular tx to pool, blob tx to buffer.
+ errs := append(f.addTxs(poolTxs), f.buffer.AddTx(blobTxs, peer)...)
+
+ hashes := make([]common.Hash, len(batch))
+ for j := range batch {
+ hashes[j] = batch[j].Hash()
+ }
+ var accepted []common.Hash
+ for j, err := range errs {
+ if err == nil {
accepted = append(accepted, batch[j].Hash())
-
- case errors.Is(err, txpool.ErrAlreadyKnown):
- duplicate++
-
- case errors.Is(err, txpool.ErrUnderpriced) || errors.Is(err, txpool.ErrReplaceUnderpriced) || errors.Is(err, txpool.ErrTxGasPriceTooLow):
- underpriced++
-
- case errors.Is(err, txpool.ErrKZGVerificationError):
+ }
+ if errors.Is(err, txpool.ErrKZGVerificationError) || errors.Is(err, txpool.ErrSidecarFormatError) {
// KZG verification failed, terminate transaction processing immediately.
// Since KZG verification is computationally expensive, this acts as a
// defensive measure against potential DoS attacks.
violation = err
-
- default:
- otherreject++
}
added = append(added, batch[j].Hash())
metas = append(metas, txMetadata{
@@ -386,17 +411,16 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool)
break
}
}
- knownMeter.Mark(duplicate)
- underpricedMeter.Mark(underpriced)
- otherRejectMeter.Mark(otherreject)
+ otherreject := f.handleAddErrors(hashes, errs, metrics)
// Notify the tracker which txs from this peer were accepted.
if f.onAccepted != nil && len(accepted) > 0 {
f.onAccepted(peer, accepted)
}
- // If 'other reject' is >25% of the deliveries in any batch, sleep a bit.
- if otherreject > int64((len(batch)+3)/4) {
- log.Debug("Peer delivering stale or invalid transactions", "peer", peer, "rejected", otherreject)
+ // If 'other reject' is >25% of the deliveries in any batch, sleep a bit
+ // to throttle the misbehaving peer.
+ if otherreject > int64((len(hashes)+3)/4) {
+ log.Debug("Peer delivering stale or invalid transactions", "rejected", otherreject)
time.Sleep(200 * time.Millisecond)
}
// If we encountered a protocol violation, disconnect this peer.
@@ -412,6 +436,36 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool)
}
}
+func (f *TxFetcher) handleAddErrors(txs []common.Hash, errs []error, metrics deliveryMetrics) (otherreject int64) {
+ var (
+ duplicate int64
+ underpriced int64
+ )
+ for i, err := range errs {
+ // Track a few interesting failure types
+ switch {
+ case err == nil: // Noop, but need to handle to not count these
+
+ case errors.Is(err, txpool.ErrAlreadyKnown):
+ duplicate++
+
+ // Track the transaction hash if the price is too low for us.
+ // Avoid re-request this transaction when we receive another
+ // announcement.
+ case errors.Is(err, txpool.ErrUnderpriced) || errors.Is(err, txpool.ErrReplaceUnderpriced) || errors.Is(err, txpool.ErrTxGasPriceTooLow):
+ f.underpriced.Add(txs[i], f.realTime())
+ underpriced++
+
+ default:
+ otherreject++
+ }
+ }
+ metrics.knownMeter.Mark(duplicate)
+ metrics.underpricedMeter.Mark(underpriced)
+ metrics.otherRejectMeter.Mark(otherreject)
+ return otherreject
+}
+
// Drop should be called when a peer disconnects. It cleans up all the internal
// data structures of the given node.
func (f *TxFetcher) Drop(peer string) error {
@@ -457,6 +511,14 @@ func (f *TxFetcher) loop() {
}
for {
+ txs, errs := f.buffer.Flush()
+ f.handleAddErrors(txs, errs, deliveryMetrics{
+ inMeter: txReplyInMeter,
+ knownMeter: txReplyKnownMeter,
+ underpricedMeter: txReplyUnderpricedMeter,
+ otherRejectMeter: txReplyOtherRejectMeter,
+ })
+
select {
case ann := <-f.notify:
// Drop part of the new announcements if there are too many accumulated.
diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go
index 3fe11fda21..1c13c5bfc7 100644
--- a/eth/fetcher/tx_fetcher_test.go
+++ b/eth/fetcher/tx_fetcher_test.go
@@ -28,9 +28,11 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/txpool"
+ "github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
+ "github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/params"
"github.com/holiman/uint256"
)
@@ -60,9 +62,10 @@ type doTxNotify struct {
sizes []uint32
}
type doTxEnqueue struct {
- peer string
- txs []*types.Transaction
- direct bool
+ peer string
+ version uint
+ txs []*types.Transaction
+ direct bool
}
type doWait struct {
time time.Duration
@@ -87,6 +90,16 @@ type txFetcherTest struct {
steps []interface{}
}
+// newTestBlobBuffer returns a BlobBuffer with no-op callbacks for tests that
+// don't exercise blob handling but still need a non-nil buffer.
+func newTestBlobBuffer() *blobpool.BlobBuffer {
+ return blobpool.NewBlobBuffer(blobpool.BlobBufferFunctions{
+ ValidateTx: func(*types.Transaction) error { return nil },
+ AddToPool: func(*blobpool.BlobTxForPool) error { return nil },
+ DropPeer: func(string) {},
+ })
+}
+
// newTestTxFetcher creates a tx fetcher with noop callbacks, simulated clock,
// and deterministic randomness.
func newTestTxFetcher() *TxFetcher {
@@ -97,7 +110,9 @@ func newTestTxFetcher() *TxFetcher {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
- nil, nil,
+ nil,
+ nil,
+ newTestBlobBuffer(),
)
}
@@ -1888,7 +1903,7 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) {
// Process the original or expanded steps
switch step := step.(type) {
case doTxNotify:
- if err := fetcher.Notify(step.peer, step.types, step.sizes, step.hashes); err != nil {
+ if _, err := fetcher.Notify(step.peer, step.types, step.sizes, step.hashes); err != nil {
t.Errorf("step %d: %v", i, err)
}
<-wait // Fetcher needs to process this, wait until it's done
@@ -1899,7 +1914,7 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) {
}
case doTxEnqueue:
- if err := fetcher.Enqueue(step.peer, step.txs, step.direct); err != nil {
+ if err := fetcher.Enqueue(step.peer, step.version, step.txs, step.direct); err != nil {
t.Errorf("step %d: %v", i, err)
}
<-wait // Fetcher needs to process this, wait until it's done
@@ -2204,6 +2219,7 @@ func TestTransactionForgotten(t *testing.T) {
func(string, []common.Hash) error { return nil },
func(string) {},
nil,
+ newTestBlobBuffer(),
mockClock,
mockTime,
rand.New(rand.NewSource(0)), // Use fixed seed for deterministic behavior
@@ -2220,7 +2236,7 @@ func TestTransactionForgotten(t *testing.T) {
tx2.SetTime(now)
// Initial state: both transactions should be marked as underpriced
- if err := fetcher.Enqueue("peer", []*types.Transaction{tx1, tx2}, false); err != nil {
+ if err := fetcher.Enqueue("peer", eth.ETH70, []*types.Transaction{tx1, tx2}, false); err != nil {
t.Fatal(err)
}
if !fetcher.isKnownUnderpriced(tx1.Hash()) {
@@ -2269,7 +2285,7 @@ func TestTransactionForgotten(t *testing.T) {
// Re-enqueue tx1 with updated timestamp
tx1.SetTime(mockTime())
- if err := fetcher.Enqueue("peer", []*types.Transaction{tx1}, false); err != nil {
+ if err := fetcher.Enqueue("peer", eth.ETH70, []*types.Transaction{tx1}, false); err != nil {
t.Fatal(err)
}
if !fetcher.isKnownUnderpriced(tx1.Hash()) {
diff --git a/eth/handler.go b/eth/handler.go
index 4a9573efbe..ed0149075f 100644
--- a/eth/handler.go
+++ b/eth/handler.go
@@ -32,7 +32,9 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool"
+ "github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/fetcher"
@@ -76,7 +78,7 @@ type txPool interface {
// GetRLP retrieves the RLP-encoded transaction from local txpool
// with given tx hash.
- GetRLP(hash common.Hash) []byte
+ GetRLP(hash common.Hash, version uint) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
@@ -98,18 +100,31 @@ type txPool interface {
FilterType(kind byte) bool
}
+// blobPool defines the methods needed from a blob pool implementation to
+// support cell-based blob data availability.
+type blobPool interface {
+ Has(hash common.Hash) bool
+ GetBlobHashes(hash common.Hash) []common.Hash
+ GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error)
+ GetCustody(hash common.Hash) *types.CustodyBitmap
+ AddPooledTx(pooledTx *blobpool.BlobTxForPool) error
+ ValidateTxBasics(pooledTx *types.Transaction) error
+}
+
// handlerConfig is the collection of initialization parameters to create a full
// node network handler.
type handlerConfig struct {
- NodeID enode.ID // P2P node ID used for tx propagation topology
- Database ethdb.Database // Database for direct sync insertions
- Chain *core.BlockChain // Blockchain to serve data from
- TxPool txPool // Transaction pool to propagate from
- Network uint64 // Network identifier to advertise
- Sync ethconfig.SyncMode // Whether to snap or full sync
- BloomCache uint64 // Megabytes to alloc for snap sync bloom
- RequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges
- SnapV2 bool // Whether to advertise and sync via the snap/2 protocol
+ NodeID enode.ID // P2P node ID used for tx propagation topology
+ Database ethdb.Database // Database for direct sync insertions
+ Chain *core.BlockChain // Blockchain to serve data from
+ TxPool txPool // Transaction pool to propagate from
+ BlobPool blobPool // Blob pool for cell-based blob data availability
+ Network uint64 // Network identifier to advertise
+ Sync ethconfig.SyncMode // Whether to snap or full sync
+ BloomCache uint64 // Megabytes to alloc for snap sync bloom
+ RequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges
+ SnapV2 bool // Whether to advertise and sync via the snap/2 protocol
+ FetchProbability uint64 // Full blob fetch probability for sparse blobpool (blobFetcher)
}
type handler struct {
@@ -119,11 +134,13 @@ type handler struct {
database ethdb.Database
txpool txPool
+ blobpool blobPool
chain *core.BlockChain
maxPeers int
downloader *downloader.Downloader
txFetcher *fetcher.TxFetcher
+ blobFetcher *fetcher.BlobFetcher
txTracker *txtracker.Tracker
peers *peerSet
txBroadcastKey [16]byte
@@ -150,6 +167,7 @@ func newHandler(config *handlerConfig) (*handler, error) {
networkID: config.Network,
database: config.Database,
txpool: config.TxPool,
+ blobpool: config.BlobPool,
chain: config.Chain,
peers: newPeerSet(),
txBroadcastKey: newBroadcastChoiceKey(),
@@ -172,11 +190,19 @@ func newHandler(config *handlerConfig) (*handler, error) {
}
return p.RequestTxs(hashes)
}
+
+ // Construct the blob buffer for assembling blob txs from separate tx and cell deliveries.
+ blobBuffer := blobpool.NewBlobBuffer(blobpool.BlobBufferFunctions{
+ ValidateTx: h.blobpool.ValidateTxBasics,
+ AddToPool: h.blobpool.AddPooledTx,
+ DropPeer: h.removePeer,
+ })
+
addTxs := func(txs []*types.Transaction) []error {
return h.txpool.Add(txs, false)
}
validateMeta := func(tx common.Hash, kind byte) error {
- if h.txpool.Has(tx) {
+ if h.txpool.Has(tx) || blobBuffer.HasTx(tx) {
return txpool.ErrAlreadyKnown
}
if !h.txpool.FilterType(kind) {
@@ -185,7 +211,30 @@ func newHandler(config *handlerConfig) (*handler, error) {
return nil
}
h.txTracker = txtracker.New()
- h.txFetcher = fetcher.NewTxFetcher(h.chain, validateMeta, addTxs, fetchTx, h.removePeer, h.txTracker.NotifyAccepted)
+ h.txFetcher = fetcher.NewTxFetcher(h.chain, validateMeta, addTxs, fetchTx, h.removePeer, h.txTracker.NotifyAccepted, blobBuffer)
+
+ // Construct the blob fetcher for cell-based blob data availability
+ blobCallbacks := fetcher.BlobFetcherFunctions{
+ FetchPayloads: func(peer string, hashes []common.Hash, cells types.CustodyBitmap) error {
+ p := h.peers.peer(peer)
+ if p == nil {
+ return errors.New("unknown peer")
+ }
+ return p.RequestPayload(hashes, cells)
+ },
+ HasPayload: func(hash common.Hash) bool {
+ return h.blobpool.Has(hash) || blobBuffer.HasCells(hash)
+ },
+ AddCells: func(hash common.Hash, deliveries map[string]*fetcher.PeerCellDelivery, custody types.CustodyBitmap) {
+ converted := make(map[string]*blobpool.PeerDelivery, len(deliveries))
+ for peer, d := range deliveries {
+ converted[peer] = &blobpool.PeerDelivery{Cells: d.Cells, Indices: d.Indices}
+ }
+ blobBuffer.AddCells(hash, converted, custody)
+ },
+ DropPeer: h.removePeer,
+ }
+ h.blobFetcher = fetcher.NewBlobFetcher(blobCallbacks, types.CustodyBitmapAll, nil, config.FetchProbability)
return h, nil
}
@@ -406,6 +455,7 @@ func (h *handler) unregisterPeer(id string) {
}
h.downloader.UnregisterPeer(id)
h.txFetcher.Drop(id)
+ h.blobFetcher.Drop(id)
h.txTracker.NotifyPeerDrop(id)
if err := h.peers.unregisterPeer(id); err != nil {
@@ -429,6 +479,7 @@ func (h *handler) Start(maxPeers int) {
// start sync handlers
h.txFetcher.Start()
+ h.blobFetcher.Start()
// Start the transaction tracker (records tx deliveries, credits peer inclusions).
h.txTracker.Start(h.chain)
@@ -442,6 +493,7 @@ func (h *handler) Stop() {
h.txsSub.Unsubscribe() // quits txBroadcastLoop
h.blockRange.stop()
h.txFetcher.Stop()
+ h.blobFetcher.Stop()
h.downloader.Terminate()
// Quit chainSync and txsync64.
diff --git a/eth/handler_eth.go b/eth/handler_eth.go
index 7922687e0c..3fae75f3c5 100644
--- a/eth/handler_eth.go
+++ b/eth/handler_eth.go
@@ -23,8 +23,10 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/p2p/enode"
+ "github.com/ethereum/go-ethereum/params"
)
// ethHandler implements the eth.Backend interface to handle the various network
@@ -33,6 +35,7 @@ type ethHandler handler
func (h *ethHandler) Chain() *core.BlockChain { return h.chain }
func (h *ethHandler) TxPool() eth.TxPool { return h.txpool }
+func (h *ethHandler) BlobPool() eth.BlobPool { return h.blobpool }
// RunPeer is invoked when a peer joins on the `eth` protocol.
func (h *ethHandler) RunPeer(peer *eth.Peer, hand eth.Handler) error {
@@ -58,8 +61,19 @@ func (h *ethHandler) AcceptTxs() bool {
func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
// Consume any broadcasts and announces, forwarding the rest to the downloader
switch packet := packet.(type) {
- case *eth.NewPooledTransactionHashesPacket:
- return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
+ case *eth.NewPooledTransactionHashesPacket72:
+ hashes, err := h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
+ if err != nil {
+ return err
+ }
+ if len(hashes) != 0 {
+ return h.blobFetcher.Notify(peer.ID(), hashes, packet.Mask)
+ }
+ return nil
+
+ case *eth.NewPooledTransactionHashesPacket71:
+ _, err := h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
+ return err
case *eth.TransactionsPacket:
txs, err := packet.Items()
@@ -69,7 +83,7 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
if err := handleTransactions(peer, txs, true); err != nil {
return fmt.Errorf("Transactions: %v", err)
}
- return h.txFetcher.Enqueue(peer.ID(), txs, false)
+ return h.txFetcher.Enqueue(peer.ID(), peer.Version(), txs, false)
case *eth.PooledTransactionsPacket:
txs, err := packet.List.Items()
@@ -79,7 +93,23 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
if err := handleTransactions(peer, txs, false); err != nil {
return fmt.Errorf("PooledTransactions: %v", err)
}
- return h.txFetcher.Enqueue(peer.ID(), txs, true)
+ return h.txFetcher.Enqueue(peer.ID(), peer.Version(), txs, true)
+
+ case *eth.CellsResponse:
+ outer, err := packet.Cells.Items()
+ if err != nil {
+ return fmt.Errorf("Cells: %v", err)
+ }
+ cells := make([][]kzg4844.Cell, len(outer))
+ for i := range outer {
+ if outer[i].Len() > params.BlobTxMaxBlobs*kzg4844.CellsPerBlob {
+ return fmt.Errorf("Cells: cells per tx exceeded the possible maximum")
+ }
+ if cells[i], err = outer[i].Items(); err != nil {
+ return fmt.Errorf("Cells: %v", err)
+ }
+ }
+ return h.blobFetcher.Enqueue(peer.ID(), packet.Hashes, cells, packet.Mask)
default:
return fmt.Errorf("unexpected eth packet type: %T", packet)
diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go
index 4f74f7672f..55525e1e04 100644
--- a/eth/handler_eth_test.go
+++ b/eth/handler_eth_test.go
@@ -44,13 +44,14 @@ type testEthHandler struct {
func (h *testEthHandler) Chain() *core.BlockChain { panic("no backing chain") }
func (h *testEthHandler) TxPool() eth.TxPool { panic("no backing tx pool") }
+func (h *testEthHandler) BlobPool() eth.BlobPool { return nil }
func (h *testEthHandler) AcceptTxs() bool { return true }
func (h *testEthHandler) RunPeer(*eth.Peer, eth.Handler) error { panic("not used in tests") }
func (h *testEthHandler) PeerInfo(enode.ID) interface{} { panic("not used in tests") }
func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
switch packet := packet.(type) {
- case *eth.NewPooledTransactionHashesPacket:
+ case *eth.NewPooledTransactionHashesPacket71:
h.txAnnounces.Send(packet.Hashes)
return nil
@@ -105,10 +106,12 @@ func testForkIDSplit(t *testing.T, protocol uint) {
_, blocksNoFork, _ = core.GenerateChainWithGenesis(gspecNoFork, engine, 2, nil)
_, blocksProFork, _ = core.GenerateChainWithGenesis(gspecProFork, engine, 2, nil)
+ txPool = newTestTxPool()
ethNoFork, _ = newHandler(&handlerConfig{
Database: dbNoFork,
Chain: chainNoFork,
- TxPool: newTestTxPool(),
+ TxPool: txPool,
+ BlobPool: txPool,
Network: 1,
Sync: ethconfig.FullSync,
BloomCache: 1,
@@ -116,7 +119,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
ethProFork, _ = newHandler(&handlerConfig{
Database: dbProFork,
Chain: chainProFork,
- TxPool: newTestTxPool(),
+ TxPool: txPool,
+ BlobPool: txPool,
Network: 1,
Sync: ethconfig.FullSync,
BloomCache: 1,
@@ -137,8 +141,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
- peerNoFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil)
- peerProFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil)
+ peerNoFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil, nil)
+ peerProFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@@ -168,8 +172,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
- peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil, nil)
- peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil, nil)
+ peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil, nil, nil)
+ peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@@ -199,8 +203,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
- peerNoFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil)
- peerProFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil)
+ peerNoFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil, nil)
+ peerProFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@@ -249,8 +253,8 @@ func testRecvTransactions(t *testing.T, protocol uint) {
defer p2pSrc.Close()
defer p2pSink.Close()
- src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, nil)
- sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, nil)
+ src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, handler.txpool, nil)
+ sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, handler.txpool, nil)
defer src.Close()
defer sink.Close()
@@ -305,8 +309,8 @@ func testSendTransactions(t *testing.T, protocol uint) {
defer p2pSrc.Close()
defer p2pSink.Close()
- src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, nil)
- sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, nil)
+ src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, handler.blobpool, nil)
+ sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, handler.blobpool, nil)
defer src.Close()
defer sink.Close()
@@ -380,8 +384,8 @@ func testTransactionPropagation(t *testing.T, protocol uint) {
defer sourcePipe.Close()
defer sinkPipe.Close()
- sourcePeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{byte(i + 1)}, "", nil, sourcePipe), sourcePipe, source.txpool, nil)
- sinkPeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, sink.txpool, nil)
+ sourcePeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{byte(i + 1)}, "", nil, sourcePipe), sourcePipe, source.txpool, source.txpool, nil)
+ sinkPeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, sink.txpool, sink.txpool, nil)
defer sourcePeer.Close()
defer sinkPeer.Close()
diff --git a/eth/handler_test.go b/eth/handler_test.go
index 9cd955d29d..f53bbaecd2 100644
--- a/eth/handler_test.go
+++ b/eth/handler_test.go
@@ -29,8 +29,10 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool"
+ "github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/ethdb"
@@ -54,7 +56,10 @@ var (
// Its goal is to get around setting up a valid statedb for the balance and nonce
// checks.
type testTxPool struct {
- pool map[common.Hash]*types.Transaction // Hash map of collected transactions
+ txPool map[common.Hash]*types.Transaction // Hash map of collected transactions
+ cellPool map[common.Hash][]kzg4844.Cell
+
+ custody map[common.Hash]types.CustodyBitmap
txFeed event.Feed // Notification feed to allow waiting for inclusion
lock sync.RWMutex // Protects the transaction pool
@@ -63,7 +68,9 @@ type testTxPool struct {
// newTestTxPool creates a mock transaction pool.
func newTestTxPool() *testTxPool {
return &testTxPool{
- pool: make(map[common.Hash]*types.Transaction),
+ txPool: make(map[common.Hash]*types.Transaction),
+ cellPool: make(map[common.Hash][]kzg4844.Cell),
+ custody: make(map[common.Hash]types.CustodyBitmap),
}
}
@@ -73,7 +80,16 @@ func (p *testTxPool) Has(hash common.Hash) bool {
p.lock.Lock()
defer p.lock.Unlock()
- return p.pool[hash] != nil
+ return p.txPool[hash] != nil
+}
+
+// Has returns an indicator whether txpool has a transaction
+// cached with the given hash.
+func (p *testTxPool) HasPayload(hash common.Hash) bool {
+ p.lock.Lock()
+ defer p.lock.Unlock()
+
+ return p.cellPool[hash] != nil
}
// Get retrieves the transaction from local txpool with given
@@ -81,16 +97,16 @@ func (p *testTxPool) Has(hash common.Hash) bool {
func (p *testTxPool) Get(hash common.Hash) *types.Transaction {
p.lock.Lock()
defer p.lock.Unlock()
- return p.pool[hash]
+ return p.txPool[hash]
}
// Get retrieves the transaction from local txpool with given
// tx hash.
-func (p *testTxPool) GetRLP(hash common.Hash) []byte {
+func (p *testTxPool) GetRLP(hash common.Hash, _ uint) []byte {
p.lock.Lock()
defer p.lock.Unlock()
- tx := p.pool[hash]
+ tx := p.txPool[hash]
if tx != nil {
blob, _ := rlp.EncodeToBytes(tx)
return blob
@@ -104,7 +120,7 @@ func (p *testTxPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
p.lock.Lock()
defer p.lock.Unlock()
- tx := p.pool[hash]
+ tx := p.txPool[hash]
if tx != nil {
return &txpool.TxMetadata{
Type: tx.Type(),
@@ -121,7 +137,7 @@ func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error {
defer p.lock.Unlock()
for _, tx := range txs {
- p.pool[tx.Hash()] = tx
+ p.txPool[tx.Hash()] = tx
}
p.txFeed.Send(core.NewTxsEvent{Txs: txs})
return make([]error, len(txs))
@@ -134,7 +150,7 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]
var count int
batches := make(map[common.Address][]*types.Transaction)
- for _, tx := range p.pool {
+ for _, tx := range p.txPool {
from, _ := types.Sender(types.HomesteadSigner{}, tx)
batches[from] = append(batches[from], tx)
}
@@ -164,6 +180,87 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]
func (p *testTxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool) event.Subscription {
return p.txFeed.Subscribe(ch)
}
+func (p *testTxPool) GetBlobHashes(hash common.Hash) []common.Hash {
+ p.lock.RLock()
+ defer p.lock.RUnlock()
+
+ tx, exists := p.txPool[hash]
+ if !exists {
+ return nil
+ }
+ return tx.BlobHashes()
+}
+
+func (p *testTxPool) GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error) {
+ p.lock.RLock()
+ defer p.lock.RUnlock()
+
+ requestedIndices := mask.Indices()
+ cells := make([][]*kzg4844.Cell, len(vhashes))
+ proofs := make([][]*kzg4844.Proof, len(vhashes))
+
+ for i, vhash := range vhashes {
+ // Find the tx containing this versioned hash
+ var foundTx *types.Transaction
+ var blobIdx int
+ for _, tx := range p.txPool {
+ for j, bh := range tx.BlobHashes() {
+ if bh == vhash {
+ foundTx = tx
+ blobIdx = j
+ break
+ }
+ }
+ if foundTx != nil {
+ break
+ }
+ }
+ if foundTx == nil {
+ continue
+ }
+ txCells, ok := p.cellPool[foundTx.Hash()]
+ if !ok {
+ continue
+ }
+ _ = blobIdx // cells in the mock are stored flat by cell index
+ blobCells := make([]*kzg4844.Cell, len(requestedIndices))
+ for j, idx := range requestedIndices {
+ if int(idx) < len(txCells) {
+ cell := txCells[idx]
+ blobCells[j] = &cell
+ }
+ }
+ cells[i] = blobCells
+ }
+ return cells, proofs, nil
+}
+
+func (p *testTxPool) GetCustody(hash common.Hash) *types.CustodyBitmap {
+ p.lock.RLock()
+ defer p.lock.RUnlock()
+ mask, ok := p.custody[hash]
+ if !ok {
+ return nil
+ }
+ return &mask
+}
+
+// AddCells adds cells for a specific transaction hash (for testing)
+func (p *testTxPool) AddCells(hash common.Hash, cells []kzg4844.Cell, mask types.CustodyBitmap) {
+ p.lock.Lock()
+ defer p.lock.Unlock()
+ p.cellPool[hash] = cells
+ p.custody[hash] = mask
+}
+
+func (p *testTxPool) AddPooledTx(pooledTx *blobpool.BlobTxForPool) error {
+ p.lock.Lock()
+ defer p.lock.Unlock()
+ hash := pooledTx.Tx.Hash()
+ p.cellPool[hash] = pooledTx.CellSidecar.Cells
+ p.txPool[hash] = pooledTx.Tx
+ return nil
+}
// FilterType should check whether the pool supports the given type of transactions.
func (p *testTxPool) FilterType(kind byte) bool {
@@ -174,14 +271,19 @@ func (p *testTxPool) FilterType(kind byte) bool {
return false
}
+func (p *testTxPool) ValidateTxBasics(_ *types.Transaction) error {
+ return nil
+}
+
// testHandler is a live implementation of the Ethereum protocol handler, just
// preinitialized with some sane testing defaults and the transaction pool mocked
// out.
type testHandler struct {
- db ethdb.Database
- chain *core.BlockChain
- txpool *testTxPool
- handler *handler
+ db ethdb.Database
+ chain *core.BlockChain
+ txpool *testTxPool
+ blobpool *testTxPool
+ handler *handler
}
// newTestHandler creates a new handler for testing purposes with no blocks.
@@ -210,6 +312,7 @@ func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler
Database: db,
Chain: chain,
TxPool: txpool,
+ BlobPool: txpool,
Network: 1,
Sync: mode,
BloomCache: 1,
@@ -217,10 +320,11 @@ func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler
handler.Start(1000)
return &testHandler{
- db: db,
- chain: chain,
- txpool: txpool,
- handler: handler,
+ db: db,
+ chain: chain,
+ txpool: txpool,
+ blobpool: txpool,
+ handler: handler,
}
}
@@ -317,7 +421,7 @@ func createTestPeers(rand *rand.Rand, n int) []*ethPeer {
var id enode.ID
rand.Read(id[:])
p2pPeer := p2p.NewPeer(id, "test", nil)
- ep := eth.NewPeer(eth.ETH69, p2pPeer, nil, nil, nil)
+ ep := eth.NewPeer(eth.ETH69, p2pPeer, nil, nil, nil, nil)
peers[i] = ðPeer{Peer: ep}
}
return peers
diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go
index 21cea0d4ef..2dff4e3b5e 100644
--- a/eth/protocols/eth/broadcast.go
+++ b/eth/protocols/eth/broadcast.go
@@ -113,29 +113,58 @@ func (p *Peer) announceTransactions() {
pending []common.Hash
pendingTypes []byte
pendingSizes []uint32
+ mask types.CustodyBitmap
size common.StorageSize
+ processed = make(map[int]bool)
)
for count = 0; count < len(queue) && size < maxTxPacketSize; count++ {
if meta := p.txpool.GetMetadata(queue[count]); meta != nil {
+ custody := p.blobpool.GetCustody(queue[count])
+ if custody != nil {
+ // Blob txs should be batched into the same announcement
+ // if they share the same custody.
+ if mask.OneCount() == 0 {
+ // This is the first blob tx in this batch, so use its
+ // custody as a mask in this batch.
+ mask = *custody
+ } else {
+ if mask != *custody {
+ // Leave this in the queue so that it can be included
+ // in a later announcement.
+ continue
+ }
+ }
+ }
pending = append(pending, queue[count])
pendingTypes = append(pendingTypes, meta.Type)
- pendingSizes = append(pendingSizes, uint32(meta.Size))
+ if p.version >= ETH72 && meta.SizeWithoutBlob > 0 {
+ pendingSizes = append(pendingSizes, uint32(meta.SizeWithoutBlob))
+ } else {
+ pendingSizes = append(pendingSizes, uint32(meta.Size))
+ }
size += common.HashLength
}
+ processed[count] = true
}
- // Shift and trim queue
- queue = queue[:copy(queue, queue[count:])]
+ // Shift and trim queue using processed map
+ var remaining []common.Hash
+ for i, h := range queue {
+ if !processed[i] {
+ remaining = append(remaining, h)
+ }
+ }
+ queue = remaining
// If there's anything available to transfer, fire up an async writer
if len(pending) > 0 {
done = make(chan struct{})
go func() {
- if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes); err != nil {
+ if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes, mask); err != nil {
fail <- err
return
}
close(done)
- p.Log().Trace("Sent transaction announcements", "count", len(pending))
+ p.Log().Trace("Sent transaction announcements", "count", len(pending), "mask", mask, "tx", pending)
}()
}
}
diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go
index 325ffa0738..83a1275f46 100644
--- a/eth/protocols/eth/handler.go
+++ b/eth/protocols/eth/handler.go
@@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
@@ -71,6 +72,9 @@ type Backend interface {
// TxPool retrieves the transaction pool object to serve data.
TxPool() TxPool
+ // BlobPool retrieves the blob pool object to serve cell requests.
+ BlobPool() BlobPool
+
// AcceptTxs retrieves whether transaction processing is enabled on the node
// or if inbound transactions should simply be dropped.
AcceptTxs() bool
@@ -90,6 +94,18 @@ type Backend interface {
Handle(peer *Peer, packet Packet) error
}
+// BlobPool defines the methods needed by the protocol handler to serve cell requests.
+type BlobPool interface {
+ // GetBlobHashes returns the blob versioned hashes for a given transaction hash.
+ GetBlobHashes(hash common.Hash) []common.Hash
+ // GetBlobCells retrieves cells and proofs for given versioned blob hashes filtered by the custody bitmap.
+ GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error)
+ // GetCustody returns the custody bitmap for a given transaction hash.
+ GetCustody(hash common.Hash) *types.CustodyBitmap
+ // Has returns whether the blob pool contains a transaction with the given hash.
+ Has(hash common.Hash) bool
+}
+
// TxPool defines the methods needed by the protocol handler to serve transactions.
type TxPool interface {
// Get retrieves the transaction from the local txpool with the given hash.
@@ -97,7 +113,7 @@ type TxPool interface {
// GetRLP retrieves the RLP-encoded transaction from the local txpool with
// the given hash.
- GetRLP(hash common.Hash) []byte
+ GetRLP(hash common.Hash, version uint) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
@@ -113,7 +129,7 @@ func MakeProtocols(backend Backend, network uint64, disc enode.Iterator) []p2p.P
Version: version,
Length: protocolLengths[version],
Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
- peer := NewPeer(version, p, rw, backend.TxPool(), backend.Chain().Config())
+ peer := NewPeer(version, p, rw, backend.TxPool(), backend.BlobPool(), backend.Chain().Config())
defer peer.Close()
return backend.RunPeer(peer, func(peer *Peer) error {
@@ -216,6 +232,24 @@ var eth71 = map[uint64]msgHandler{
BlockAccessListsMsg: handleBlockAccessLists,
}
+var eth72 = map[uint64]msgHandler{
+ TransactionsMsg: handleTransactions,
+ NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes72,
+ GetBlockHeadersMsg: handleGetBlockHeaders,
+ BlockHeadersMsg: handleBlockHeaders,
+ GetBlockBodiesMsg: handleGetBlockBodies,
+ BlockBodiesMsg: handleBlockBodies,
+ GetReceiptsMsg: handleGetReceipts70,
+ ReceiptsMsg: handleReceipts70,
+ GetPooledTransactionsMsg: handleGetPooledTransactions,
+ PooledTransactionsMsg: handlePooledTransactions,
+ BlockRangeUpdateMsg: handleBlockRangeUpdate,
+ GetBlockAccessListsMsg: handleGetBlockAccessLists,
+ BlockAccessListsMsg: handleBlockAccessLists,
+ GetCellsMsg: handleGetCells,
+ CellsMsg: handleCells,
+}
+
// handleMessage is invoked whenever an inbound message is received from a remote
// peer. The remote connection is torn down upon returning any error.
func handleMessage(backend Backend, peer *Peer) error {
@@ -237,6 +271,8 @@ func handleMessage(backend Backend, peer *Peer) error {
handlers = eth70
case ETH71:
handlers = eth71
+ case ETH72:
+ handlers = eth72
default:
return fmt.Errorf("unknown eth protocol version: %v", peer.version)
}
diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go
index 117e1b0aba..c3b15b999a 100644
--- a/eth/protocols/eth/handler_test.go
+++ b/eth/protocols/eth/handler_test.go
@@ -63,9 +63,10 @@ func u64(val uint64) *uint64 { return &val }
// purpose is to allow testing the request/reply workflows and wire serialization
// in the `eth` protocol without actually doing any data processing.
type testBackend struct {
- db ethdb.Database
- chain *core.BlockChain
- txpool *txpool.TxPool
+ db ethdb.Database
+ chain *core.BlockChain
+ txpool *txpool.TxPool
+ blobpool *blobpool.BlobPool
}
// newTestBackend creates an empty chain and wraps it into a mock backend.
@@ -143,9 +144,10 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, cancun bool, generat
txpool, _ := txpool.New(txconfig.PriceLimit, chain, []txpool.SubPool{legacyPool, blobPool})
return &testBackend{
- db: db,
- chain: chain,
- txpool: txpool,
+ db: db,
+ chain: chain,
+ txpool: txpool,
+ blobpool: blobPool,
}
}
@@ -157,6 +159,7 @@ func (b *testBackend) close() {
func (b *testBackend) Chain() *core.BlockChain { return b.chain }
func (b *testBackend) TxPool() TxPool { return b.txpool }
+func (b *testBackend) BlobPool() BlobPool { return b.blobpool }
func (b *testBackend) RunPeer(peer *Peer, handler Handler) error {
// Normally the backend would do peer maintenance and handshakes. All that
diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go
index f8f5c8e8e7..4beb0a7f48 100644
--- a/eth/protocols/eth/handlers.go
+++ b/eth/protocols/eth/handlers.go
@@ -26,6 +26,7 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/tracker"
"github.com/ethereum/go-ethereum/rlp"
@@ -568,7 +569,27 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer)
if !backend.AcceptTxs() {
return nil
}
- ann := new(NewPooledTransactionHashesPacket)
+ ann := new(NewPooledTransactionHashesPacket71)
+ if err := msg.Decode(ann); err != nil {
+ return err
+ }
+ if len(ann.Hashes) != len(ann.Types) || len(ann.Hashes) != len(ann.Sizes) {
+ return fmt.Errorf("NewPooledTransactionHashes: invalid len of fields in %v %v %v", len(ann.Hashes), len(ann.Types), len(ann.Sizes))
+ }
+ // Schedule all the unknown hashes for retrieval
+ for _, hash := range ann.Hashes {
+ peer.MarkTransaction(hash)
+ }
+ return backend.Handle(peer, ann)
+}
+
+func handleNewPooledTransactionHashes72(backend Backend, msg Decoder, peer *Peer) error {
+ // New transaction announcement arrived, make sure we have
+ // a valid and fresh chain to handle them
+ if !backend.AcceptTxs() {
+ return nil
+ }
+ ann := new(NewPooledTransactionHashesPacket72)
if err := msg.Decode(ann); err != nil {
return err
}
@@ -588,11 +609,11 @@ func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error
if err := msg.Decode(&query); err != nil {
return err
}
- hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsRequest)
+ hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsRequest, peer.version)
return peer.ReplyPooledTransactionsRLP(query.RequestId, hashes, txs)
}
-func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsRequest) ([]common.Hash, []rlp.RawValue) {
+func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsRequest, version uint) ([]common.Hash, []rlp.RawValue) {
// Gather transactions until the fetch or network limits is reached
var (
bytes int
@@ -604,7 +625,7 @@ func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsReq
break
}
// Retrieve the requested transaction, skipping if unknown to us
- encoded := backend.TxPool().GetRLP(hash)
+ encoded := backend.TxPool().GetRLP(hash, version)
if len(encoded) == 0 {
continue
}
@@ -667,6 +688,80 @@ func handleBlockRangeUpdate(backend Backend, msg Decoder, peer *Peer) error {
return nil
}
+func handleGetCells(backend Backend, msg Decoder, peer *Peer) error {
+ // Decode the cell retrieval message
+ var query GetCellsRequestPacket
+ if err := msg.Decode(&query); err != nil {
+ return err
+ }
+ hashes, cells, custody := answerGetCells(backend, query.GetCellsRequest)
+ return peer.ReplyCells(query.RequestId, hashes, cells, custody)
+}
+
+func answerGetCells(backend Backend, query GetCellsRequest) ([]common.Hash, [][]kzg4844.Cell, types.CustodyBitmap) {
+ var (
+ cellCounts int
+ hashes []common.Hash
+ cells [][]kzg4844.Cell
+ )
+ maxCells := softResponseLimit / 2048
+ for _, hash := range query.Hashes {
+ if cellCounts >= maxCells {
+ break
+ }
+ // Look up the blob versioned hashes for this transaction
+ vhashes := backend.BlobPool().GetBlobHashes(hash)
+ if len(vhashes) == 0 {
+ continue
+ }
+ blobCells, _, _ := backend.BlobPool().GetBlobCells(vhashes, query.Mask)
+
+ // Flatten per-blob cells into a single slice. If any blob has a nil
+ // entry (unavailable cell), skip the entire transaction.
+ var flat []kzg4844.Cell
+ skip := false
+ for _, bc := range blobCells {
+ if bc == nil {
+ skip = true
+ break
+ }
+ for _, c := range bc {
+ if c == nil {
+ skip = true
+ break
+ }
+ flat = append(flat, *c)
+ }
+ if skip {
+ break
+ }
+ }
+ if skip || len(flat) == 0 {
+ continue
+ }
+ hashes = append(hashes, hash)
+ cells = append(cells, flat)
+ cellCounts += len(flat)
+ }
+ return hashes, cells, query.Mask
+}
+
+func handleCells(backend Backend, msg Decoder, peer *Peer) error {
+ var cellsResponse CellsPacket
+ if err := msg.Decode(&cellsResponse); err != nil {
+ return err
+ }
+ tresp := tracker.Response{
+ ID: cellsResponse.RequestId,
+ MsgCode: CellsMsg,
+ Size: cellsResponse.CellsResponse.Cells.Len(),
+ }
+ if err := peer.tracker.Fulfil(tresp); err != nil {
+ return fmt.Errorf("Cells: %w", err)
+ }
+ return backend.Handle(peer, &cellsResponse.CellsResponse)
+}
+
// handleGetBlockAccessLists serves a GetBlockAccessLists request.
func handleGetBlockAccessLists(backend Backend, msg Decoder, peer *Peer) error {
var query GetBlockAccessListsPacket
diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go
index 5746d5896d..95d307d639 100644
--- a/eth/protocols/eth/handshake_test.go
+++ b/eth/protocols/eth/handshake_test.go
@@ -77,7 +77,7 @@ func testHandshake(t *testing.T, protocol uint) {
defer app.Close()
defer net.Close()
- peer := NewPeer(protocol, p2p.NewPeer(enode.ID{}, "peer", nil), net, nil, nil)
+ peer := NewPeer(protocol, p2p.NewPeer(enode.ID{}, "peer", nil), net, nil, nil, nil)
defer peer.Close()
// Send the junk test with one peer, check the handshake failure
diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go
index 1e33fbe408..583b2dce38 100644
--- a/eth/protocols/eth/peer.go
+++ b/eth/protocols/eth/peer.go
@@ -27,6 +27,7 @@ import (
mapset "github.com/deckarep/golang-set/v2"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/tracker"
"github.com/ethereum/go-ethereum/params"
@@ -66,7 +67,8 @@ type Peer struct {
version uint // Protocol version negotiated
lastRange atomic.Pointer[BlockRangeUpdatePacket]
- txpool TxPool // Transaction pool used by the broadcasters for liveness checks
+ txpool TxPool // Transaction pool used by the broadcasters for liveness checks
+ blobpool BlobPool
knownTxs *knownCache // Set of transaction hashes known to be known by this peer
txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests
txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests
@@ -86,11 +88,11 @@ type Peer struct {
// NewPeer creates a wrapper for a network connection and negotiated protocol
// version.
-func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool, chainConfig *params.ChainConfig) *Peer {
+func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool, blobpool BlobPool, chainConfig *params.ChainConfig) *Peer {
cap := p2p.Cap{Name: ProtocolName, Version: version}
id := p.ID().String()
peer := &Peer{
- id: p.ID().String(),
+ id: id,
Peer: p,
rw: rw,
version: version,
@@ -102,6 +104,7 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool, cha
reqCancel: make(chan *cancel),
resDispatch: make(chan *response),
txpool: txpool,
+ blobpool: blobpool,
chainConfig: chainConfig,
receiptBuffer: make(map[uint64]*receiptRequest),
term: make(chan struct{}),
@@ -188,13 +191,13 @@ func (p *Peer) AsyncSendTransactions(hashes []common.Hash) {
// This method is a helper used by the async transaction announcer. Don't call it
// directly as the queueing (memory) and transmission (bandwidth) costs should
// not be managed directly.
-func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32) error {
- if err := p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket{Types: types, Sizes: sizes, Hashes: hashes}); err != nil {
- return err
- }
+func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32, cells types.CustodyBitmap) error {
// Mark all the transactions as known, but ensure we don't overflow our limits
p.knownTxs.Add(hashes...)
- return nil
+ if p.version >= ETH72 {
+ return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket72{Types: types, Sizes: sizes, Hashes: hashes, Mask: cells})
+ }
+ return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket71{Types: types, Sizes: sizes, Hashes: hashes})
}
// AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually
@@ -248,6 +251,53 @@ func (p *Peer) ReplyReceiptsRLP69(id uint64, receipts rlp.RawList[*ReceiptList])
})
}
+// ReplyCells is the response to GetCells.
+func (p *Peer) ReplyCells(id uint64, hashes []common.Hash, cells [][]kzg4844.Cell, mask types.CustodyBitmap) error {
+ inner := make([]rlp.RawList[kzg4844.Cell], len(cells))
+ for i, c := range cells {
+ raw, err := rlp.EncodeToRawList(c)
+ if err != nil {
+ return err
+ }
+ inner[i] = raw
+ }
+ rawCells, err := rlp.EncodeToRawList(inner)
+ if err != nil {
+ return err
+ }
+ return p2p.Send(p.rw, CellsMsg, &CellsPacket{
+ RequestId: id,
+ CellsResponse: CellsResponse{
+ Hashes: hashes,
+ Cells: rawCells,
+ Mask: mask,
+ },
+ })
+}
+
+// RequestPayload fetches a batch of cells from a remote node.
+func (p *Peer) RequestPayload(hashes []common.Hash, cell types.CustodyBitmap) error {
+ p.Log().Debug("Fetching batch of cells", "txcount", len(hashes), "cellcount", cell.OneCount())
+ id := rand.Uint64()
+
+ err := p.tracker.Track(tracker.Request{
+ ID: id,
+ ReqCode: GetCellsMsg,
+ RespCode: CellsMsg,
+ Size: len(hashes),
+ })
+ if err != nil {
+ return err
+ }
+ return p2p.Send(p.rw, GetCellsMsg, &GetCellsRequestPacket{
+ RequestId: id,
+ GetCellsRequest: GetCellsRequest{
+ Hashes: hashes,
+ Mask: cell,
+ },
+ })
+}
+
// ReplyReceiptsRLP70 is the response to GetReceipts.
func (p *Peer) ReplyReceiptsRLP70(id uint64, receipts rlp.RawList[*ReceiptList], lastBlockIncomplete bool) error {
return p2p.Send(p.rw, ReceiptsMsg, &ReceiptsPacket70{
diff --git a/eth/protocols/eth/peer_test.go b/eth/protocols/eth/peer_test.go
index 81b762452e..c0f646433e 100644
--- a/eth/protocols/eth/peer_test.go
+++ b/eth/protocols/eth/peer_test.go
@@ -45,7 +45,7 @@ func newTestPeer(name string, version uint, backend Backend) (*testPeer, <-chan
var id enode.ID
rand.Read(id[:])
- peer := NewPeer(version, p2p.NewPeer(id, name, nil), net, backend.TxPool(), nil)
+ peer := NewPeer(version, p2p.NewPeer(id, name, nil), net, backend.TxPool(), backend.BlobPool(), nil)
errc := make(chan error, 1)
go func() {
defer app.Close()
diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go
index e7ffcf6d5e..6dd1e8bec3 100644
--- a/eth/protocols/eth/protocol.go
+++ b/eth/protocols/eth/protocol.go
@@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/forkid"
"github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/rlp"
)
@@ -32,6 +33,7 @@ const (
ETH69 = 69
ETH70 = 70
ETH71 = 71
+ ETH72 = 72
)
// ProtocolName is the official short name of the `eth` protocol used during
@@ -40,11 +42,11 @@ const ProtocolName = "eth"
// ProtocolVersions are the supported versions of the `eth` protocol (first
// is primary).
-var ProtocolVersions = []uint{ETH71, ETH70, ETH69}
+var ProtocolVersions = []uint{ETH72, ETH71, ETH70, ETH69}
// protocolLengths are the number of implemented message corresponding to
// different protocol versions.
-var protocolLengths = map[uint]uint64{ETH71: 20, ETH69: 18, ETH70: 18}
+var protocolLengths = map[uint]uint64{ETH69: 18, ETH70: 18, ETH71: 20, ETH72: 22}
// maxMessageSize is the maximum cap on the size of a protocol message.
const maxMessageSize = 10 * 1024 * 1024
@@ -69,6 +71,8 @@ const (
BlockRangeUpdateMsg = 0x11
GetBlockAccessListsMsg = 0x12
BlockAccessListsMsg = 0x13
+ GetCellsMsg = 0x14
+ CellsMsg = 0x15
)
var (
@@ -248,13 +252,23 @@ type ReceiptsPacket70 struct {
// ReceiptsRLPResponse is used for receipts, when we already have it encoded
type ReceiptsRLPResponse []rlp.RawValue
-// NewPooledTransactionHashesPacket represents a transaction announcement packet on eth/68 and newer.
-type NewPooledTransactionHashesPacket struct {
+// NewPooledTransactionHashesPacket71 represents a transaction announcement packet on protocol version
+// less than or equal to 71.
+type NewPooledTransactionHashesPacket71 struct {
Types []byte
Sizes []uint32
Hashes []common.Hash
}
+// NewPooledTransactionHashesPacket72 represents a transaction announcement packet on ETH/72
+// with an additional custody bitmap field for cell-based blob data availability.
+type NewPooledTransactionHashesPacket72 struct {
+ Types []byte
+ Sizes []uint32
+ Hashes []common.Hash
+ Mask types.CustodyBitmap
+}
+
// GetPooledTransactionsRequest represents a transaction query.
type GetPooledTransactionsRequest []common.Hash
@@ -291,6 +305,31 @@ type BlockRangeUpdatePacket struct {
LatestBlockHash common.Hash
}
+// GetCellsRequest represents a request for cells of blob transactions.
+type GetCellsRequest struct {
+ Hashes []common.Hash
+ Mask types.CustodyBitmap
+}
+
+// GetCellsRequestPacket represents a cell request with request ID wrapping.
+type GetCellsRequestPacket struct {
+ RequestId uint64
+ GetCellsRequest
+}
+
+// CellsResponse represents a response containing cells for blob transactions.
+type CellsResponse struct {
+ Hashes []common.Hash
+ Cells rlp.RawList[rlp.RawList[kzg4844.Cell]]
+ Mask types.CustodyBitmap
+}
+
+// CellsPacket represents a cells response with request ID wrapping.
+type CellsPacket struct {
+ RequestId uint64
+ CellsResponse
+}
+
type GetBlockAccessListsRequest []common.Hash
type GetBlockAccessListsPacket struct {
@@ -326,8 +365,11 @@ func (*GetBlockBodiesRequest) Kind() byte { return GetBlockBodiesMsg }
func (*BlockBodiesResponse) Name() string { return "BlockBodies" }
func (*BlockBodiesResponse) Kind() byte { return BlockBodiesMsg }
-func (*NewPooledTransactionHashesPacket) Name() string { return "NewPooledTransactionHashes" }
-func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransactionHashesMsg }
+func (*NewPooledTransactionHashesPacket71) Name() string { return "NewPooledTransactionHashes" }
+func (*NewPooledTransactionHashesPacket71) Kind() byte { return NewPooledTransactionHashesMsg }
+
+func (*NewPooledTransactionHashesPacket72) Name() string { return "NewPooledTransactionHashes" }
+func (*NewPooledTransactionHashesPacket72) Kind() byte { return NewPooledTransactionHashesMsg }
func (*GetPooledTransactionsRequest) Name() string { return "GetPooledTransactions" }
func (*GetPooledTransactionsRequest) Kind() byte { return GetPooledTransactionsMsg }
@@ -352,3 +394,9 @@ func (*GetBlockAccessListsRequest) Kind() byte { return GetBlockAccessListsMsg
func (*BlockAccessListResponse) Name() string { return "BlockAccessLists" }
func (*BlockAccessListResponse) Kind() byte { return BlockAccessListsMsg }
+
+func (*GetCellsRequest) Name() string { return "GetCells" }
+func (*GetCellsRequest) Kind() byte { return GetCellsMsg }
+
+func (*CellsResponse) Name() string { return "Cells" }
+func (*CellsResponse) Kind() byte { return CellsMsg }
diff --git a/eth/sync_test.go b/eth/sync_test.go
index e22c495275..6d8ed696b5 100644
--- a/eth/sync_test.go
+++ b/eth/sync_test.go
@@ -50,8 +50,8 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) {
defer emptyPipeEth.Close()
defer fullPipeEth.Close()
- emptyPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{1}, "", caps), emptyPipeEth, empty.txpool, nil)
- fullPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{2}, "", caps), fullPipeEth, full.txpool, nil)
+ emptyPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{1}, "", caps), emptyPipeEth, empty.txpool, empty.blobpool, nil)
+ fullPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{2}, "", caps), fullPipeEth, full.txpool, full.blobpool, nil)
defer emptyPeerEth.Close()
defer fullPeerEth.Close()
diff --git a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go
index 69674d0e62..6e9b8acd83 100644
--- a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go
+++ b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go
@@ -25,22 +25,28 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
+ "github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/eth/fetcher"
+ "github.com/ethereum/go-ethereum/eth/protocols/eth"
)
var (
- peers []string
- txs []*types.Transaction
+ peers []string
+ peerVersions map[string]uint
+ txs []*types.Transaction
)
func init() {
// Random is nice, but we need it deterministic
rand := rand.New(rand.NewSource(0x3a29))
+ supportedVersions := []uint{eth.ETH69, eth.ETH70, eth.ETH72}
peers = make([]string, 10)
+ peerVersions = make(map[string]uint, len(peers))
for i := 0; i < len(peers); i++ {
peers[i] = fmt.Sprintf("Peer #%d", i)
+ peerVersions[peers[i]] = supportedVersions[i%len(supportedVersions)]
}
txs = make([]*types.Transaction, 65536) // We need to bump enough to hit all the limits
for i := 0; i < len(txs); i++ {
@@ -84,7 +90,13 @@ func fuzz(input []byte) int {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
- nil, nil,
+ nil,
+ nil,
+ blobpool.NewBlobBuffer(blobpool.BlobBufferFunctions{
+ ValidateTx: func(*types.Transaction) error { return nil },
+ AddToPool: func(*blobpool.BlobTxForPool) error { return nil },
+ DropPeer: func(string) {},
+ }),
clock,
func() time.Time {
nanoTime := int64(clock.Now())
@@ -139,7 +151,7 @@ func fuzz(input []byte) int {
if verbose {
fmt.Println("Notify", peer, announceIdxs)
}
- if err := f.Notify(peer, types, sizes, announces); err != nil {
+ if _, err := f.Notify(peer, types, sizes, announces); err != nil {
panic(err)
}
@@ -180,7 +192,7 @@ func fuzz(input []byte) int {
if verbose {
fmt.Println("Enqueue", peer, deliverIdxs, direct)
}
- if err := f.Enqueue(peer, deliveries, direct); err != nil {
+ if err := f.Enqueue(peer, peerVersions[peer], deliveries, direct); err != nil {
panic(err)
}