implement sparse blobpool

This commit is contained in:
healthykim 2026-03-19 02:52:39 +09:00
parent ab357151da
commit e84b6eb568
41 changed files with 3638 additions and 361 deletions

View file

@ -167,7 +167,7 @@ func (c *Conn) ReadEth() (any, error) {
case eth.TransactionsMsg: case eth.TransactionsMsg:
msg = new(eth.TransactionsPacket) msg = new(eth.TransactionsPacket)
case eth.NewPooledTransactionHashesMsg: case eth.NewPooledTransactionHashesMsg:
msg = new(eth.NewPooledTransactionHashesPacket) msg = new(eth.NewPooledTransactionHashesPacket70)
case eth.GetPooledTransactionsMsg: case eth.GetPooledTransactionsMsg:
msg = new(eth.GetPooledTransactionsPacket) msg = new(eth.GetPooledTransactionsPacket)
case eth.PooledTransactionsMsg: case eth.PooledTransactionsMsg:

View file

@ -865,7 +865,7 @@ the transactions using a GetPooledTransactions request.`)
} }
// Send announcement. // Send announcement.
ann := eth.NewPooledTransactionHashesPacket{Types: txTypes, Sizes: sizes, Hashes: hashes} ann := eth.NewPooledTransactionHashesPacket70{Types: txTypes, Sizes: sizes, Hashes: hashes}
err = conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann) err = conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann)
if err != nil { if err != nil {
t.Fatalf("failed to write to connection: %v", err) t.Fatalf("failed to write to connection: %v", err)
@ -883,7 +883,7 @@ the transactions using a GetPooledTransactions request.`)
t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsRequest)) t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsRequest))
} }
return return
case *eth.NewPooledTransactionHashesPacket: case *eth.NewPooledTransactionHashesPacket70:
continue continue
case *eth.TransactionsPacket: case *eth.TransactionsPacket:
continue continue
@ -949,12 +949,12 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
t2 = s.makeBlobTxs(2, 3, 0x2) t2 = s.makeBlobTxs(2, 3, 0x2)
) )
for _, test := range []struct { for _, test := range []struct {
ann eth.NewPooledTransactionHashesPacket ann eth.NewPooledTransactionHashesPacket70
resp eth.PooledTransactionsResponse resp eth.PooledTransactionsResponse
}{ }{
// Invalid tx size. // Invalid tx size.
{ {
ann: eth.NewPooledTransactionHashesPacket{ ann: eth.NewPooledTransactionHashesPacket70{
Types: []byte{types.BlobTxType, types.BlobTxType}, Types: []byte{types.BlobTxType, types.BlobTxType},
Sizes: []uint32{uint32(t1[0].Size()), uint32(t1[1].Size() + 10)}, Sizes: []uint32{uint32(t1[0].Size()), uint32(t1[1].Size() + 10)},
Hashes: []common.Hash{t1[0].Hash(), t1[1].Hash()}, Hashes: []common.Hash{t1[0].Hash(), t1[1].Hash()},
@ -963,7 +963,7 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
}, },
// Wrong tx type. // Wrong tx type.
{ {
ann: eth.NewPooledTransactionHashesPacket{ ann: eth.NewPooledTransactionHashesPacket70{
Types: []byte{types.DynamicFeeTxType, types.BlobTxType}, Types: []byte{types.DynamicFeeTxType, types.BlobTxType},
Sizes: []uint32{uint32(t2[0].Size()), uint32(t2[1].Size())}, Sizes: []uint32{uint32(t2[0].Size()), uint32(t2[1].Size())},
Hashes: []common.Hash{t2[0].Hash(), t2[1].Hash()}, Hashes: []common.Hash{t2[0].Hash(), t2[1].Hash()},
@ -1092,7 +1092,7 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types
return return
} }
ann := eth.NewPooledTransactionHashesPacket{ ann := eth.NewPooledTransactionHashesPacket70{
Types: []byte{types.BlobTxType}, Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(badTx.Size())}, Sizes: []uint32{uint32(badTx.Size())},
Hashes: []common.Hash{badTx.Hash()}, Hashes: []common.Hash{badTx.Hash()},
@ -1143,7 +1143,7 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types
return return
} }
ann := eth.NewPooledTransactionHashesPacket{ ann := eth.NewPooledTransactionHashesPacket70{
Types: []byte{types.BlobTxType}, Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(tx.Size())}, Sizes: []uint32{uint32(tx.Size())},
Hashes: []common.Hash{tx.Hash()}, Hashes: []common.Hash{tx.Hash()},

View file

@ -74,7 +74,7 @@ func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error {
for _, tx := range txs { for _, tx := range txs {
got[tx.Hash()] = true got[tx.Hash()] = true
} }
case *eth.NewPooledTransactionHashesPacket: case *eth.NewPooledTransactionHashesPacket70:
for _, hash := range msg.Hashes { for _, hash := range msg.Hashes {
got[hash] = true got[hash] = true
} }
@ -160,7 +160,7 @@ func (s *Suite) sendInvalidTxs(t *utesting.T, txs []*types.Transaction) error {
return fmt.Errorf("received bad tx: %s", tx.Hash()) return fmt.Errorf("received bad tx: %s", tx.Hash())
} }
} }
case *eth.NewPooledTransactionHashesPacket: case *eth.NewPooledTransactionHashesPacket70:
for _, hash := range msg.Hashes { for _, hash := range msg.Hashes {
if _, ok := invalids[hash]; ok { if _, ok := invalids[hash]; ok {
return fmt.Errorf("received bad tx: %s", hash) return fmt.Errorf("received bad tx: %s", hash)

View file

@ -131,6 +131,8 @@ type blobTxMeta struct {
storageSize uint32 // Byte size in the pool's persistent store storageSize uint32 // Byte size in the pool's persistent store
size uint64 // RLP-encoded size of transaction including the attached blob size uint64 // RLP-encoded size of transaction including the attached blob
custody *types.CustodyBitmap
nonce uint64 // Needed to prioritize inclusion order within an account nonce uint64 // Needed to prioritize inclusion order within an account
costCap *uint256.Int // Needed to validate cumulative balance sufficiency costCap *uint256.Int // Needed to validate cumulative balance sufficiency
execTipCap *uint256.Int // Needed to prioritize inclusion order across accounts and validate replacement price bump execTipCap *uint256.Int // Needed to prioritize inclusion order across accounts and validate replacement price bump
@ -147,28 +149,28 @@ type blobTxMeta struct {
evictionBlobFeeJumps float64 // Worse blob fee (converted to fee jumps) across all previous nonces evictionBlobFeeJumps float64 // Worse blob fee (converted to fee jumps) across all previous nonces
} }
// newBlobTxMeta retrieves the indexed metadata fields from a blob transaction // newBlobTxMeta retrieves the indexed metadata fields from a pooled blob transaction
// and assembles a helper struct to track in memory. // and assembles a helper struct to track in memory.
// Requires the transaction to have a sidecar (or that we introduce a special version tag for no-sidecar). func newBlobTxMeta(id uint64, size uint64, storageSize uint32, pooledTx *pooledBlobTx) *blobTxMeta {
func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transaction) *blobTxMeta { if pooledTx.Sidecar == nil {
if tx.BlobTxSidecar() == nil {
// This should never happen, as the pool only admits blob transactions with a sidecar // This should never happen, as the pool only admits blob transactions with a sidecar
panic("missing blob tx sidecar") panic("missing blob tx sidecar")
} }
meta := &blobTxMeta{ meta := &blobTxMeta{
hash: tx.Hash(), hash: pooledTx.Transaction.Hash(),
vhashes: tx.BlobHashes(), vhashes: pooledTx.Transaction.BlobHashes(),
version: tx.BlobTxSidecar().Version, version: pooledTx.Sidecar.Version,
id: id, id: id,
storageSize: storageSize, storageSize: storageSize,
size: size, size: size,
nonce: tx.Nonce(), nonce: pooledTx.Transaction.Nonce(),
costCap: uint256.MustFromBig(tx.Cost()), costCap: uint256.MustFromBig(pooledTx.Transaction.Cost()),
execTipCap: uint256.MustFromBig(tx.GasTipCap()), execTipCap: uint256.MustFromBig(pooledTx.Transaction.GasTipCap()),
execFeeCap: uint256.MustFromBig(tx.GasFeeCap()), execFeeCap: uint256.MustFromBig(pooledTx.Transaction.GasFeeCap()),
blobFeeCap: uint256.MustFromBig(tx.BlobGasFeeCap()), blobFeeCap: uint256.MustFromBig(pooledTx.Transaction.BlobGasFeeCap()),
execGas: tx.Gas(), execGas: pooledTx.Transaction.Gas(),
blobGas: tx.BlobGas(), blobGas: pooledTx.Transaction.BlobGas(),
custody: &pooledTx.Sidecar.Custody,
} }
meta.basefeeJumps = dynamicFeeJumps(meta.execFeeCap) meta.basefeeJumps = dynamicFeeJumps(meta.execFeeCap)
meta.blobfeeJumps = dynamicBlobFeeJumps(meta.blobFeeCap) meta.blobfeeJumps = dynamicBlobFeeJumps(meta.blobFeeCap)
@ -176,6 +178,42 @@ func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transac
return meta return meta
} }
type pooledBlobTx struct {
Transaction *types.Transaction
Sidecar *types.BlobTxCellSidecar
Size uint64 // original transaction size (including blobs)
}
// newPooledBlobTx creates pooledBlobTx struct.
func newPooledBlobTx(tx *types.Transaction) (*pooledBlobTx, error) {
if tx.BlobTxSidecar() == nil {
return nil, errors.New("missing blob sidecar")
}
sidecar, err := tx.BlobTxSidecar().ToBlobTxCellSidecar()
if err != nil {
return nil, err
}
return &pooledBlobTx{
Transaction: tx.WithoutBlobTxSidecar(),
Sidecar: sidecar,
Size: tx.Size(),
}, nil
}
// convert recovers blobs from cell sidecar and returns a full transaction with blob sidecar.
func (ptx *pooledBlobTx) convert() (*types.Transaction, error) {
if ptx.Sidecar == nil {
return nil, errors.New("cell sidecar missing")
}
cellSidecar := ptx.Sidecar
blobs, err := kzg4844.RecoverBlobs(cellSidecar.Cells, cellSidecar.Custody.Indices())
if err != nil {
return nil, err
}
sidecar := types.NewBlobTxSidecar(cellSidecar.Version, blobs, cellSidecar.Commitments, cellSidecar.Proofs)
return ptx.Transaction.WithBlobTxSidecar(sidecar), nil
}
// BlobPool is the transaction pool dedicated to EIP-4844 blob transactions. // BlobPool is the transaction pool dedicated to EIP-4844 blob transactions.
// //
// Blob transactions are special snowflakes that are designed for a very specific // Blob transactions are special snowflakes that are designed for a very specific
@ -367,9 +405,17 @@ type BlobPool struct {
stored uint64 // Useful data size of all transactions on disk stored uint64 // Useful data size of all transactions on disk
limbo *limbo // Persistent data store for the non-finalized blobs limbo *limbo // Persistent data store for the non-finalized blobs
gapped map[common.Address][]*types.Transaction // Transactions that are currently gapped (nonce too high) gapped map[common.Address][]*pooledBlobTx // Transactions that are currently gapped (nonce too high)
gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion
queue map[common.Hash]*types.Transaction // buffer
indexQueue map[common.Address][]*blobTxMeta // tx hashes in queue per address, sorted by nonce
spentQueue map[common.Address]*uint256.Int // Expenditure tracking for accounts, only for buffered txs
replacementQueue map[common.Address]map[uint64]*blobTxMeta // Replacement queue for pooled transactions
cellQueue map[common.Hash][]kzg4844.Cell // cell buffer
custodyQueue map[common.Hash]*types.CustodyBitmap
signer types.Signer // Transaction signer to use for sender recovery signer types.Signer // Transaction signer to use for sender recovery
chain BlockChain // Chain object to access the state through chain BlockChain // Chain object to access the state through
@ -403,8 +449,14 @@ func New(config Config, chain BlockChain, hasPendingAuth func(common.Address) bo
lookup: newLookup(), lookup: newLookup(),
index: make(map[common.Address][]*blobTxMeta), index: make(map[common.Address][]*blobTxMeta),
spent: make(map[common.Address]*uint256.Int), spent: make(map[common.Address]*uint256.Int),
gapped: make(map[common.Address][]*types.Transaction), gapped: make(map[common.Address][]*pooledBlobTx),
gappedSource: make(map[common.Hash]common.Address), gappedSource: make(map[common.Hash]common.Address),
queue: make(map[common.Hash]*types.Transaction),
indexQueue: make(map[common.Address][]*blobTxMeta),
spentQueue: make(map[common.Address]*uint256.Int),
cellQueue: make(map[common.Hash][]kzg4844.Cell),
custodyQueue: make(map[common.Hash]*types.CustodyBitmap),
replacementQueue: make(map[common.Address]map[uint64]*blobTxMeta),
} }
} }
@ -461,9 +513,13 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
} }
// Index all transactions on disk and delete anything unprocessable // Index all transactions on disk and delete anything unprocessable
var fails []uint64 var fails []uint64
var convertTxs []*types.Transaction
index := func(id uint64, size uint32, blob []byte) { index := func(id uint64, size uint32, blob []byte) {
if p.parseTransaction(id, size, blob) != nil { if tx, err := p.parseTransaction(id, size, blob); err != nil {
fails = append(fails, id) fails = append(fails, id)
} else if tx != nil {
fails = append(fails, id)
convertTxs = append(convertTxs, tx)
} }
} }
store, err := billy.Open(billy.Options{Path: queuedir, Repair: true}, slotter, index) store, err := billy.Open(billy.Options{Path: queuedir, Repair: true}, slotter, index)
@ -472,6 +528,40 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
} }
p.store = store p.store = store
// Migrate legacy transactions (types.Transaction) to pooledBlobTx format.
// Legacy entries are deleted (via fails) and re-stored as pooledBlobTx.
if len(convertTxs) > 0 {
log.Trace("Convert plain transaction to pooled transaction", "len", len(convertTxs))
for _, tx := range convertTxs {
// Note that we skip errors here.
// Just like parseTransaction failure does not abort the blobpool creation,
// conversion process also cannot abort the entire process.
pooledTx, err := newPooledBlobTx(tx)
if err != nil {
log.Error("Failed to convert legacy tx to pooledBlobTx", "hash", tx.Hash(), "err", err)
continue
}
blob, err := rlp.EncodeToBytes(pooledTx)
if err != nil {
continue
}
id, err := p.store.Put(blob)
if err != nil {
continue
}
meta := newBlobTxMeta(id, pooledTx.Size, p.store.Size(id), pooledTx)
sender, err := types.Sender(p.signer, pooledTx.Transaction)
if err != nil {
fails = append(fails, id)
continue
}
if err := p.trackTransaction(meta, sender); err != nil {
fails = append(fails, id)
continue
}
}
}
if len(fails) > 0 { if len(fails) > 0 {
log.Warn("Dropping invalidated blob transactions", "ids", fails) log.Warn("Dropping invalidated blob transactions", "ids", fails)
dropInvalidMeter.Mark(int64(len(fails))) dropInvalidMeter.Mark(int64(len(fails)))
@ -558,36 +648,54 @@ func (p *BlobPool) Close() error {
// parseTransaction is a callback method on pool creation that gets called for // parseTransaction is a callback method on pool creation that gets called for
// each transaction on disk to create the in-memory metadata index. // each transaction on disk to create the in-memory metadata index.
// Announced state is not initialized here, it needs to be iniitalized seprately. // Announced state is not initialized here, it needs to be initialized separately.
func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { //
// If a legacy types.Transaction is found on disk, it is returned for migration
// in Init (the old ID will be deleted and a new pooledBlobTx written).
// If a pooledBlobTx is found, it is indexed directly and nil is returned.
func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) (*types.Transaction, error) {
tx := new(types.Transaction) tx := new(types.Transaction)
if err := rlp.DecodeBytes(blob, tx); err != nil { pooledTx := new(pooledBlobTx)
if err := rlp.DecodeBytes(blob, pooledTx); err != nil {
// This path is impossible unless the disk data representation changes // This path is impossible unless the disk data representation changes
// across restarts. For that ever improbable case, recover gracefully // across restarts. For that ever improbable case, recover gracefully
// by ignoring this data entry. // by ignoring this data entry.
if err := rlp.DecodeBytes(blob, tx); err != nil {
log.Error("Failed to decode blob pool entry", "id", id, "err", err) log.Error("Failed to decode blob pool entry", "id", id, "err", err)
return err return nil, err
} }
if tx.BlobTxSidecar() == nil { if tx.BlobTxSidecar() == nil {
log.Error("Missing sidecar in blob pool entry", "id", id, "hash", tx.Hash()) log.Error("Missing sidecar in blob pool entry", "id", id, "hash", tx.Hash())
return errors.New("missing blob sidecar") return nil, errors.New("missing blob sidecar")
} }
return tx, nil
}
if pooledTx.Sidecar == nil {
log.Error("Missing sidecar in blob pool entry", "id", id, "hash", pooledTx.Transaction.Hash())
return nil, errors.New("missing blob sidecar")
}
meta := newBlobTxMeta(id, pooledTx.Size, size, pooledTx)
meta := newBlobTxMeta(id, tx.Size(), size, tx) sender, err := types.Sender(p.signer, pooledTx.Transaction)
if p.lookup.exists(meta.hash) {
// This path is only possible after a crash, where deleted items are not
// removed via the normal shutdown-startup procedure and thus may get
// partially resurrected.
log.Error("Rejecting duplicate blob pool entry", "id", id, "hash", tx.Hash())
return errors.New("duplicate blob entry")
}
sender, err := types.Sender(p.signer, tx)
if err != nil { if err != nil {
// This path is impossible unless the signature validity changes across // This path is impossible unless the signature validity changes across
// restarts. For that ever improbable case, recover gracefully by ignoring // restarts. For that ever improbable case, recover gracefully by ignoring
// this data entry. // this data entry.
log.Error("Failed to recover blob tx sender", "id", id, "hash", tx.Hash(), "err", err) log.Error("Failed to recover blob tx sender", "id", id, "hash", pooledTx.Transaction.Hash(), "err", err)
return err return nil, err
}
return nil, p.trackTransaction(meta, sender)
}
// trackTransaction registers a transaction's metadata in the pool's indices.
func (p *BlobPool) trackTransaction(meta *blobTxMeta, sender common.Address) error {
if p.lookup.exists(meta.hash) {
// This path is only possible after a crash, where deleted items are not
// removed via the normal shutdown-startup procedure and thus may get
// partially resurrected.
log.Error("Rejecting duplicate blob pool entry", "id", meta.id, "hash", meta.hash)
return fmt.Errorf("duplicate blob entry %d, %s", meta.id, meta.hash)
} }
if _, ok := p.index[sender]; !ok { if _, ok := p.index[sender]; !ok {
if err := p.reserver.Hold(sender); err != nil { if err := p.reserver.Hold(sender); err != nil {
@ -863,17 +971,17 @@ func (p *BlobPool) offload(addr common.Address, nonce uint64, id uint64, inclusi
log.Error("Blobs missing for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err) log.Error("Blobs missing for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err)
return return
} }
var tx types.Transaction var pooledTx pooledBlobTx
if err = rlp.DecodeBytes(data, &tx); err != nil { if err = rlp.DecodeBytes(data, &pooledTx); err != nil {
log.Error("Blobs corrupted for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err) log.Error("Blobs corrupted for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err)
return return
} }
block, ok := inclusions[tx.Hash()] block, ok := inclusions[pooledTx.Transaction.Hash()]
if !ok { if !ok {
log.Warn("Blob transaction swapped out by signer", "from", addr, "nonce", nonce, "id", id) log.Warn("Blob transaction swapped out by signer", "from", addr, "nonce", nonce, "id", id)
return return
} }
if err := p.limbo.push(&tx, block); err != nil { if err := p.limbo.push(&pooledTx, block); err != nil {
log.Warn("Failed to offload blob tx into limbo", "err", err) log.Warn("Failed to offload blob tx into limbo", "err", err)
return return
} }
@ -951,13 +1059,13 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) {
log.Error("Blobs missing for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err) log.Error("Blobs missing for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err)
continue continue
} }
var tx types.Transaction var pooledTx pooledBlobTx
if err = rlp.DecodeBytes(data, &tx); err != nil { if err = rlp.DecodeBytes(data, &pooledTx); err != nil {
log.Error("Blobs corrupted for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err) log.Error("Blobs corrupted for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err)
continue continue
} }
announcable = append(announcable, tx.WithoutBlobTxSidecar()) announcable = append(announcable, pooledTx.Transaction)
log.Trace("Blob transaction now announcable", "from", addr, "nonce", meta.nonce, "id", meta.id, "hash", tx.Hash()) log.Trace("Blob transaction now announcable", "from", addr, "nonce", meta.nonce, "id", meta.id, "hash", pooledTx.Transaction.Hash())
} }
} }
} }
@ -1116,38 +1224,23 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error {
// TODO: seems like an easy optimization here would be getting the serialized tx // TODO: seems like an easy optimization here would be getting the serialized tx
// from limbo instead of re-serializing it here. // from limbo instead of re-serializing it here.
// Converts reorged-out legacy blob transactions to the new format to prevent
// them from becoming stuck in the pool until eviction.
//
// Performance note: Conversion takes ~140ms (Mac M1 Pro). Since a maximum of
// 9 legacy blob transactions are allowed in a block pre-Osaka, an adversary
// could theoretically halt a Geth node for ~1.2s by reorging per block. However,
// this attack is financially inefficient to execute.
head := p.head.Load()
if p.chain.Config().IsOsaka(head.Number, head.Time) && tx.BlobTxSidecar().Version == types.BlobSidecarVersion0 {
if err := tx.BlobTxSidecar().ToV1(); err != nil {
log.Error("Failed to convert the legacy sidecar", "err", err)
return err
}
log.Info("Legacy blob transaction is reorged", "hash", tx.Hash())
}
// Serialize the transaction back into the primary datastore. // Serialize the transaction back into the primary datastore.
blob, err := rlp.EncodeToBytes(tx) blob, err := rlp.EncodeToBytes(tx)
if err != nil { if err != nil {
log.Error("Failed to encode transaction for storage", "hash", tx.Hash(), "err", err) log.Error("Failed to encode transaction for storage", "hash", tx.Transaction.Hash(), "err", err)
return err return err
} }
id, err := p.store.Put(blob) id, err := p.store.Put(blob)
if err != nil { if err != nil {
log.Error("Failed to write transaction into storage", "hash", tx.Hash(), "err", err) log.Error("Failed to write transaction into storage", "hash", tx.Transaction.Hash(), "err", err)
return err return err
} }
// Update the indices and metrics // Update the indices and metrics
meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), tx) meta := newBlobTxMeta(id, tx.Size, p.store.Size(id), tx)
if _, ok := p.index[addr]; !ok { if _, ok := p.index[addr]; !ok {
if err := p.reserver.Hold(addr); err != nil { if err := p.reserver.Hold(addr); err != nil {
log.Warn("Failed to reserve account for blob pool", "tx", tx.Hash(), "from", addr, "err", err) log.Warn("Failed to reserve account for blob pool", "tx", tx.Transaction.Hash(), "from", addr, "err", err)
return err return err
} }
p.index[addr] = []*blobTxMeta{meta} p.index[addr] = []*blobTxMeta{meta}
@ -1276,12 +1369,40 @@ func (p *BlobPool) checkDelegationLimit(tx *types.Transaction) error {
return txpool.ErrInflightTxLimitReached return txpool.ErrInflightTxLimitReached
} }
// ValidateCells validates cells against transaction commitments and proofs.
func (p *BlobPool) ValidateCells(txs []common.Hash, cells [][]kzg4844.Cell, custody *types.CustodyBitmap) []error {
errs := make([]error, len(txs))
for i, tx := range txs {
if _, ok := p.queue[tx]; !ok {
errs[i] = fmt.Errorf("transaction %s not found", tx)
continue
}
sidecar := p.queue[tx].BlobTxSidecar()
cellProofs := make([]kzg4844.Proof, 0)
for _, proofIdx := range custody.Indices() {
// should store all proofs
for blobIdx := range len(sidecar.Commitments) {
idx := blobIdx*kzg4844.CellProofsPerBlob + int(proofIdx)
cellProofs = append(cellProofs, sidecar.Proofs[idx])
}
}
errs[i] = kzg4844.VerifyCells(cells[i], sidecar.Commitments, cellProofs, custody.Indices())
}
return errs
}
// validateTx checks whether a transaction is valid according to the consensus // validateTx checks whether a transaction is valid according to the consensus
// rules and adheres to some heuristic limits of the local node (price and size). // rules and adheres to some heuristic limits of the local node (price and size).
//
// This function assumes the static validation has been performed already and // This function assumes the static validation has been performed already and
// only runs the stateful checks with lock protection. // only runs the stateful checks with lock protection.
func (p *BlobPool) validateTx(tx *types.Transaction) error { // If buffer field is set to true, consider txs in the queue as well.
// This is to prevent fetching cells of invalid transactions, which would be expensive.
func (p *BlobPool) validateTx(tx *types.Transaction, buffer bool) error {
if err := p.ValidateTxBasics(tx); err != nil {
return err
}
// Ensure the transaction adheres to the stateful pool filters (nonce, balance) // Ensure the transaction adheres to the stateful pool filters (nonce, balance)
stateOpts := &txpool.ValidationOptionsWithState{ stateOpts := &txpool.ValidationOptionsWithState{
State: p.state, State: p.state,
@ -1292,23 +1413,61 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error {
// then handle the error by adding to the buffer. The first gap will // then handle the error by adding to the buffer. The first gap will
// be the next nonce shifted by however many transactions we already // be the next nonce shifted by however many transactions we already
// have pooled. // have pooled.
return p.state.GetNonce(addr) + uint64(len(p.index[addr])) result := p.state.GetNonce(addr) + uint64(len(p.index[addr]))
if buffer {
return result + uint64(len(p.indexQueue[addr]))
}
return result
}, },
UsedAndLeftSlots: func(addr common.Address) (int, int) { UsedAndLeftSlots: func(addr common.Address) (int, int) {
have := len(p.index[addr]) have := len(p.index[addr])
if buffer {
have += len(p.indexQueue[addr])
}
if have >= maxTxsPerAccount { if have >= maxTxsPerAccount {
return have, 0 return have, 0
} }
return have, maxTxsPerAccount - have return have, maxTxsPerAccount - have
}, },
ExistingExpenditure: func(addr common.Address) *big.Int { ExistingExpenditure: func(addr common.Address) *big.Int {
result := new(big.Int)
if spent := p.spent[addr]; spent != nil { if spent := p.spent[addr]; spent != nil {
return spent.ToBig() result.Add(result, spent.ToBig())
} }
return new(big.Int)
// calculate expenditure after replacements
if buffer {
if replacements := p.replacementQueue[addr]; replacements != nil {
next := p.state.GetNonce(addr)
for nonce, replacement := range replacements {
if len(p.index[addr]) > int(nonce-next) {
// replacement
originalCost := p.index[addr][nonce-next].costCap
replacementCost := replacement.costCap
result.Add(result, new(uint256.Int).Sub(replacementCost, originalCost).ToBig())
}
}
}
if spentQueue := p.spentQueue[addr]; spentQueue != nil {
result.Add(result, spentQueue.ToBig())
}
}
return result
}, },
ExistingCost: func(addr common.Address, nonce uint64) *big.Int { ExistingCost: func(addr common.Address, nonce uint64) *big.Int {
next := p.state.GetNonce(addr) next := p.state.GetNonce(addr)
if buffer {
if p.replacementQueue[addr] != nil && p.replacementQueue[addr][nonce] != nil {
return p.replacementQueue[addr][nonce].costCap.ToBig()
}
if uint64(len(p.indexQueue[addr])) > nonce-next-uint64(len(p.index[addr])) {
return p.indexQueue[addr][nonce-next-uint64(len(p.index[addr]))].costCap.ToBig()
}
}
if uint64(len(p.index[addr])) > nonce-next { if uint64(len(p.index[addr])) > nonce-next {
return p.index[addr][int(nonce-next)].costCap.ToBig() return p.index[addr][int(nonce-next)].costCap.ToBig()
} }
@ -1327,8 +1486,29 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error {
from, _ = types.Sender(p.signer, tx) // already validated above from, _ = types.Sender(p.signer, tx) // already validated above
next = p.state.GetNonce(from) next = p.state.GetNonce(from)
) )
if uint64(len(p.index[from])) > tx.Nonce()-next { var prev *blobTxMeta
prev := p.index[from][int(tx.Nonce()-next)] nonce := tx.Nonce()
if nonce < next+uint64(len(p.index[from])) {
// pooled tx
prev = p.index[from][nonce-next]
// check replacement if it is buffer tx validation
if buffer && p.replacementQueue[from] != nil {
if replacement := p.replacementQueue[from][nonce]; replacement != nil {
prev = replacement
}
}
} else if buffer {
offset := nonce - next - uint64(len(p.index[from]))
if uint64(len(p.indexQueue[from])) > offset && offset > 0 {
// buffer tx replacement
prev = p.indexQueue[from][nonce-next-uint64(len(p.index[from]))]
}
}
if prev == nil {
return nil
}
// Ensure the transaction is different than the one tracked locally // Ensure the transaction is different than the one tracked locally
if prev.hash == tx.Hash() { if prev.hash == tx.Hash() {
return txpool.ErrAlreadyKnown return txpool.ErrAlreadyKnown
@ -1342,6 +1522,7 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error {
case tx.BlobGasFeeCapIntCmp(prev.blobFeeCap.ToBig()) <= 0: case tx.BlobGasFeeCapIntCmp(prev.blobFeeCap.ToBig()) <= 0:
return fmt.Errorf("%w: new tx blob gas fee cap %v <= %v queued", txpool.ErrReplaceUnderpriced, tx.BlobGasFeeCap(), prev.blobFeeCap) return fmt.Errorf("%w: new tx blob gas fee cap %v <= %v queued", txpool.ErrReplaceUnderpriced, tx.BlobGasFeeCap(), prev.blobFeeCap)
} }
var ( var (
multiplier = uint256.NewInt(100 + p.config.PriceBump) multiplier = uint256.NewInt(100 + p.config.PriceBump)
onehundred = uint256.NewInt(100) onehundred = uint256.NewInt(100)
@ -1358,7 +1539,6 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error {
case tx.BlobGasFeeCapIntCmp(minBlobGasFeeCap.ToBig()) < 0: case tx.BlobGasFeeCapIntCmp(minBlobGasFeeCap.ToBig()) < 0:
return fmt.Errorf("%w: new tx blob gas fee cap %v < %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.BlobGasFeeCap(), prev.blobFeeCap, p.config.PriceBump) return fmt.Errorf("%w: new tx blob gas fee cap %v < %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.BlobGasFeeCap(), prev.blobFeeCap, p.config.PriceBump)
} }
}
return nil return nil
} }
@ -1368,11 +1548,16 @@ func (p *BlobPool) Has(hash common.Hash) bool {
p.lock.RLock() p.lock.RLock()
defer p.lock.RUnlock() defer p.lock.RUnlock()
if p.lookup.exists(hash) || p.queue[hash] != nil {
return true
}
poolHas := p.lookup.exists(hash) poolHas := p.lookup.exists(hash)
_, gapped := p.gappedSource[hash] _, gapped := p.gappedSource[hash]
return poolHas || gapped return poolHas || gapped
} }
// getRLP returns the raw RLP-encoded pooledBlobTx data from the store.
func (p *BlobPool) getRLP(hash common.Hash) []byte { func (p *BlobPool) getRLP(hash common.Hash) []byte {
// Track the amount of time waiting to retrieve a fully resolved blob tx from // Track the amount of time waiting to retrieve a fully resolved blob tx from
// the pool and the amount of time actually spent on pulling the data from disk. // the pool and the amount of time actually spent on pulling the data from disk.
@ -1404,20 +1589,48 @@ func (p *BlobPool) Get(hash common.Hash) *types.Transaction {
if len(data) == 0 { if len(data) == 0 {
return nil return nil
} }
item := new(types.Transaction) var pooledTx pooledBlobTx
if err := rlp.DecodeBytes(data, item); err != nil { if err := rlp.DecodeBytes(data, &pooledTx); err != nil {
id, _ := p.lookup.storeidOfTx(hash) log.Error("Blobs corrupted for traced transaction", "hash", hash, "err", err)
log.Error("Blobs corrupted for traced transaction",
"hash", hash, "id", id, "err", err)
return nil return nil
} }
return item tx, err := pooledTx.convert()
if err != nil {
log.Error("Failed to convert transaction in blobpool", "hash", hash, "err", err)
return nil
}
return tx
} }
// GetRLP returns a RLP-encoded transaction if it is contained in the pool. // GetRLP returns an RLP-encoded transaction if it is contained in the pool.
func (p *BlobPool) GetRLP(hash common.Hash) []byte { // TODO: The pool internally stores pooledBlobTx (cell sidecar format), but callers expect
return p.getRLP(hash) // types.Transaction RLP. This requires an additional decode-encode step, which is inefficient
// and contradicts the original purpose of this function.
// Possible improvements: Drop eth70 and store the cell and transaction separately.
func (p *BlobPool) GetRLP(hash common.Hash, includeBlob bool) []byte {
data := p.getRLP(hash)
if len(data) == 0 {
return nil
}
var pooledTx pooledBlobTx
if err := rlp.DecodeBytes(data, &pooledTx); err != nil {
log.Error("Failed to decode transaction in blobpool", "hash", hash, "err", err)
return nil
}
tx, err := pooledTx.convert()
if err != nil {
log.Error("Failed to convert transaction in blobpool", "hash", hash, "err", err)
return nil
}
if !includeBlob {
tx = tx.WithoutBlob()
}
encoded, err := rlp.EncodeToBytes(tx)
if err != nil {
log.Error("Failed to encode transaction in blobpool", "hash", hash, "err", err)
return nil
}
return encoded
} }
// GetMetadata returns the transaction type and transaction size with the // GetMetadata returns the transaction type and transaction size with the
@ -1486,11 +1699,15 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte) ([]*kzg4844.Blo
} }
// Decode the blob transaction // Decode the blob transaction
tx := new(types.Transaction) var pooledTx pooledBlobTx
if err := rlp.DecodeBytes(data, tx); err != nil { if err := rlp.DecodeBytes(data, &pooledTx); err != nil {
log.Error("Blobs corrupted for traced transaction", "id", txID, "err", err) log.Error("Blobs corrupted for traced transaction", "id", txID, "err", err)
continue continue
} }
tx, err := pooledTx.convert()
if err != nil {
return nil, nil, nil, err
}
sidecar := tx.BlobTxSidecar() sidecar := tx.BlobTxSidecar()
if sidecar == nil { if sidecar == nil {
log.Error("Blob tx without sidecar", "hash", tx.Hash(), "id", txID) log.Error("Blob tx without sidecar", "hash", tx.Hash(), "id", txID)
@ -1554,14 +1771,115 @@ func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error {
if errs[i] = p.ValidateTxBasics(tx); errs[i] != nil { if errs[i] = p.ValidateTxBasics(tx); errs[i] != nil {
continue continue
} }
errs[i] = p.add(tx) if len(tx.BlobTxSidecar().Blobs) != 0 {
// from user: convert to pooledBlobTx and add
pooledTx, err := newPooledBlobTx(tx)
if err != nil {
errs[i] = err
continue
}
errs[i] = p.add(pooledTx)
} else {
// from p2p, buffer until the corresponding cells arrive
errs[i] = p.addBuffer(tx)
}
} }
return errs return errs
} }
func (p *BlobPool) addBuffer(tx *types.Transaction) (err error) {
p.lock.Lock()
defer p.lock.Unlock()
if cells, ok := p.cellQueue[tx.Hash()]; ok {
sidecar := tx.BlobTxSidecar()
var cellSidecar types.BlobTxCellSidecar
if len(cells) >= kzg4844.DataPerBlob {
blob, err := kzg4844.RecoverBlobs(cells, p.custodyQueue[tx.Hash()].Indices())
if err != nil {
return err
}
extendedCells, err := kzg4844.ComputeCells(blob)
if err != nil {
return err
}
cellSidecar = types.BlobTxCellSidecar{
Version: sidecar.Version,
Cells: extendedCells,
Commitments: sidecar.Commitments,
Proofs: sidecar.Proofs,
Custody: *types.CustodyBitmapAll,
}
} else {
cellSidecar = types.BlobTxCellSidecar{
Version: sidecar.Version,
Cells: cells,
Commitments: sidecar.Commitments,
Proofs: sidecar.Proofs,
Custody: *p.custodyQueue[tx.Hash()],
}
}
err := p.addLocked(&pooledBlobTx{Transaction: tx.WithoutBlobTxSidecar(), Sidecar: &cellSidecar, Size: tx.Size()}, true)
if err == nil {
delete(p.cellQueue, tx.Hash())
delete(p.custodyQueue, tx.Hash())
}
return err
}
if err := p.validateTx(tx, true); err != nil {
return err
}
p.queue[tx.Hash()] = tx
from, _ := types.Sender(p.signer, tx)
next := p.state.GetNonce(from)
nonce := tx.Nonce()
pooledCount := uint64(len(p.index[from]))
meta := newBlobTxMeta(0, tx.Size(), 0, &pooledBlobTx{Transaction: tx, Size: tx.Size()})
if nonce < next+pooledCount {
// Pooled transaction replacements are stored in replacementQueue for expenditure validation
// for future transactions from the same account. This overestimates expenditure considering
// that replacement transaction payload fetch may fail and the tx can be dropped.
// However, this conservative approach prevents transactions that passed validation when
// entering the buffer from failing expenditure validation due to transaction replacements.
if p.replacementQueue[from] == nil {
p.replacementQueue[from] = make(map[uint64]*blobTxMeta)
}
if existingReplacement := p.replacementQueue[from][nonce]; existingReplacement != nil {
delete(p.queue, existingReplacement.hash)
}
p.replacementQueue[from][nonce] = meta
} else {
if p.spentQueue[from] == nil {
p.spentQueue[from] = new(uint256.Int)
}
bufferOffset := int(nonce - (next + pooledCount))
if len(p.indexQueue[from]) > bufferOffset {
// Replace buffer transaction
prev := p.indexQueue[from][bufferOffset]
delete(p.queue, prev.hash)
p.indexQueue[from][bufferOffset] = meta
p.spentQueue[from] = new(uint256.Int).Sub(p.spentQueue[from], prev.costCap)
p.spentQueue[from] = new(uint256.Int).Add(p.spentQueue[from], meta.costCap)
dropReplacedMeter.Mark(1)
} else {
p.indexQueue[from] = append(p.indexQueue[from], meta)
p.spentQueue[from] = new(uint256.Int).Add(p.spentQueue[from], meta.costCap)
}
}
return nil
}
// add inserts a new blob transaction into the pool if it passes validation (both // add inserts a new blob transaction into the pool if it passes validation (both
// consensus validity and pool restrictions). // consensus validity and pool restrictions).
func (p *BlobPool) add(tx *types.Transaction) (err error) { func (p *BlobPool) add(pooledTx *pooledBlobTx) (err error) {
// The blob pool blocks on adding a transaction. This is because blob txs are // The blob pool blocks on adding a transaction. This is because blob txs are
// only even pulled from the network, so this method will act as the overload // only even pulled from the network, so this method will act as the overload
// protection for fetches. // protection for fetches.
@ -1574,15 +1892,18 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
addtimeHist.Update(time.Since(start).Nanoseconds()) addtimeHist.Update(time.Since(start).Nanoseconds())
}(time.Now()) }(time.Now())
return p.addLocked(tx, true) return p.addLocked(pooledTx, true)
} }
// addLocked inserts a new blob transaction into the pool if it passes validation (both // addLocked inserts a new blob transaction into the pool if it passes validation (both
// consensus validity and pool restrictions). It must be called with the pool lock held. // consensus validity and pool restrictions). It must be called with the pool lock held.
// Only for internal use. // Only for internal use.
func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error) { func (p *BlobPool) addLocked(pooledTx *pooledBlobTx, checkGapped bool) (err error) {
tx := pooledTx.Transaction
cellSidecar := pooledTx.Sidecar
// Ensure the transaction is valid from all perspectives // Ensure the transaction is valid from all perspectives
if err := p.validateTx(tx); err != nil { if err := p.validateTx(tx, false); err != nil {
log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err) log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err)
switch { switch {
case errors.Is(err, txpool.ErrUnderpriced): case errors.Is(err, txpool.ErrUnderpriced):
@ -1593,11 +1914,11 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
addStaleMeter.Mark(1) addStaleMeter.Mark(1)
case errors.Is(err, core.ErrNonceTooHigh): case errors.Is(err, core.ErrNonceTooHigh):
addGappedMeter.Mark(1) addGappedMeter.Mark(1)
// Store the tx in memory, and revalidate later // Store the tx in memory as pooledBlobTx, and revalidate later
from, _ := types.Sender(p.signer, tx) from, _ := types.Sender(p.signer, tx)
allowance := p.gappedAllowance(from) allowance := p.gappedAllowance(from)
if allowance >= 1 && len(p.gapped) < maxGapped { if allowance >= 1 && len(p.gapped) < maxGapped {
p.gapped[from] = append(p.gapped[from], tx) p.gapped[from] = append(p.gapped[from], pooledTx)
p.gappedSource[tx.Hash()] = from p.gappedSource[tx.Hash()] = from
log.Trace("added tx to gapped blob queue", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from])) log.Trace("added tx to gapped blob queue", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
return nil return nil
@ -1619,6 +1940,13 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
} }
return err return err
} }
if err := txpool.ValidateBlobSidecar(tx, cellSidecar, p.head.Load(), &txpool.ValidationOptions{
Config: p.chain.Config(),
MaxBlobCount: maxBlobsPerTx,
}); err != nil {
log.Trace("Sidecar validation failed", "hash", tx.Hash(), "err", err)
return err
}
// If the address is not yet known, request exclusivity to track the account // If the address is not yet known, request exclusivity to track the account
// only by this subpool until all transactions are evicted // only by this subpool until all transactions are evicted
from, _ := types.Sender(p.signer, tx) // already validated above from, _ := types.Sender(p.signer, tx) // already validated above
@ -1627,6 +1955,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
addNonExclusiveMeter.Mark(1) addNonExclusiveMeter.Mark(1)
return err return err
} }
//todo release when evicted from the buffer
defer func() { defer func() {
// If the transaction is rejected by some post-validation check, remove // If the transaction is rejected by some post-validation check, remove
// the lock on the reservation set. // the lock on the reservation set.
@ -1641,7 +1970,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
} }
// Transaction permitted into the pool from a nonce and cost perspective, // Transaction permitted into the pool from a nonce and cost perspective,
// insert it into the database and update the indices // insert it into the database and update the indices
blob, err := rlp.EncodeToBytes(tx) blob, err := rlp.EncodeToBytes(pooledTx)
if err != nil { if err != nil {
log.Error("Failed to encode transaction for storage", "hash", tx.Hash(), "err", err) log.Error("Failed to encode transaction for storage", "hash", tx.Hash(), "err", err)
return err return err
@ -1650,7 +1979,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
if err != nil { if err != nil {
return err return err
} }
meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), tx) meta := newBlobTxMeta(id, pooledTx.Size, p.store.Size(id), pooledTx)
var ( var (
next = p.state.GetNonce(from) next = p.state.GetNonce(from)
@ -1761,13 +2090,13 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
// We have to add in nonce order, but we want to stable sort to cater for situations // We have to add in nonce order, but we want to stable sort to cater for situations
// where transactions are replaced, keeping the original receive order for same nonce // where transactions are replaced, keeping the original receive order for same nonce
sort.SliceStable(gtxs, func(i, j int) bool { sort.SliceStable(gtxs, func(i, j int) bool {
return gtxs[i].Nonce() < gtxs[j].Nonce() return gtxs[i].Transaction.Nonce() < gtxs[j].Transaction.Nonce()
}) })
for len(gtxs) > 0 { for len(gtxs) > 0 {
stateNonce := p.state.GetNonce(from) stateNonce := p.state.GetNonce(from)
firstgap := stateNonce + uint64(len(p.index[from])) firstgap := stateNonce + uint64(len(p.index[from]))
if gtxs[0].Nonce() > firstgap { if gtxs[0].Transaction.Nonce() > firstgap {
// Anything beyond the first gap is not addable yet // Anything beyond the first gap is not addable yet
break break
} }
@ -1775,25 +2104,25 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
// Drop any buffered transactions that became stale in the meantime (included in chain or replaced) // Drop any buffered transactions that became stale in the meantime (included in chain or replaced)
// If we arrive to the transaction in the pending range (between the state Nonce and first gap, we // If we arrive to the transaction in the pending range (between the state Nonce and first gap, we
// try to add them now while removing from here. // try to add them now while removing from here.
tx := gtxs[0] ptx := gtxs[0]
gtxs[0] = nil gtxs[0] = nil
gtxs = gtxs[1:] gtxs = gtxs[1:]
delete(p.gappedSource, tx.Hash()) delete(p.gappedSource, ptx.Transaction.Hash())
if tx.Nonce() < stateNonce { if ptx.Transaction.Nonce() < stateNonce {
// Stale, drop it. Eventually we could add to limbo here if hash matches. // Stale, drop it. Eventually we could add to limbo here if hash matches.
log.Trace("Gapped blob transaction became stale", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from])) log.Trace("Gapped blob transaction became stale", "hash", ptx.Transaction.Hash(), "from", from, "nonce", ptx.Transaction.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from]))
continue continue
} }
if tx.Nonce() <= firstgap { if ptx.Transaction.Nonce() <= firstgap {
// If we hit the pending range, including the first gap, add it and continue to try to add more. // If we hit the pending range, including the first gap, add it and continue to try to add more.
// We do not recurse here, but continue to loop instead. // We do not recurse here, but continue to loop instead.
// We are under lock, so we can add the transaction directly. // We are under lock, so we can add the transaction directly.
if err := p.addLocked(tx, false); err == nil { if err := p.addLocked(ptx, false); err == nil {
log.Trace("Gapped blob transaction added to pool", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from])) log.Trace("Gapped blob transaction added to pool", "hash", ptx.Transaction.Hash(), "from", from, "nonce", ptx.Transaction.Nonce(), "qlen", len(p.gapped[from]))
} else { } else {
log.Trace("Gapped blob transaction not accepted", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "err", err) log.Trace("Gapped blob transaction not accepted", "hash", ptx.Transaction.Hash(), "from", from, "nonce", ptx.Transaction.Nonce(), "err", err)
} }
} }
} }
@ -2060,10 +2389,10 @@ func (p *BlobPool) evictGapped() {
// and we overwrite the slice for this account after filtering. // and we overwrite the slice for this account after filtering.
keep := txs[:0] keep := txs[:0]
for i, gtx := range txs { for i, gtx := range txs {
if gtx.Time().Before(cutoff) || gtx.Nonce() < nonce { if gtx.Transaction.Time().Before(cutoff) || gtx.Transaction.Nonce() < nonce {
// Evict old or stale transactions // Evict old or stale transactions
// Should we add stale to limbo here if it would belong? // Should we add stale to limbo here if it would belong?
delete(p.gappedSource, gtx.Hash()) delete(p.gappedSource, gtx.Transaction.Hash())
txs[i] = nil // Explicitly nil out evicted element txs[i] = nil // Explicitly nil out evicted element
} else { } else {
keep = append(keep, gtx) keep = append(keep, gtx)
@ -2126,6 +2455,9 @@ func (p *BlobPool) Status(hash common.Hash) txpool.TxStatus {
if p.lookup.exists(hash) { if p.lookup.exists(hash) {
return txpool.TxStatusPending return txpool.TxStatusPending
} }
if _, ok := p.queue[hash]; ok {
return txpool.TxStatusQueued
}
if _, gapped := p.gappedSource[hash]; gapped { if _, gapped := p.gappedSource[hash]; gapped {
return txpool.TxStatusQueued return txpool.TxStatusQueued
} }
@ -2180,7 +2512,7 @@ func (p *BlobPool) Clear() {
// Reset counters and the gapped buffer // Reset counters and the gapped buffer
p.stored = 0 p.stored = 0
p.gapped = make(map[common.Address][]*types.Transaction) p.gapped = make(map[common.Address][]*pooledBlobTx)
p.gappedSource = make(map[common.Hash]common.Address) p.gappedSource = make(map[common.Hash]common.Address)
var ( var (
@ -2189,3 +2521,119 @@ func (p *BlobPool) Clear() {
) )
p.evict = newPriceHeap(basefee, blobfee, p.index) p.evict = newPriceHeap(basefee, blobfee, p.index)
} }
// GetCustody returns the custody bitmap for a given transaction hash.
func (p *BlobPool) GetCustody(hash common.Hash) *types.CustodyBitmap {
p.lock.RLock()
defer p.lock.RUnlock()
if meta := p.lookup.txIndex[hash]; meta != nil {
return &meta.custody
}
return nil
}
// GetCells returns the cells matching the given custody bitmap for a transaction.
func (p *BlobPool) GetCells(hash common.Hash, mask types.CustodyBitmap) ([]kzg4844.Cell, error) {
p.lock.RLock()
defer p.lock.RUnlock()
id, ok := p.lookup.storeidOfTx(hash)
if !ok {
return nil, errors.New("requested cells don't exist")
}
data, err := p.store.Get(id)
if err != nil {
return nil, errors.New("tracked blob transaction missing from store")
}
// Decode the blob transaction
var pooledTx pooledBlobTx
if err := rlp.DecodeBytes(data, &pooledTx); err != nil {
return nil, errors.New("blobs corrupted for traced transaction")
}
tx := pooledTx.Transaction
sidecar := pooledTx.Sidecar
cells := make([]kzg4844.Cell, 0, mask.OneCount()*len(tx.BlobHashes()))
for cellIdx, custodyIdx := range sidecar.Custody.Indices() {
if mask.IsSet(custodyIdx) {
for blobIdx := 0; blobIdx < len(tx.BlobHashes()); blobIdx++ {
idx := blobIdx*sidecar.Custody.OneCount() + cellIdx
cells = append(cells, sidecar.Cells[idx])
}
}
}
if len(cells) != mask.OneCount()*len(tx.BlobHashes()) {
return nil, fmt.Errorf("not enough cells: tx %s, needed %d, have %d", tx.Hash(), len(tx.BlobHashes())*mask.OneCount(), len(cells))
}
return cells, nil
}
// AddPayload adds cell payloads for blob transactions.
func (p *BlobPool) AddPayload(txs []common.Hash, cells [][]kzg4844.Cell, custody *types.CustodyBitmap) []error {
p.lock.Lock()
defer p.lock.Unlock()
errs := make([]error, len(txs))
for i, hash := range txs {
if _, ok := p.queue[hash]; !ok {
p.cellQueue[hash] = cells[i]
p.custodyQueue[hash] = custody
continue
}
sidecar := p.queue[hash].BlobTxSidecar()
var cellSidecar types.BlobTxCellSidecar
if len(cells[i]) >= kzg4844.DataPerBlob {
blob, err := kzg4844.RecoverBlobs(cells[i], custody.Indices())
if err != nil {
errs[i] = err
continue
}
extendedCells, err := kzg4844.ComputeCells(blob)
if err != nil {
errs[i] = err
continue
}
cellSidecar = types.BlobTxCellSidecar{
Version: sidecar.Version,
Cells: extendedCells,
Commitments: sidecar.Commitments,
Proofs: sidecar.Proofs,
Custody: *types.CustodyBitmapAll,
}
} else {
cellSidecar = types.BlobTxCellSidecar{
Version: sidecar.Version,
Cells: cells[i],
Commitments: sidecar.Commitments,
Proofs: sidecar.Proofs,
Custody: *custody,
}
}
errs[i] = p.addLocked(&pooledBlobTx{Transaction: p.queue[hash].WithoutBlobTxSidecar(), Sidecar: &cellSidecar, Size: p.queue[hash].Size()}, true)
// todo nonce gap
// clean up queues
tx := p.queue[hash]
delete(p.queue, hash)
from, _ := types.Sender(p.signer, tx)
nonce := tx.Nonce()
next := p.state.GetNonce(from)
if p.replacementQueue[from] != nil {
delete(p.replacementQueue[from], nonce)
if len(p.replacementQueue[from]) == 0 {
delete(p.replacementQueue, from)
}
continue
}
// plain tx
offset := int(nonce - next - uint64(len(p.index[from])))
if offset > 0 && offset < len(p.indexQueue[from]) {
removed := p.indexQueue[from][offset]
p.indexQueue[from] = append(p.indexQueue[from][:offset], p.indexQueue[from][offset+1:]...)
p.spentQueue[from] = new(uint256.Int).Sub(p.spentQueue[from], removed.costCap)
}
}
return errs
}

View file

@ -513,75 +513,76 @@ func TestOpenDrops(t *testing.T) {
S: new(uint256.Int), S: new(uint256.Int),
}) })
blob, _ := rlp.EncodeToBytes(tx) blob, _ := rlp.EncodeToBytes(tx)
badsig, _ := store.Put(blob) badsig := tx.Hash()
store.Put(blob)
// Insert a sequence of transactions with a nonce gap in between to verify // Insert a sequence of transactions with a nonce gap in between to verify
// that anything gapped will get evicted (case 3). // that anything gapped will get evicted (case 3).
var ( var (
gapper, _ = crypto.GenerateKey() gapper, _ = crypto.GenerateKey()
valids = make(map[uint64]struct{}) valids = make(map[common.Hash]struct{})
gapped = make(map[uint64]struct{}) gapped = make(map[common.Hash]struct{})
) )
for _, nonce := range []uint64{0, 1, 3, 4, 6, 7} { // first gap at #2, another at #5 for _, nonce := range []uint64{0, 1, 3, 4, 6, 7} { // first gap at #2, another at #5
tx := makeTx(nonce, 1, 1, 1, gapper) tx := makeTx(nonce, 1, 1, 1, gapper)
blob, _ := rlp.EncodeToBytes(tx) blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob) store.Put(blob)
if nonce < 2 { if nonce < 2 {
valids[id] = struct{}{} valids[tx.Hash()] = struct{}{}
} else { } else {
gapped[id] = struct{}{} gapped[tx.Hash()] = struct{}{}
} }
} }
// Insert a sequence of transactions with a gapped starting nonce to verify // Insert a sequence of transactions with a gapped starting nonce to verify
// that the entire set will get dropped (case 3). // that the entire set will get dropped (case 3).
var ( var (
dangler, _ = crypto.GenerateKey() dangler, _ = crypto.GenerateKey()
dangling = make(map[uint64]struct{}) dangling = make(map[common.Hash]struct{})
) )
for _, nonce := range []uint64{1, 2, 3} { // first gap at #0, all set dangling for _, nonce := range []uint64{1, 2, 3} { // first gap at #0, all set dangling
tx := makeTx(nonce, 1, 1, 1, dangler) tx := makeTx(nonce, 1, 1, 1, dangler)
blob, _ := rlp.EncodeToBytes(tx) blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob) store.Put(blob)
dangling[id] = struct{}{} dangling[tx.Hash()] = struct{}{}
} }
// Insert a sequence of transactions with already passed nonces to veirfy // Insert a sequence of transactions with already passed nonces to veirfy
// that the entire set will get dropped (case 4). // that the entire set will get dropped (case 4).
var ( var (
filler, _ = crypto.GenerateKey() filler, _ = crypto.GenerateKey()
filled = make(map[uint64]struct{}) filled = make(map[common.Hash]struct{})
) )
for _, nonce := range []uint64{0, 1, 2} { // account nonce at 3, all set filled for _, nonce := range []uint64{0, 1, 2} { // account nonce at 3, all set filled
tx := makeTx(nonce, 1, 1, 1, filler) tx := makeTx(nonce, 1, 1, 1, filler)
blob, _ := rlp.EncodeToBytes(tx) blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob) store.Put(blob)
filled[id] = struct{}{} filled[tx.Hash()] = struct{}{}
} }
// Insert a sequence of transactions with partially passed nonces to verify // Insert a sequence of transactions with partially passed nonces to verify
// that the included part of the set will get dropped (case 4). // that the included part of the set will get dropped (case 4).
var ( var (
overlapper, _ = crypto.GenerateKey() overlapper, _ = crypto.GenerateKey()
overlapped = make(map[uint64]struct{}) overlapped = make(map[common.Hash]struct{})
) )
for _, nonce := range []uint64{0, 1, 2, 3} { // account nonce at 2, half filled for _, nonce := range []uint64{0, 1, 2, 3} { // account nonce at 2, half filled
tx := makeTx(nonce, 1, 1, 1, overlapper) tx := makeTx(nonce, 1, 1, 1, overlapper)
blob, _ := rlp.EncodeToBytes(tx) blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob) store.Put(blob)
if nonce >= 2 { if nonce >= 2 {
valids[id] = struct{}{} valids[tx.Hash()] = struct{}{}
} else { } else {
overlapped[id] = struct{}{} overlapped[tx.Hash()] = struct{}{}
} }
} }
// Insert a sequence of transactions with an underpriced first to verify that // Insert a sequence of transactions with an underpriced first to verify that
// the entire set will get dropped (case 5). // the entire set will get dropped (case 5).
var ( var (
underpayer, _ = crypto.GenerateKey() underpayer, _ = crypto.GenerateKey()
underpaid = make(map[uint64]struct{}) underpaid = make(map[common.Hash]struct{})
) )
for i := 0; i < 5; i++ { // make #0 underpriced for i := 0; i < 5; i++ { // make #0 underpriced
var tx *types.Transaction var tx *types.Transaction
@ -592,15 +593,15 @@ func TestOpenDrops(t *testing.T) {
} }
blob, _ := rlp.EncodeToBytes(tx) blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob) store.Put(blob)
underpaid[id] = struct{}{} underpaid[tx.Hash()] = struct{}{}
} }
// Insert a sequence of transactions with an underpriced in between to verify // Insert a sequence of transactions with an underpriced in between to verify
// that it and anything newly gapped will get evicted (case 5). // that it and anything newly gapped will get evicted (case 5).
var ( var (
outpricer, _ = crypto.GenerateKey() outpricer, _ = crypto.GenerateKey()
outpriced = make(map[uint64]struct{}) outpriced = make(map[common.Hash]struct{})
) )
for i := 0; i < 5; i++ { // make #2 underpriced for i := 0; i < 5; i++ { // make #2 underpriced
var tx *types.Transaction var tx *types.Transaction
@ -611,18 +612,18 @@ func TestOpenDrops(t *testing.T) {
} }
blob, _ := rlp.EncodeToBytes(tx) blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob) store.Put(blob)
if i < 2 { if i < 2 {
valids[id] = struct{}{} valids[tx.Hash()] = struct{}{}
} else { } else {
outpriced[id] = struct{}{} outpriced[tx.Hash()] = struct{}{}
} }
} }
// Insert a sequence of transactions fully overdrafted to verify that the // Insert a sequence of transactions fully overdrafted to verify that the
// entire set will get invalidated (case 6). // entire set will get invalidated (case 6).
var ( var (
exceeder, _ = crypto.GenerateKey() exceeder, _ = crypto.GenerateKey()
exceeded = make(map[uint64]struct{}) exceeded = make(map[common.Hash]struct{})
) )
for _, nonce := range []uint64{0, 1, 2} { // nonce 0 overdrafts the account for _, nonce := range []uint64{0, 1, 2} { // nonce 0 overdrafts the account
var tx *types.Transaction var tx *types.Transaction
@ -633,14 +634,14 @@ func TestOpenDrops(t *testing.T) {
} }
blob, _ := rlp.EncodeToBytes(tx) blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob) store.Put(blob)
exceeded[id] = struct{}{} exceeded[tx.Hash()] = struct{}{}
} }
// Insert a sequence of transactions partially overdrafted to verify that part // Insert a sequence of transactions partially overdrafted to verify that part
// of the set will get invalidated (case 6). // of the set will get invalidated (case 6).
var ( var (
overdrafter, _ = crypto.GenerateKey() overdrafter, _ = crypto.GenerateKey()
overdrafted = make(map[uint64]struct{}) overdrafted = make(map[common.Hash]struct{})
) )
for _, nonce := range []uint64{0, 1, 2} { // nonce 1 overdrafts the account for _, nonce := range []uint64{0, 1, 2} { // nonce 1 overdrafts the account
var tx *types.Transaction var tx *types.Transaction
@ -651,44 +652,46 @@ func TestOpenDrops(t *testing.T) {
} }
blob, _ := rlp.EncodeToBytes(tx) blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob) store.Put(blob)
if nonce < 1 { if nonce < 1 {
valids[id] = struct{}{} valids[tx.Hash()] = struct{}{}
} else { } else {
overdrafted[id] = struct{}{} overdrafted[tx.Hash()] = struct{}{}
} }
} }
// Insert a sequence of transactions overflowing the account cap to verify // Insert a sequence of transactions overflowing the account cap to verify
// that part of the set will get invalidated (case 7). // that part of the set will get invalidated (case 7).
var ( var (
overcapper, _ = crypto.GenerateKey() overcapper, _ = crypto.GenerateKey()
overcapped = make(map[uint64]struct{}) overcapped = make(map[common.Hash]struct{})
) )
for nonce := uint64(0); nonce < maxTxsPerAccount+3; nonce++ { for nonce := uint64(0); nonce < maxTxsPerAccount+3; nonce++ {
blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, 1, 1, overcapper)) tx := makeTx(nonce, 1, 1, 1, overcapper)
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob) store.Put(blob)
if nonce < maxTxsPerAccount { if nonce < maxTxsPerAccount {
valids[id] = struct{}{} valids[tx.Hash()] = struct{}{}
} else { } else {
overcapped[id] = struct{}{} overcapped[tx.Hash()] = struct{}{}
} }
} }
// Insert a batch of duplicated transactions to verify that only one of each // Insert a batch of duplicated transactions to verify that only one of each
// version will remain (case 8). // version will remain (case 8).
var ( var (
duplicater, _ = crypto.GenerateKey() duplicater, _ = crypto.GenerateKey()
duplicated = make(map[uint64]struct{}) duplicated = make(map[common.Hash]struct{})
) )
for _, nonce := range []uint64{0, 1, 2} { for _, nonce := range []uint64{0, 1, 2} {
blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, 1, 1, duplicater)) tx := makeTx(nonce, 1, 1, 1, duplicater)
blob, _ := rlp.EncodeToBytes(tx)
for i := 0; i < int(nonce)+1; i++ { for i := 0; i < int(nonce)+1; i++ {
id, _ := store.Put(blob) store.Put(blob)
if i == 0 { if i == 0 {
valids[id] = struct{}{} valids[tx.Hash()] = struct{}{}
} else { } else {
duplicated[id] = struct{}{} duplicated[tx.Hash()] = struct{}{}
} }
} }
} }
@ -696,17 +699,18 @@ func TestOpenDrops(t *testing.T) {
// remain (case 9). // remain (case 9).
var ( var (
repeater, _ = crypto.GenerateKey() repeater, _ = crypto.GenerateKey()
repeated = make(map[uint64]struct{}) repeated = make(map[common.Hash]struct{})
) )
for _, nonce := range []uint64{0, 1, 2} { for _, nonce := range []uint64{0, 1, 2} {
for i := 0; i < int(nonce)+1; i++ { for i := 0; i < int(nonce)+1; i++ {
blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater)) tx := makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater)
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob) store.Put(blob)
if i == 0 { if i == 0 {
valids[id] = struct{}{} valids[tx.Hash()] = struct{}{}
} else { } else {
repeated[id] = struct{}{} repeated[tx.Hash()] = struct{}{}
} }
} }
} }
@ -743,39 +747,41 @@ func TestOpenDrops(t *testing.T) {
// Verify that the malformed (case 1), badly signed (case 2) and gapped (case // Verify that the malformed (case 1), badly signed (case 2) and gapped (case
// 3) txs have been deleted from the pool // 3) txs have been deleted from the pool
alive := make(map[uint64]struct{}) alive := make(map[common.Hash]struct{})
for _, txs := range pool.index { for _, txs := range pool.index {
for _, tx := range txs { for _, tx := range txs {
switch tx.id { switch tx.id {
case malformed: case malformed:
t.Errorf("malformed RLP transaction remained in storage") t.Errorf("malformed RLP transaction remained in storage")
case badsig:
t.Errorf("invalidly signed transaction remained in storage")
default: default:
if _, ok := dangling[tx.id]; ok { if badsig == tx.hash {
t.Errorf("invalidly signed transaction remained in storage")
}
if _, ok := dangling[tx.hash]; ok {
t.Errorf("dangling transaction remained in storage: %d", tx.id) t.Errorf("dangling transaction remained in storage: %d", tx.id)
} else if _, ok := filled[tx.id]; ok { } else if _, ok := filled[tx.hash]; ok {
t.Errorf("filled transaction remained in storage: %d", tx.id) t.Errorf("filled transaction remained in storage: %d", tx.id)
} else if _, ok := overlapped[tx.id]; ok { } else if _, ok := overlapped[tx.hash]; ok {
t.Errorf("overlapped transaction remained in storage: %d", tx.id) t.Errorf("overlapped transaction remained in storage: %d", tx.id)
} else if _, ok := gapped[tx.id]; ok { } else if _, ok := gapped[tx.hash]; ok {
t.Errorf("gapped transaction remained in storage: %d", tx.id) t.Errorf("gapped transaction remained in storage: %d", tx.id)
} else if _, ok := underpaid[tx.id]; ok { } else if _, ok := underpaid[tx.hash]; ok {
t.Errorf("underpaid transaction remained in storage: %d", tx.id) t.Errorf("underpaid transaction remained in storage: %d", tx.id)
} else if _, ok := outpriced[tx.id]; ok { } else if _, ok := outpriced[tx.hash]; ok {
t.Errorf("outpriced transaction remained in storage: %d", tx.id) t.Errorf("outpriced transaction remained in storage: %d", tx.id)
} else if _, ok := exceeded[tx.id]; ok { } else if _, ok := exceeded[tx.hash]; ok {
t.Errorf("fully overdrafted transaction remained in storage: %d", tx.id) t.Errorf("fully overdrafted transaction remained in storage: %d", tx.id)
} else if _, ok := overdrafted[tx.id]; ok { } else if _, ok := overdrafted[tx.hash]; ok {
t.Errorf("partially overdrafted transaction remained in storage: %d", tx.id) t.Errorf("partially overdrafted transaction remained in storage: %d", tx.id)
} else if _, ok := overcapped[tx.id]; ok { } else if _, ok := overcapped[tx.hash]; ok {
t.Errorf("overcapped transaction remained in storage: %d", tx.id) t.Errorf("overcapped transaction remained in storage: %d", tx.id)
} else if _, ok := duplicated[tx.id]; ok { } else if _, ok := repeated[tx.hash]; ok {
t.Errorf("duplicated transaction remained in storage: %d", tx.id)
} else if _, ok := repeated[tx.id]; ok {
t.Errorf("repeated nonce transaction remained in storage: %d", tx.id) t.Errorf("repeated nonce transaction remained in storage: %d", tx.id)
} else { } else {
alive[tx.id] = struct{}{} if _, ok := alive[tx.hash]; ok {
t.Errorf("duplicated transaction remained in storage: %d", tx.id)
}
alive[tx.hash] = struct{}{}
} }
} }
} }
@ -784,14 +790,14 @@ func TestOpenDrops(t *testing.T) {
if len(alive) != len(valids) { if len(alive) != len(valids) {
t.Errorf("valid transaction count mismatch: have %d, want %d", len(alive), len(valids)) t.Errorf("valid transaction count mismatch: have %d, want %d", len(alive), len(valids))
} }
for id := range alive { for hash := range alive {
if _, ok := valids[id]; !ok { if _, ok := valids[hash]; !ok {
t.Errorf("extra transaction %d", id) t.Errorf("extra transaction %s", hash)
} }
} }
for id := range valids { for hash := range valids {
if _, ok := alive[id]; !ok { if _, ok := alive[hash]; !ok {
t.Errorf("missing transaction %d", id) t.Errorf("missing transaction %s", hash)
} }
} }
// Verify all the calculated pool internals. Interestingly, this is **not** // Verify all the calculated pool internals. Interestingly, this is **not**
@ -1010,7 +1016,10 @@ func TestOpenCap(t *testing.T) {
keep = []common.Address{addr1, addr3} keep = []common.Address{addr1, addr3}
drop = []common.Address{addr2} drop = []common.Address{addr2}
size = 2 * (txAvgSize + blobSize + uint64(txBlobOverhead)) // After migration to pooledBlobTx, cells (128 x 2048 = 2*blobSize) replace blobs.
// The actual billy slot size for pooledBlobTx is 2*(blobSize+txBlobOverhead)+txAvgSize.
pooledSlotSize uint64 = 2*(blobSize+uint64(txBlobOverhead)) + txAvgSize
size = 2 * pooledSlotSize
) )
store.Put(blob1) store.Put(blob1)
store.Put(blob2) store.Put(blob2)
@ -1019,7 +1028,7 @@ func TestOpenCap(t *testing.T) {
// Verify pool capping twice: first by reducing the data cap, then restarting // Verify pool capping twice: first by reducing the data cap, then restarting
// with a high cap to ensure everything was persisted previously // with a high cap to ensure everything was persisted previously
for _, datacap := range []uint64{2 * (txAvgSize + blobSize + uint64(txBlobOverhead)), 1000 * (txAvgSize + blobSize + uint64(txBlobOverhead))} { for _, datacap := range []uint64{size, 1000 * pooledSlotSize} {
// Create a blob pool out of the pre-seeded data, but cap it to 2 blob transaction // Create a blob pool out of the pre-seeded data, but cap it to 2 blob transaction
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
@ -1325,7 +1334,7 @@ func TestBlobCountLimit(t *testing.T) {
// Check that first succeeds second fails. // Check that first succeeds second fails.
if errs[0] != nil { if errs[0] != nil {
t.Fatalf("expected tx with 7 blobs to succeed, got %v", errs[0]) t.Fatalf("expected tx with 7 blobs to succeed, got: %v", errs[0])
} }
if !errors.Is(errs[1], txpool.ErrTxBlobLimitExceeded) { if !errors.Is(errs[1], txpool.ErrTxBlobLimitExceeded) {
t.Fatalf("expected tx with 8 blobs to fail, got: %v", errs[1]) t.Fatalf("expected tx with 8 blobs to fail, got: %v", errs[1])
@ -2112,7 +2121,8 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) {
b.Fatal(err) b.Fatal(err)
} }
statedb.AddBalance(addr, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified) statedb.AddBalance(addr, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
pool.add(tx) pooledTx, _ := newPooledBlobTx(tx)
pool.add(pooledTx)
} }
statedb.Commit(0, true, false) statedb.Commit(0, true, false)
defer pool.Close() defer pool.Close()
@ -2133,3 +2143,117 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) {
} }
} }
} }
func TestGetCells(t *testing.T) {
storage := t.TempDir()
os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700)
store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(params.BlobTxMaxBlobs), nil)
var (
key1, _ = crypto.GenerateKey()
addr1 = crypto.PubkeyToAddress(key1.PublicKey)
tx1 = makeMultiBlobTx(0, 1, 1000, 100, 3, 0, key1, types.BlobSidecarVersion1) // blobs [0, 3)
pooledTx1, _ = newPooledBlobTx(tx1)
blob1, _ = rlp.EncodeToBytes(pooledTx1)
)
store.Put(blob1)
store.Close()
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
statedb.Commit(0, true, false)
chain := &testBlockChain{
config: params.MainnetChainConfig,
basefee: uint256.NewInt(params.InitialBaseFee),
blobfee: uint256.NewInt(params.BlobTxMinBlobGasprice),
statedb: statedb,
}
pool := New(Config{Datadir: storage}, chain, nil)
if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil {
t.Fatalf("failed to create blob pool: %v", err)
}
defer pool.Close()
tests := []struct {
name string
hash common.Hash
mask types.CustodyBitmap
expectedLen int
shouldFail bool
}{
{
name: "Get cells with single index",
hash: tx1.Hash(),
mask: types.NewCustodyBitmap([]uint64{5}),
expectedLen: 3,
shouldFail: false,
},
{
name: "Get cells with all indices",
hash: tx1.Hash(),
mask: *types.CustodyBitmapAll,
expectedLen: 384,
shouldFail: false,
},
{
name: "Get cells with no indices",
hash: tx1.Hash(),
mask: types.CustodyBitmap{},
expectedLen: 0,
shouldFail: false,
},
{
name: "Get cells for non-existent transaction",
hash: common.Hash{0x01, 0x02, 0x03},
mask: types.NewCustodyBitmap([]uint64{0, 1}),
expectedLen: 0,
shouldFail: true,
},
{
name: "Get cells with random indices",
hash: tx1.Hash(),
mask: types.NewRandomCustodyBitmap(8),
expectedLen: 24,
shouldFail: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cells, err := pool.GetCells(tt.hash, tt.mask)
if err != nil && !tt.shouldFail {
t.Errorf("expected to success, got %v", err)
}
if err == nil && tt.shouldFail {
t.Errorf("expected to fail, got %v", err)
}
if len(cells) != tt.expectedLen {
t.Errorf("expected %d cells, got %d", tt.expectedLen, len(cells))
}
if tt.expectedLen > 0 && tt.expectedLen%3 == 0 {
blobCount := 3
cellsPerBlob := tt.expectedLen / blobCount
for blobIdx := 0; blobIdx < blobCount; blobIdx++ {
startIdx := blobIdx * cellsPerBlob
endIdx := startIdx + cellsPerBlob
if endIdx > len(cells) {
t.Errorf("blob %d: expected cells up to index %d, but only have %d cells",
blobIdx, endIdx-1, len(cells))
}
}
}
})
}
}

View file

@ -33,7 +33,7 @@ import (
type limboBlob struct { type limboBlob struct {
TxHash common.Hash // Owner transaction's hash to support resurrecting reorged txs TxHash common.Hash // Owner transaction's hash to support resurrecting reorged txs
Block uint64 // Block in which the blob transaction was included Block uint64 // Block in which the blob transaction was included
Tx *types.Transaction Tx *pooledBlobTx
} }
// limbo is a light, indexed database to temporarily store recently included // limbo is a light, indexed database to temporarily store recently included
@ -146,15 +146,15 @@ func (l *limbo) finalize(final *types.Header) {
// push stores a new blob transaction into the limbo, waiting until finality for // push stores a new blob transaction into the limbo, waiting until finality for
// it to be automatically evicted. // it to be automatically evicted.
func (l *limbo) push(tx *types.Transaction, block uint64) error { func (l *limbo) push(tx *pooledBlobTx, block uint64) error {
// If the blobs are already tracked by the limbo, consider it a programming // If the blobs are already tracked by the limbo, consider it a programming
// error. There's not much to do against it, but be loud. // error. There's not much to do against it, but be loud.
if _, ok := l.index[tx.Hash()]; ok { if _, ok := l.index[tx.Transaction.Hash()]; ok {
log.Error("Limbo cannot push already tracked blobs", "tx", tx.Hash()) log.Error("Limbo cannot push already tracked blobs", "tx", tx.Transaction.Hash())
return errors.New("already tracked blob transaction") return errors.New("already tracked blob transaction")
} }
if err := l.setAndIndex(tx, block); err != nil { if err := l.setAndIndex(tx, block); err != nil {
log.Error("Failed to set and index limboed blobs", "tx", tx.Hash(), "err", err) log.Error("Failed to set and index limboed blobs", "tx", tx.Transaction.Hash(), "err", err)
return err return err
} }
return nil return nil
@ -163,7 +163,7 @@ func (l *limbo) push(tx *types.Transaction, block uint64) error {
// pull retrieves a previously pushed set of blobs back from the limbo, removing // pull retrieves a previously pushed set of blobs back from the limbo, removing
// it at the same time. This method should be used when a previously included blob // it at the same time. This method should be used when a previously included blob
// transaction gets reorged out. // transaction gets reorged out.
func (l *limbo) pull(tx common.Hash) (*types.Transaction, error) { func (l *limbo) pull(tx common.Hash) (*pooledBlobTx, error) {
// If the blobs are not tracked by the limbo, there's not much to do. This // If the blobs are not tracked by the limbo, there's not much to do. This
// can happen for example if a blob transaction is mined without pushing it // can happen for example if a blob transaction is mined without pushing it
// into the network first. // into the network first.
@ -240,8 +240,8 @@ func (l *limbo) getAndDrop(id uint64) (*limboBlob, error) {
// setAndIndex assembles a limbo blob database entry and stores it, also updating // setAndIndex assembles a limbo blob database entry and stores it, also updating
// the in-memory indices. // the in-memory indices.
func (l *limbo) setAndIndex(tx *types.Transaction, block uint64) error { func (l *limbo) setAndIndex(tx *pooledBlobTx, block uint64) error {
txhash := tx.Hash() txhash := tx.Transaction.Hash()
item := &limboBlob{ item := &limboBlob{
TxHash: txhash, TxHash: txhash,
Block: block, Block: block,

View file

@ -18,11 +18,13 @@ package blobpool
import ( import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
) )
type txMetadata struct { type txMetadata struct {
id uint64 // the billy id of transction id uint64 // the billy id of transction
size uint64 // the RLP encoded size of transaction (blobs are included) size uint64 // the RLP encoded size of transaction (blobs are included)
custody types.CustodyBitmap
} }
// lookup maps blob versioned hashes to transaction hashes that include them, // lookup maps blob versioned hashes to transaction hashes that include them,
@ -93,6 +95,7 @@ func (l *lookup) track(tx *blobTxMeta) {
l.txIndex[tx.hash] = &txMetadata{ l.txIndex[tx.hash] = &txMetadata{
id: tx.id, id: tx.id,
size: tx.size, size: tx.size,
custody: *tx.custody,
} }
} }

View file

@ -1011,7 +1011,7 @@ func (pool *LegacyPool) get(hash common.Hash) *types.Transaction {
} }
// GetRLP returns a RLP-encoded transaction if it is contained in the pool. // GetRLP returns a RLP-encoded transaction if it is contained in the pool.
func (pool *LegacyPool) GetRLP(hash common.Hash) []byte { func (pool *LegacyPool) GetRLP(hash common.Hash, _ bool) []byte {
tx := pool.all.Get(hash) tx := pool.all.Get(hash)
if tx == nil { if tx == nil {
return nil return nil

View file

@ -132,7 +132,8 @@ type SubPool interface {
Get(hash common.Hash) *types.Transaction Get(hash common.Hash) *types.Transaction
// GetRLP returns a RLP-encoded transaction if it is contained in the pool. // GetRLP returns a RLP-encoded transaction if it is contained in the pool.
GetRLP(hash common.Hash) []byte // If includeBlob is false, blob data is stripped from blob transactions (ETH/71).
GetRLP(hash common.Hash, includeBlob bool) []byte
// GetMetadata returns the transaction type and transaction size with the // GetMetadata returns the transaction type and transaction size with the
// given transaction hash. // given transaction hash.

View file

@ -284,9 +284,9 @@ func (p *TxPool) Get(hash common.Hash) *types.Transaction {
} }
// GetRLP returns a RLP-encoded transaction if it is contained in the pool. // GetRLP returns a RLP-encoded transaction if it is contained in the pool.
func (p *TxPool) GetRLP(hash common.Hash) []byte { func (p *TxPool) GetRLP(hash common.Hash, includeBlob bool) []byte {
for _, subpool := range p.subpools { for _, subpool := range p.subpools {
encoded := subpool.GetRLP(hash) encoded := subpool.GetRLP(hash, includeBlob)
if len(encoded) != 0 { if len(encoded) != 0 {
return encoded return encoded
} }

View file

@ -64,9 +64,6 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
if opts.Accept&(1<<tx.Type()) == 0 { if opts.Accept&(1<<tx.Type()) == 0 {
return fmt.Errorf("%w: tx type %v not supported by this pool", core.ErrTxTypeNotSupported, tx.Type()) return fmt.Errorf("%w: tx type %v not supported by this pool", core.ErrTxTypeNotSupported, tx.Type())
} }
if blobCount := len(tx.BlobHashes()); blobCount > opts.MaxBlobCount {
return fmt.Errorf("%w: blob count %v, limit %v", ErrTxBlobLimitExceeded, blobCount, opts.MaxBlobCount)
}
// Before performing any expensive validations, sanity check that the tx is // Before performing any expensive validations, sanity check that the tx is
// smaller than the maximum limit the pool can meaningfully handle // smaller than the maximum limit the pool can meaningfully handle
if tx.Size() > opts.MaxSize { if tx.Size() > opts.MaxSize {
@ -146,9 +143,6 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
if tx.GasTipCapIntCmp(opts.MinTip) < 0 { if tx.GasTipCapIntCmp(opts.MinTip) < 0 {
return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.GasTipCap(), opts.MinTip) return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.GasTipCap(), opts.MinTip)
} }
if tx.Type() == types.BlobTxType {
return validateBlobTx(tx, head, opts)
}
if tx.Type() == types.SetCodeTxType { if tx.Type() == types.SetCodeTxType {
if len(tx.SetCodeAuthorizations()) == 0 { if len(tx.SetCodeAuthorizations()) == 0 {
return errors.New("set code tx must have at least one authorization tuple") return errors.New("set code tx must have at least one authorization tuple")
@ -157,14 +151,33 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
return nil return nil
} }
// validateBlobTx implements the blob-transaction specific validations. func ValidateBlobSidecar(tx *types.Transaction, sidecar *types.BlobTxCellSidecar, head *types.Header, opts *ValidationOptions) error {
func validateBlobTx(tx *types.Transaction, head *types.Header, opts *ValidationOptions) error { if sidecar.Custody.OneCount() == 0 {
sidecar := tx.BlobTxSidecar() return errors.New("blobless blob transaction")
if sidecar == nil {
return errors.New("missing sidecar in blob transaction")
} }
// Ensure the sidecar is constructed with the correct version, consistent // Ensure the blob fee cap satisfies the minimum blob gas price
// with the current fork. if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
}
// Verify whether the blob count is consistent with other parts of the sidecar and the transaction
blobCount := len(sidecar.Cells) / sidecar.Custody.OneCount()
hashes := tx.BlobHashes()
if blobCount == 0 {
return errors.New("blobless blob transaction")
}
if blobCount != len(sidecar.Commitments) || blobCount != len(hashes) {
return fmt.Errorf("invalid number of %d blobs compared to %d commitments and %d blob hashes", blobCount, len(sidecar.Commitments), len(tx.BlobHashes()))
}
// Check whether the blob count does not exceed the max blob count
if blobCount > opts.MaxBlobCount {
return fmt.Errorf("%w: blob count %v, limit %v", ErrTxBlobLimitExceeded, blobCount, opts.MaxBlobCount)
}
if err := sidecar.ValidateBlobCommitmentHashes(hashes); err != nil {
return err
}
// Ensure the sidecar version is correct for the current fork (master: bd77b77ed)
version := types.BlobSidecarVersion0 version := types.BlobSidecarVersion0
if opts.Config.IsOsaka(head.Number, head.Time) { if opts.Config.IsOsaka(head.Number, head.Time) {
version = types.BlobSidecarVersion1 version = types.BlobSidecarVersion1
@ -172,50 +185,42 @@ func validateBlobTx(tx *types.Transaction, head *types.Header, opts *ValidationO
if sidecar.Version != version { if sidecar.Version != version {
return fmt.Errorf("unexpected sidecar version, want: %d, got: %d", version, sidecar.Version) return fmt.Errorf("unexpected sidecar version, want: %d, got: %d", version, sidecar.Version)
} }
// Ensure the blob fee cap satisfies the minimum blob gas price
if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
}
// Ensure the number of items in the blob transaction and various side
// data match up before doing any expensive validations
hashes := tx.BlobHashes()
if len(hashes) == 0 {
return errors.New("blobless blob transaction")
}
if len(hashes) > params.BlobTxMaxBlobs {
return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.BlobTxMaxBlobs)
}
if len(sidecar.Blobs) != len(hashes) {
return fmt.Errorf("invalid number of %d blobs compared to %d blob hashes", len(sidecar.Blobs), len(hashes))
}
if err := sidecar.ValidateBlobCommitmentHashes(hashes); err != nil {
return err
}
// Fork-specific sidecar checks, including proof verification. // Fork-specific sidecar checks, including proof verification.
if sidecar.Version == types.BlobSidecarVersion1 { if sidecar.Version == types.BlobSidecarVersion1 {
return validateBlobSidecarOsaka(sidecar, hashes) return validateBlobSidecarOsaka(sidecar, hashes)
} else { }
return validateBlobSidecarLegacy(sidecar, hashes) return validateBlobSidecarLegacy(sidecar, hashes)
} }
}
func validateBlobSidecarLegacy(sidecar *types.BlobTxSidecar, hashes []common.Hash) error { func validateBlobSidecarLegacy(sidecar *types.BlobTxCellSidecar, hashes []common.Hash) error {
if len(sidecar.Proofs) != len(hashes) { if len(sidecar.Proofs) != len(hashes) {
return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes)) return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes))
} }
for i := range sidecar.Blobs { blobs, err := kzg4844.RecoverBlobs(sidecar.Cells, sidecar.Custody.Indices())
if err := kzg4844.VerifyBlobProof(&sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil { if err != nil {
return fmt.Errorf("%w: invalid blob proof: %v", ErrKZGVerificationError, err) return fmt.Errorf("%w: %v", ErrKZGVerificationError, err)
}
for i := range blobs {
if err := kzg4844.VerifyBlobProof(&blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil {
return fmt.Errorf("%w: invalid blob %d: %v", ErrKZGVerificationError, i, err)
} }
} }
return nil return nil
} }
func validateBlobSidecarOsaka(sidecar *types.BlobTxSidecar, hashes []common.Hash) error { func validateBlobSidecarOsaka(sidecar *types.BlobTxCellSidecar, hashes []common.Hash) error {
if len(sidecar.Proofs) != len(hashes)*kzg4844.CellProofsPerBlob { if len(sidecar.Proofs) != len(hashes)*kzg4844.CellProofsPerBlob {
return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes)*kzg4844.CellProofsPerBlob) return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes)*kzg4844.CellProofsPerBlob)
} }
if err := kzg4844.VerifyCellProofs(sidecar.Blobs, sidecar.Commitments, sidecar.Proofs); err != nil { indices := sidecar.Custody.Indices()
cellProofs := make([]kzg4844.Proof, 0)
for blobIdx := range len(sidecar.Commitments) {
for _, proofIdx := range indices {
idx := blobIdx*kzg4844.CellProofsPerBlob + int(proofIdx)
cellProofs = append(cellProofs, sidecar.Proofs[idx])
}
}
if err := kzg4844.VerifyCells(sidecar.Cells, sidecar.Commitments, cellProofs, sidecar.Custody.Indices()); err != nil {
return fmt.Errorf("%w: %v", ErrKZGVerificationError, err) return fmt.Errorf("%w: %v", ErrKZGVerificationError, err)
} }
return nil return nil

View file

@ -0,0 +1,132 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package types
import (
"math/bits"
"math/rand"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
// `CustodyBitmap` is a bitmap to represent which custody index to store (little endian)
type CustodyBitmap [16]byte
var (
CustodyBitmapAll = func() *CustodyBitmap {
var result CustodyBitmap
for i := range result {
result[i] = 0xFF
}
return &result
}()
CustodyBitmapData = func() *CustodyBitmap {
var result CustodyBitmap
for i := 0; i < kzg4844.DataPerBlob/8; i++ {
result[i] = 0xFF
}
return &result
}()
)
func NewCustodyBitmap(indices []uint64) CustodyBitmap {
var result CustodyBitmap
for _, i := range indices {
if i >= uint64(kzg4844.CellsPerBlob) {
panic("CustodyBitmap: bit index out of range")
}
result[i/8] |= 1 << (i % 8)
}
return result
}
// NewRandomCustodyBitmap creates a CustodyBitmap with n randomly selected indices.
// This should be used only for tests.
func NewRandomCustodyBitmap(n int) CustodyBitmap {
if n <= 0 || n > kzg4844.CellsPerBlob {
panic("CustodyBitmap: invalid number of indices")
}
indices := make([]uint64, 0, n)
used := make(map[uint64]bool)
for len(indices) < n {
idx := uint64(rand.Intn(kzg4844.CellsPerBlob))
if !used[idx] {
used[idx] = true
indices = append(indices, idx)
}
}
return NewCustodyBitmap(indices)
}
// IsSet returns whether bit i is set.
func (b CustodyBitmap) IsSet(i uint64) bool {
if i >= uint64(kzg4844.CellsPerBlob) {
return false
}
return (b[i/8]>>(i%8))&1 == 1
}
// OneCount returns the number of bits set to 1.
func (b CustodyBitmap) OneCount() int {
total := 0
for _, v := range b {
total += bits.OnesCount8(v)
}
return total
}
// Indices returns the bit positions set to 1, in ascending order.
func (b CustodyBitmap) Indices() []uint64 {
out := make([]uint64, 0, b.OneCount())
for byteIdx, val := range b {
v := val
for v != 0 {
tz := bits.TrailingZeros8(v)
out = append(out, uint64(byteIdx*8+tz))
v &^= 1 << tz
}
}
return out
}
// Difference returns b AND NOT set (bits in b but not in set).
func (b CustodyBitmap) Difference(set *CustodyBitmap) *CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] &^ set[i]
}
return &out
}
// Intersection returns b AND set.
func (b CustodyBitmap) Intersection(set *CustodyBitmap) *CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] & set[i]
}
return &out
}
// Union returns b OR set.
func (b CustodyBitmap) Union(set *CustodyBitmap) *CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] | set[i]
}
return &out
}

View file

@ -510,6 +510,18 @@ func (tx *Transaction) WithoutBlobTxSidecar() *Transaction {
return cpy return cpy
} }
// WithoutBlob returns a copy of tx with the blob data removed from the sidecar,
// keeping commitments, proofs and other metadata intact.
func (tx *Transaction) WithoutBlob() *Transaction {
blobtx, ok := tx.inner.(*BlobTx)
if !ok || blobtx.Sidecar == nil {
return tx
}
sidecarWithoutBlob := blobtx.Sidecar.Copy()
sidecarWithoutBlob.Blobs = nil
return tx.WithBlobTxSidecar(sidecarWithoutBlob)
}
// WithBlobTxSidecar returns a copy of tx with the blob sidecar added. // WithBlobTxSidecar returns a copy of tx with the blob sidecar added.
func (tx *Transaction) WithBlobTxSidecar(sideCar *BlobTxSidecar) *Transaction { func (tx *Transaction) WithBlobTxSidecar(sideCar *BlobTxSidecar) *Transaction {
blobtx, ok := tx.inner.(*BlobTx) blobtx, ok := tx.inner.(*BlobTx)

View file

@ -176,6 +176,37 @@ func (sc *BlobTxSidecar) Copy() *BlobTxSidecar {
} }
} }
func (sc *BlobTxSidecar) ToBlobTxCellSidecar() (*BlobTxCellSidecar, error) {
cells, err := kzg4844.ComputeCells(sc.Blobs)
if err != nil {
return nil, err
}
return &BlobTxCellSidecar{
Version: sc.Version,
Cells: cells,
Commitments: sc.Commitments,
Proofs: sc.Proofs,
Custody: *CustodyBitmapAll,
}, nil
}
type BlobTxCellSidecar struct {
Version byte
Cells []kzg4844.Cell
Commitments []kzg4844.Commitment
Proofs []kzg4844.Proof
Custody CustodyBitmap
}
// ValidateBlobCommitmentHashes checks whether the given hashes correspond to the
// commitments in the sidecar
func (c *BlobTxCellSidecar) ValidateBlobCommitmentHashes(hashes []common.Hash) error {
sc := BlobTxSidecar{
Commitments: c.Commitments,
}
return sc.ValidateBlobCommitmentHashes(hashes)
}
// blobTxWithBlobs represents blob tx with its corresponding sidecar. // blobTxWithBlobs represents blob tx with its corresponding sidecar.
// This is an interface because sidecars are versioned. // This is an interface because sidecars are versioned.
type blobTxWithBlobs interface { type blobTxWithBlobs interface {

View file

@ -34,9 +34,27 @@ var (
blobT = reflect.TypeFor[Blob]() blobT = reflect.TypeFor[Blob]()
commitmentT = reflect.TypeFor[Commitment]() commitmentT = reflect.TypeFor[Commitment]()
proofT = reflect.TypeFor[Proof]() proofT = reflect.TypeFor[Proof]()
cellT = reflect.TypeFor[Cell]()
) )
const CellProofsPerBlob = 128 const (
CellProofsPerBlob = 128
CellsPerBlob = 128
DataPerBlob = 64
)
// Cell represents a single cell in a blob.
type Cell [2048]byte
// UnmarshalJSON parses a cell in hex syntax.
func (c *Cell) UnmarshalJSON(input []byte) error {
return hexutil.UnmarshalFixedJSON(cellT, input, c[:])
}
// MarshalText returns the hex representation of c.
func (c *Cell) MarshalText() ([]byte, error) {
return hexutil.Bytes(c[:]).MarshalText()
}
// Blob represents a 4844 data blob. // Blob represents a 4844 data blob.
type Blob [131072]byte type Blob [131072]byte
@ -189,3 +207,27 @@ func CalcBlobHashV1(hasher hash.Hash, commit *Commitment) (vh [32]byte) {
func IsValidVersionedHash(h []byte) bool { func IsValidVersionedHash(h []byte) bool {
return len(h) == 32 && h[0] == 0x01 return len(h) == 32 && h[0] == 0x01
} }
// VerifyCells verifies a batch of proofs corresponding to the cells and commitments.
func VerifyCells(cells []Cell, commitments []Commitment, proofs []Proof, cellIndices []uint64) error {
if useCKZG.Load() {
return ckzgVerifyCells(cells, commitments, proofs, cellIndices)
}
return gokzgVerifyCells(cells, commitments, proofs, cellIndices)
}
// ComputeCells computes the cells from the given blobs.
func ComputeCells(blobs []Blob) ([]Cell, error) {
if useCKZG.Load() {
return ckzgComputeCells(blobs)
}
return gokzgComputeCells(blobs)
}
// RecoverBlobs recovers blobs from the given cells and cell indices.
func RecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
if useCKZG.Load() {
return ckzgRecoverBlobs(cells, cellIndices)
}
return gokzgRecoverBlobs(cells, cellIndices)
}

View file

@ -190,3 +190,90 @@ func ckzgVerifyCellProofBatch(blobs []Blob, commitments []Commitment, cellProofs
} }
return nil return nil
} }
func ckzgVerifyCells(cells []Cell, commitments []Commitment, cellProofs []Proof, cellIndices []uint64) error {
ckzgIniter.Do(ckzgInit)
var (
proofs = make([]ckzg4844.Bytes48, len(cellProofs))
commits = make([]ckzg4844.Bytes48, 0, len(cellProofs))
indices = make([]uint64, 0, len(cellProofs))
kzgcells = make([]ckzg4844.Cell, 0, len(cellProofs))
)
for i := range cellProofs {
proofs[i] = (ckzg4844.Bytes48)(cellProofs[i])
kzgcells = append(kzgcells, (ckzg4844.Cell)(cells[i]))
}
if len(cellProofs)%len(commitments) != 0 {
return errors.New("wrong cell proofs and commitments length")
}
cellCounts := len(cellProofs) / len(commitments)
for _, commitment := range commitments {
for j := 0; j < cellCounts; j++ {
commits = append(commits, (ckzg4844.Bytes48)(commitment))
}
}
blobCounts := len(cellProofs) / len(cellIndices)
for j := 0; j < blobCounts; j++ {
indices = append(indices, cellIndices...)
}
valid, err := ckzg4844.VerifyCellKZGProofBatch(commits, indices, kzgcells, proofs)
if err != nil {
return err
}
if !valid {
return errors.New("invalid proof")
}
return nil
}
func ckzgComputeCells(blobs []Blob) ([]Cell, error) {
ckzgIniter.Do(ckzgInit)
cells := make([]Cell, 0, ckzg4844.CellsPerExtBlob*len(blobs))
for i := range blobs {
cellsI, err := ckzg4844.ComputeCells((*ckzg4844.Blob)(&blobs[i]))
if err != nil {
return []Cell{}, err
}
for _, c := range cellsI {
cells = append(cells, Cell(c))
}
}
return cells, nil
}
func ckzgRecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
ckzgIniter.Do(ckzgInit)
if len(cellIndices) == 0 || len(cells)%len(cellIndices) != 0 {
return []Blob{}, errors.New("cells with wrong length")
}
blobCount := len(cells) / len(cellIndices)
blobs := make([]Blob, 0, blobCount)
offset := 0
for range blobCount {
kzgcells := make([]ckzg4844.Cell, 0, len(cellIndices))
for _, cell := range cells[offset : offset+len(cellIndices)] {
kzgcells = append(kzgcells, ckzg4844.Cell(cell))
}
extCells, err := ckzg4844.RecoverCells(cellIndices, kzgcells)
if err != nil {
return []Blob{}, err
}
var blob Blob
for i, cell := range extCells[:64] {
copy(blob[i*len(cell):], cell[:])
}
blobs = append(blobs, blob)
offset = offset + len(cellIndices)
}
return blobs, nil
}

View file

@ -73,3 +73,15 @@ func ckzgVerifyCellProofBatch(blobs []Blob, commitments []Commitment, proof []Pr
func ckzgComputeCellProofs(blob *Blob) ([]Proof, error) { func ckzgComputeCellProofs(blob *Blob) ([]Proof, error) {
panic("unsupported platform") panic("unsupported platform")
} }
func ckzgVerifyCells(cells []Cell, commitments []Commitment, cellProofs []Proof, cellIndices []uint64) error {
panic("unsupported platform")
}
func ckzgComputeCells(blobs []Blob) ([]Cell, error) {
panic("unsupported platform")
}
func ckzgRecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
panic("unsupported platform")
}

View file

@ -18,6 +18,7 @@ package kzg4844
import ( import (
"encoding/json" "encoding/json"
"errors"
"sync" "sync"
gokzg4844 "github.com/crate-crypto/go-eth-kzg" gokzg4844 "github.com/crate-crypto/go-eth-kzg"
@ -148,3 +149,92 @@ func gokzgVerifyCellProofBatch(blobs []Blob, commitments []Commitment, cellProof
} }
return context.VerifyCellKZGProofBatch(commits, cellIndices, cells[:], proofs) return context.VerifyCellKZGProofBatch(commits, cellIndices, cells[:], proofs)
} }
// gokzgVerifyCells verifies that the cell data corresponds to the provided commitment.
func gokzgVerifyCells(cells []Cell, commitments []Commitment, cellProofs []Proof, cellIndices []uint64) error {
gokzgIniter.Do(gokzgInit)
var (
proofs = make([]gokzg4844.KZGProof, len(cellProofs))
commits = make([]gokzg4844.KZGCommitment, 0, len(cellProofs))
indices = make([]uint64, 0, len(cellProofs))
kzgcells = make([]*gokzg4844.Cell, 0, len(cellProofs))
)
// Copy over the cell proofs and cells
for i := range cellProofs {
proofs[i] = gokzg4844.KZGProof(cellProofs[i])
gc := gokzg4844.Cell(cells[i])
kzgcells = append(kzgcells, &gc)
}
if len(cellProofs)%len(commitments) != 0 {
return errors.New("wrong cell proofs and commitments length")
}
cellCounts := len(cellProofs) / len(commitments)
// Blow up the commitments to be the same length as the proofs
for _, commitment := range commitments {
for j := 0; j < cellCounts; j++ {
commits = append(commits, gokzg4844.KZGCommitment(commitment))
}
}
blobCounts := len(cellProofs) / len(cellIndices)
for j := 0; j < blobCounts; j++ {
indices = append(indices, cellIndices...)
}
return context.VerifyCellKZGProofBatch(commits, indices, kzgcells, proofs)
}
// gokzgComputeCells computes cells from blobs.
func gokzgComputeCells(blobs []Blob) ([]Cell, error) {
gokzgIniter.Do(gokzgInit)
cells := make([]Cell, 0, gokzg4844.CellsPerExtBlob*len(blobs))
for i := range blobs {
cellsI, err := context.ComputeCells((*gokzg4844.Blob)(&blobs[i]), 2)
if err != nil {
return []Cell{}, err
}
for _, c := range cellsI {
if c != nil {
cells = append(cells, Cell(*c))
}
}
}
return cells, nil
}
// gokzgRecoverBlobs recovers blobs from cells and cell indices.
func gokzgRecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
gokzgIniter.Do(gokzgInit)
if len(cellIndices) == 0 || len(cells)%len(cellIndices) != 0 {
return []Blob{}, errors.New("cells with wrong length")
}
blobCount := len(cells) / len(cellIndices)
blobs := make([]Blob, 0, blobCount)
offset := 0
for range blobCount {
kzgcells := make([]*gokzg4844.Cell, 0, len(cellIndices))
for _, cell := range cells[offset : offset+len(cellIndices)] {
gc := gokzg4844.Cell(cell)
kzgcells = append(kzgcells, &gc)
}
extCells, err := context.RecoverCells(cellIndices, kzgcells, 2)
if err != nil {
return []Blob{}, err
}
var blob Blob
for i, cell := range extCells[:64] {
copy(blob[i*len(cell):], cell[:])
}
blobs = append(blobs, blob)
offset = offset + len(cellIndices)
}
return blobs, nil
}

View file

@ -253,3 +253,119 @@ func benchmarkComputeCellProofs(b *testing.B, ckzg bool) {
} }
} }
} }
func TestCKZGVerifyPartialCells(t *testing.T) { testVerifyPartialCells(t, true) }
func TestGoKZGVerifyPartialCells(t *testing.T) { testVerifyPartialCells(t, false) }
func testVerifyPartialCells(t *testing.T, ckzg bool) {
if ckzg && !ckzgAvailable {
t.Skip("CKZG unavailable in this test build")
}
defer func(old bool) { useCKZG.Store(old) }(useCKZG.Load())
useCKZG.Store(ckzg)
const blobCount = 3
var blobs []*Blob
var commitments []Commitment
for range blobCount {
blob := randBlob()
commitment, err := BlobToCommitment(blob)
if err != nil {
t.Fatalf("failed to commit blob: %v", err)
}
blobs = append(blobs, blob)
commitments = append(commitments, commitment)
}
var (
partialCells []Cell
partialProofs []Proof
commits []Commitment
indices []uint64
)
for bi, blob := range blobs {
proofs, err := ComputeCellProofs(blob)
if err != nil {
t.Fatalf("failed to compute cell proofs: %v", err)
}
cells, err := ComputeCells([]Blob{*blob})
if err != nil {
t.Fatalf("failed to compute cells: %v", err)
}
commits = append(commits, commitments[bi])
// sample 0, 31, 63, 95 cells
step := len(cells) / 4
indices = []uint64{0, uint64(step - 1), uint64(2*step - 1), uint64(3*step - 1)}
for _, idx := range indices {
partialCells = append(partialCells, cells[idx])
partialProofs = append(partialProofs, proofs[idx])
}
}
if err := VerifyCells(partialCells, commits, partialProofs, indices); err != nil {
t.Fatalf("failed to verify partial cell proofs: %v", err)
}
}
func TestCKZGRecoverBlob(t *testing.T) { testRecoverBlob(t, true) }
func TestGoKZGRecoverBlob(t *testing.T) { testRecoverBlob(t, false) }
func testRecoverBlob(t *testing.T, ckzg bool) {
if ckzg && !ckzgAvailable {
t.Skip("CKZG unavailable in this test build")
}
defer func(old bool) { useCKZG.Store(old) }(useCKZG.Load())
useCKZG.Store(ckzg)
blobs := []Blob{}
blobs = append(blobs, *randBlob())
blobs = append(blobs, *randBlob())
blobs = append(blobs, *randBlob())
cells, err := ComputeCells(blobs)
if err != nil {
t.Fatalf("failed to compute cells: %v", err)
}
proofs := make([]Proof, 0)
commitments := make([]Commitment, len(blobs))
for i, blob := range blobs {
proof, err := ComputeCellProofs(&blob)
if err != nil {
t.Fatalf("failed to compute proof: %v", err)
}
proofs = append(proofs, proof...)
commitment, err := BlobToCommitment(&blob)
if err != nil {
t.Fatalf("failed to compute commitment: %v", err)
}
commitments[i] = commitment
}
var (
partialCells []Cell
indices []uint64
)
for ci := 64; ci < 128; ci++ {
indices = append(indices, uint64(ci))
}
for i := 0; i < len(cells); i += 128 {
start := i + 64
end := i + 128
partialCells = append(partialCells, cells[start:end]...)
}
recoverBlobs, err := RecoverBlobs(partialCells, indices)
if err != nil {
t.Fatalf("failed to recover blob: %v", err)
}
if err := VerifyCellProofs(recoverBlobs, commitments, proofs); err != nil {
t.Fatalf("failed to verify recovered blob: %v", err)
}
}

View file

@ -43,6 +43,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/fetcher"
"github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/gasprice"
"github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/eth/protocols/snap" "github.com/ethereum/go-ethereum/eth/protocols/snap"
@ -335,11 +336,13 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
Database: chainDb, Database: chainDb,
Chain: eth.blockchain, Chain: eth.blockchain,
TxPool: eth.txPool, TxPool: eth.txPool,
BlobPool: eth.blobTxPool,
Network: networkID, Network: networkID,
Sync: config.SyncMode, Sync: config.SyncMode,
BloomCache: uint64(cacheLimit), BloomCache: uint64(cacheLimit),
EventMux: eth.eventMux, EventMux: eth.eventMux,
RequiredBlocks: config.RequiredBlocks, RequiredBlocks: config.RequiredBlocks,
Custody: *types.CustodyBitmapAll,
}); err != nil { }); err != nil {
return nil, err return nil, err
} }
@ -423,6 +426,7 @@ func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager
func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain } func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain }
func (s *Ethereum) TxPool() *txpool.TxPool { return s.txPool } func (s *Ethereum) TxPool() *txpool.TxPool { return s.txPool }
func (s *Ethereum) BlobTxPool() *blobpool.BlobPool { return s.blobTxPool } func (s *Ethereum) BlobTxPool() *blobpool.BlobPool { return s.blobTxPool }
func (s *Ethereum) BlobFetcher() *fetcher.BlobFetcher { return s.handler.blobFetcher }
func (s *Ethereum) Engine() consensus.Engine { return s.engine } func (s *Ethereum) Engine() consensus.Engine { return s.engine }
func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb }
func (s *Ethereum) IsListening() bool { return true } // Always listening func (s *Ethereum) IsListening() bool { return true } // Always listening

View file

@ -1163,6 +1163,11 @@ func (api *ConsensusAPI) getBodiesByRange(start, count hexutil.Uint64) ([]*engin
return bodies, nil return bodies, nil
} }
func (api *ConsensusAPI) BlobCustodyUpdatedV1(custodyColumns []uint64) {
bitmap := types.NewCustodyBitmap(custodyColumns)
api.eth.BlobFetcher().UpdateCustody(bitmap)
}
func getBody(block *types.Block) *engine.ExecutionPayloadBody { func getBody(block *types.Block) *engine.ExecutionPayloadBody {
if block == nil { if block == nil {
return nil return nil

View file

@ -1911,7 +1911,12 @@ func newGetBlobEnv(t testing.TB, version byte) (*node.Node, *ConsensusAPI) {
tx1 := makeMultiBlobTx(&config, 0, 2, 0, key1, version) // blob[0, 2) tx1 := makeMultiBlobTx(&config, 0, 2, 0, key1, version) // blob[0, 2)
tx2 := makeMultiBlobTx(&config, 0, 2, 2, key2, version) // blob[2, 4) tx2 := makeMultiBlobTx(&config, 0, 2, 2, key2, version) // blob[2, 4)
tx3 := makeMultiBlobTx(&config, 0, 2, 4, key3, version) // blob[4, 6) tx3 := makeMultiBlobTx(&config, 0, 2, 4, key3, version) // blob[4, 6)
ethServ.TxPool().Add([]*types.Transaction{tx1, tx2, tx3}, true) errs := ethServ.TxPool().Add([]*types.Transaction{tx1, tx2, tx3}, true)
for i, err := range errs {
if err != nil {
t.Logf("Add tx %d failed: %v", i, err)
}
}
api := newConsensusAPIWithoutHeartbeat(ethServ) api := newConsensusAPIWithoutHeartbeat(ethServ)
return n, api return n, api
@ -2108,6 +2113,15 @@ func runGetBlobs(t testing.TB, getBlobs getBlobsFn, start, limit int, fillRandom
} }
} }
if !reflect.DeepEqual(result, expect) { if !reflect.DeepEqual(result, expect) {
t.Logf("result len=%d, expect len=%d", len(result), len(expect))
if len(result) > 0 && result[0] != nil && len(expect) > 0 && expect[0] != nil {
t.Logf("result[0].Blob len=%d, expect[0].Blob len=%d", len(result[0].Blob), len(expect[0].Blob))
t.Logf("result[0].CellProofs len=%d, expect[0].CellProofs len=%d", len(result[0].CellProofs), len(expect[0].CellProofs))
t.Logf("result[0].Blob == expect[0].Blob: %v", reflect.DeepEqual(result[0].Blob, expect[0].Blob))
t.Logf("result[0].CellProofs == expect[0].CellProofs: %v", reflect.DeepEqual(result[0].CellProofs, expect[0].CellProofs))
} else {
t.Logf("result[0]=%v, expect[0]=%v", result, expect)
}
t.Fatalf("Unexpected result for case %s", name) t.Fatalf("Unexpected result for case %s", name)
} }
} }

784
eth/fetcher/blob_fetcher.go Normal file
View file

@ -0,0 +1,784 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package fetcher
import (
"math/rand"
"slices"
"sort"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
// todo remove partial / full
type random interface {
Intn(n int) int
}
// BlobFetcher fetches blobs of new type-3 transactions with probability p,
// and for the remaining (1-p) transactions, it performs availability checks.
// For availability checks, it fetches cells from each blob in the transaction
// according to the custody cell indices provided by the consensus client
// connected to this execution client.
var blobFetchTimeout = 5 * time.Second
// todo tuning
const (
availabilityThreshold = 2
maxPayloadRetrievals = 128
maxPayloadAnnounces = 4096
MAX_CELLS_PER_PARTIAL_REQUEST = 8
blobAvailabilityTimeout = 500 * time.Millisecond
)
type blobTxAnnounce struct {
origin string // Identifier of the peer that sent the announcement
txs []common.Hash // Hashes of transactions announced
cells types.CustodyBitmap // Custody information of transactions being announced
}
type cellRequest struct {
txs []common.Hash // Transactions that have been requested for their cells
cells *types.CustodyBitmap // Requested cell indices
time mclock.AbsTime // Timestamp when the request was made
}
type payloadDelivery struct {
origin string // Peer from which the payloads were delivered
txs []common.Hash // Hashes of transactions that were delivered
cells [][]kzg4844.Cell
cellBitmap *types.CustodyBitmap
}
type cellWithSeq struct {
seq uint64
cells *types.CustodyBitmap
}
type fetchStatus struct {
fetching *types.CustodyBitmap // To avoid fetching cells which had already been fetched / currently being fetched
fetched []uint64 // To sort cells
cells []kzg4844.Cell
}
// BlobFetcher is responsible for managing type 3 transactions based on peer announcements.
//
// BlobFetcher manages three buffers:
// - Transactions not to be fetched are moved to "waitlist"
// if a payload(blob) seems to be possessed by D(threshold) other peers, request custody cells for that.
// Accept it when the cells are received. Otherwise, it is dropped.
// - Transactions queued to be fetched are moved to "announces"
// if a payload is received, it is added to the blob pool. Otherwise, the transaction is dropped.
// - Transactions to be fetched are moved to "fetching"
// if a payload/cell announcement is received during fetch, the peer is recorded as an alternate source.
type BlobFetcher struct {
notify chan *blobTxAnnounce
cleanup chan *payloadDelivery
drop chan *txDrop
quit chan struct{}
custody *types.CustodyBitmap
txSeq uint64 // To make transactions fetched in arrival order
full map[common.Hash]struct{}
partial map[common.Hash]struct{}
// Buffer 1: Set of blob txs whose blob data is waiting for availability confirmation (not pull decision)
waitlist map[common.Hash]map[string]struct{} // Peer set that announced blob availability
waittime map[common.Hash]mclock.AbsTime // Timestamp when added to waitlist
waitslots map[string]map[common.Hash]struct{} // Waiting announcements grouped by peer (DoS protection)
// waitSlots should also include announcements with partial cells
// Buffer 2: Transactions queued for fetching (pull decision + not pull decision)
// "announces" is shared with stage 3, for DoS protection
announces map[string]map[common.Hash]*cellWithSeq // Set of announced transactions, grouped by origin peer
// Buffer 2
// Stage 3: Transactions whose payloads/cells are currently being fetched (pull decision + not pull decision)
fetches map[common.Hash]*fetchStatus // Hash -> Bitmap, in-flight transaction cells
requests map[string][]*cellRequest // In-flight transaction retrievals
// todo simplify
alternates map[common.Hash]map[string]*types.CustodyBitmap // In-flight transaction alternate origins (in case the peer is dropped)
// Callbacks
validateCells func([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error
addPayload func([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error
fetchPayloads func(string, []common.Hash, *types.CustodyBitmap) error
dropPeer func(string)
step chan struct{} // Notification channel when the fetcher loop iterates
clock mclock.Clock // Monotonic clock or simulated clock for tests
realTime func() time.Time // Real system time or simulated time for tests
rand random // Randomizer
}
func NewBlobFetcher(
validateCells func([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error,
addPayload func([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error,
fetchPayloads func(string, []common.Hash, *types.CustodyBitmap) error, dropPeer func(string),
custody *types.CustodyBitmap, rand random) *BlobFetcher {
return &BlobFetcher{
notify: make(chan *blobTxAnnounce),
cleanup: make(chan *payloadDelivery),
drop: make(chan *txDrop),
quit: make(chan struct{}),
full: make(map[common.Hash]struct{}),
partial: make(map[common.Hash]struct{}),
waitlist: make(map[common.Hash]map[string]struct{}),
waittime: make(map[common.Hash]mclock.AbsTime),
waitslots: make(map[string]map[common.Hash]struct{}),
announces: make(map[string]map[common.Hash]*cellWithSeq),
fetches: make(map[common.Hash]*fetchStatus),
requests: make(map[string][]*cellRequest),
alternates: make(map[common.Hash]map[string]*types.CustodyBitmap),
validateCells: validateCells,
addPayload: addPayload,
fetchPayloads: fetchPayloads,
dropPeer: dropPeer,
custody: custody,
clock: mclock.System{},
realTime: time.Now,
rand: rand,
}
}
// Notify is called when a Type 3 transaction is observed on the network. (TransactionPacket / NewPooledTransactionHashesPacket)
func (f *BlobFetcher) Notify(peer string, txs []common.Hash, cells types.CustodyBitmap) error {
blobAnnounce := &blobTxAnnounce{origin: peer, txs: txs, cells: cells}
select {
case f.notify <- blobAnnounce:
return nil
case <-f.quit:
return errTerminated
}
}
// Enqueue inserts a batch of received blob payloads into the blob pool.
// This is triggered by ethHandler upon receiving direct request responses.
func (f *BlobFetcher) Enqueue(peer string, hashes []common.Hash, cells [][]kzg4844.Cell, cellBitmap types.CustodyBitmap) error {
blobReplyInMeter.Mark(int64(len(hashes)))
select {
case f.cleanup <- &payloadDelivery{origin: peer, txs: hashes, cells: cells, cellBitmap: &cellBitmap}:
case <-f.quit:
return errTerminated
}
return nil
}
func (f *BlobFetcher) Drop(peer string) error {
select {
case f.drop <- &txDrop{peer: peer}:
return nil
case <-f.quit:
return errTerminated
}
}
func (f *BlobFetcher) UpdateCustody(cells types.CustodyBitmap) {
f.custody = &cells
}
func (f *BlobFetcher) Start() {
go f.loop()
}
func (f *BlobFetcher) Stop() {
close(f.quit)
}
func (f *BlobFetcher) loop() {
var (
waitTimer = new(mclock.Timer) // Timer for waitlist (availability)
waitTrigger = make(chan struct{}, 1)
timeoutTimer = new(mclock.Timer) // Timer for payload fetch request
timeoutTrigger = make(chan struct{}, 1)
)
for {
select {
case ann := <-f.notify:
// Drop part of the announcements if too many have accumulated from that peer
// This prevents a peer from dominating the queue with txs without responding to the request
// todo maxPayloadAnnounces -> according to the number of blobs
used := len(f.waitslots[ann.origin]) + len(f.announces[ann.origin])
if used >= maxPayloadAnnounces {
// Already full
break
}
want := used + len(ann.txs)
if want >= maxPayloadAnnounces {
// drop part of announcements
ann.txs = ann.txs[:maxPayloadAnnounces-used]
}
var (
idleWait = len(f.waittime) == 0
_, oldPeer = f.announces[ann.origin]
nextSeq = func() uint64 {
seq := f.txSeq
f.txSeq++
return seq
}
reschedule = make(map[string]struct{})
)
for _, hash := range ann.txs {
if oldPeer && f.announces[ann.origin][hash] != nil {
// Ignore already announced information
// We also have to prevent reannouncement by changing cells field.
// Considering cell custody transition is notified in advance of its finalization by consensus client,
// there is no reason to reannounce cells, and it has to be prevented.
continue
}
// Decide full or partial request
if _, ok := f.full[hash]; !ok {
if _, ok := f.partial[hash]; !ok {
// Not decided yet
var randomValue int
if f.rand == nil {
randomValue = rand.Intn(100)
} else {
randomValue = f.rand.Intn(100)
}
if randomValue < 15 {
f.full[hash] = struct{}{}
} else {
f.partial[hash] = struct{}{}
// Register for availability check
f.waitlist[hash] = make(map[string]struct{})
f.waittime[hash] = f.clock.Now()
}
}
}
if _, ok := f.full[hash]; ok {
// 1) Decided to send full request of the tx
if ann.cells != *types.CustodyBitmapAll {
continue
}
if f.announces[ann.origin] == nil {
f.announces[ann.origin] = make(map[common.Hash]*cellWithSeq)
}
f.announces[ann.origin][hash] = &cellWithSeq{
cells: types.CustodyBitmapData,
seq: nextSeq(),
}
reschedule[ann.origin] = struct{}{}
continue
}
if _, ok := f.partial[hash]; ok {
// 2) Decided to send partial request of the tx
if f.waitlist[hash] != nil {
if ann.cells != *types.CustodyBitmapAll {
continue
}
// Transaction is at the stage of availability check
// Add the peer to the peer list with full availability (waitlist)
f.waitlist[hash][ann.origin] = struct{}{}
if waitslots := f.waitslots[ann.origin]; waitslots != nil {
waitslots[hash] = struct{}{}
} else {
f.waitslots[ann.origin] = map[common.Hash]struct{}{
hash: {},
}
}
if len(f.waitlist[hash]) >= availabilityThreshold {
for peer := range f.waitlist[hash] {
if f.announces[peer] == nil {
f.announces[peer] = make(map[common.Hash]*cellWithSeq)
}
f.announces[peer][hash] = &cellWithSeq{
cells: f.custody,
seq: nextSeq(),
}
delete(f.waitslots[peer], hash)
if len(f.waitslots[peer]) == 0 {
delete(f.waitslots, peer)
}
reschedule[peer] = struct{}{}
}
delete(f.waitlist, hash)
}
continue
}
if ann.cells.Intersection(f.custody).OneCount() == 0 {
// No custody overlapping
continue
}
if f.announces[ann.origin] == nil {
f.announces[ann.origin] = make(map[common.Hash]*cellWithSeq)
}
f.announces[ann.origin][hash] = &cellWithSeq{
cells: ann.cells.Intersection(f.custody),
seq: nextSeq(),
}
reschedule[ann.origin] = struct{}{}
}
}
// If a new item was added to the waitlist, schedule its timeout
if idleWait && len(f.waittime) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
// If this is a new peer and that peer sent transaction with payload flag,
// schedule transaction fetches from it
//todo
if !oldPeer && len(f.announces[ann.origin]) > 0 {
f.scheduleFetches(timeoutTimer, timeoutTrigger, reschedule)
}
case <-waitTrigger:
// At least one transaction's waiting time ran out, pop all expired ones
// and update the blobpool according to availability
// Availability failure case
for hash, instance := range f.waittime {
if time.Duration(f.clock.Now()-instance)+txGatherSlack > blobAvailabilityTimeout {
// Check if enough peers have announced availability
for peer := range f.waitlist[hash] {
delete(f.waitslots[peer], hash)
if len(f.waitslots[peer]) == 0 {
delete(f.waitslots, peer)
}
}
delete(f.waittime, hash)
delete(f.waitlist, hash)
}
}
// If transactions are still waiting for availability, reschedule the wait timer
if len(f.waittime) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
case <-timeoutTrigger:
// Clean up any expired retrievals and avoid re-requesting them from the
// same peer (either overloaded or malicious, useless in both cases).
// Update blobpool according to availability result.
for peer, requests := range f.requests {
newRequests := make([]*cellRequest, 0)
for _, req := range requests {
if time.Duration(f.clock.Now()-req.time)+txGatherSlack > blobFetchTimeout {
// Reschedule all timeout cells to alternate peers
for _, hash := range req.txs {
// Do not request the same tx from this peer
delete(f.announces[peer], hash)
delete(f.alternates[hash], peer)
// Allow other candidates to be requested these cells
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(req.cells)
// Drop cells if there is no alternate source to fetch cells from
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
delete(f.fetches, hash)
}
}
if len(f.announces[peer]) == 0 {
delete(f.announces, peer)
}
} else {
newRequests = append(newRequests, req)
}
}
f.requests[peer] = newRequests
if len(f.requests[peer]) == 0 {
delete(f.requests, peer)
}
}
// Schedule a new transaction retrieval
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
// Trigger timeout for new schedule
f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
case delivery := <-f.cleanup:
// Remove from announce
addedHashes := make([]common.Hash, 0)
addedCells := make([][]kzg4844.Cell, 0)
var requestId int
request := new(cellRequest)
for _, hash := range delivery.txs {
// Find the request
for i, req := range f.requests[delivery.origin] {
if slices.Contains(req.txs, hash) && *req.cells == *delivery.cellBitmap {
request = req
requestId = i
break
}
}
if request != nil {
break
}
}
if request == nil {
// peer sent cells not requested. ignore
break
}
for i, hash := range delivery.txs {
if !slices.Contains(request.txs, hash) {
// Unexpected hash, ignore
continue
}
// Update fetch status
f.fetches[hash].fetched = append(f.fetches[hash].fetched, delivery.cellBitmap.Indices()...)
f.fetches[hash].cells = append(f.fetches[hash].cells, delivery.cells[i]...)
// Update announces of this peer
delete(f.announces[delivery.origin], hash)
if len(f.announces[delivery.origin]) == 0 {
delete(f.announces, delivery.origin)
}
delete(f.alternates[hash], delivery.origin)
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
}
// Check whether the all required cells are fetched
completed := false
if _, ok := f.full[hash]; ok && len(f.fetches[hash].fetched) >= kzg4844.DataPerBlob {
completed = true
} else if _, ok := f.partial[hash]; ok {
fetched := make([]uint64, len(f.fetches[hash].fetched))
copy(fetched, f.fetches[hash].fetched)
slices.Sort(fetched)
custodyIndices := f.custody.Indices()
completed = slices.Equal(fetched, custodyIndices)
}
if completed {
addedHashes = append(addedHashes, hash)
fetchStatus := f.fetches[hash]
sort.Slice(fetchStatus.cells, func(i, j int) bool {
return fetchStatus.fetched[i] < fetchStatus.fetched[j]
})
addedCells = append(addedCells, fetchStatus.cells)
// remove announces from other peers
for peer, txset := range f.announces {
delete(txset, hash)
if len(txset) == 0 {
delete(f.announces, peer)
}
}
delete(f.alternates, hash)
delete(f.fetches, hash)
}
}
// Update mempool status for arrived hashes
if len(addedHashes) > 0 {
f.addPayload(addedHashes, addedCells, delivery.cellBitmap)
}
// Remove the request
f.requests[delivery.origin][requestId] = f.requests[delivery.origin][len(f.requests[delivery.origin])-1]
f.requests[delivery.origin] = f.requests[delivery.origin][:len(f.requests[delivery.origin])-1]
if len(f.requests[delivery.origin]) == 0 {
delete(f.requests, delivery.origin)
}
// Reschedule missing transactions in the request
// Anything not delivered should be re-scheduled (with or without
// this peer, depending on the response cutoff)
delivered := make(map[common.Hash]struct{})
for _, hash := range delivery.txs {
delivered[hash] = struct{}{}
}
cutoff := len(request.txs)
for i, hash := range request.txs {
if _, ok := delivered[hash]; ok {
cutoff = i
continue
}
}
// Reschedule missing hashes from alternates, not-fulfilled from alt+self
for i, hash := range request.txs {
if _, ok := delivered[hash]; !ok {
// Not delivered
if i < cutoff {
// Remove origin from candidate sources for partial responses
delete(f.alternates[hash], delivery.origin)
delete(f.announces[delivery.origin], hash)
if len(f.announces[delivery.origin]) == 0 {
delete(f.announces, delivery.origin)
}
}
// Mark cells deliverable by other peers
if f.fetches[hash] != nil {
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(delivery.cellBitmap)
}
}
}
// Something was delivered, try to reschedule requests
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too
case drop := <-f.drop:
// A peer was dropped, remove all traces of it
if _, ok := f.waitslots[drop.peer]; ok {
for hash := range f.waitslots[drop.peer] {
delete(f.waitlist[hash], drop.peer)
if len(f.waitlist[hash]) == 0 {
delete(f.waitlist, hash)
delete(f.waittime, hash)
}
}
delete(f.waitslots, drop.peer)
if len(f.waitlist) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
}
// Clean up general announcement tracking
if _, ok := f.announces[drop.peer]; ok {
for hash := range f.announces[drop.peer] {
delete(f.alternates[hash], drop.peer)
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
}
}
delete(f.announces, drop.peer)
}
delete(f.announces, drop.peer)
// Clean up any active requests
if request, ok := f.requests[drop.peer]; ok && len(request) != 0 {
for _, req := range request {
for _, hash := range req.txs {
// Undelivered hash, reschedule if there's an alternative origin available
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(req.cells)
delete(f.alternates[hash], drop.peer)
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
delete(f.fetches, hash)
}
}
}
delete(f.requests, drop.peer)
// If a request was cancelled, check if anything needs to be rescheduled
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
}
case <-f.quit:
return
}
// Loop did something, ping the step notifier if needed (tests)
if f.step != nil {
f.step <- struct{}{}
}
}
}
func (f *BlobFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) {
if *timer != nil {
(*timer).Stop()
}
now := f.clock.Now()
earliest := now
for _, instance := range f.waittime {
if earliest > instance {
earliest = instance
if txArriveTimeout-time.Duration(now-earliest) < txGatherSlack {
break
}
}
}
*timer = f.clock.AfterFunc(txArriveTimeout-time.Duration(now-earliest), func() {
trigger <- struct{}{}
})
}
// Exactly same as the one in TxFetcher
func (f *BlobFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) {
if *timer != nil {
(*timer).Stop()
}
now := f.clock.Now()
earliest := now
for _, requests := range f.requests {
for _, req := range requests {
// If this request already timed out, skip it altogether
if req.txs == nil {
continue
}
if earliest > req.time {
earliest = req.time
if blobFetchTimeout-time.Duration(now-earliest) < txGatherSlack {
break
}
}
}
}
*timer = f.clock.AfterFunc(blobFetchTimeout-time.Duration(now-earliest), func() {
trigger <- struct{}{}
})
}
func (f *BlobFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, whitelist map[string]struct{}) {
// Gather the set of peers we want to retrieve from (default to all)
actives := whitelist
if actives == nil {
actives = make(map[string]struct{})
for peer := range f.announces {
actives[peer] = struct{}{}
}
}
if len(actives) == 0 {
return
}
// For each active peer, try to schedule some payload fetches
idle := len(f.requests) == 0
f.forEachPeer(actives, func(peer string) {
if len(f.announces[peer]) == 0 || len(f.requests[peer]) != 0 {
return // continue
}
var (
hashes = make([]common.Hash, 0, maxTxRetrievals)
custodies = make([]*types.CustodyBitmap, 0, maxTxRetrievals)
)
f.forEachAnnounce(f.announces[peer], func(hash common.Hash, cells *types.CustodyBitmap) bool {
var difference *types.CustodyBitmap
if f.fetches[hash] == nil {
// tx is not being fetched
difference = cells
} else {
difference = cells.Difference(f.fetches[hash].fetching)
}
// Mark fetching for differences
if difference.OneCount() != 0 {
if f.fetches[hash] == nil {
f.fetches[hash] = &fetchStatus{
fetching: difference,
fetched: make([]uint64, 0),
cells: make([]kzg4844.Cell, 0),
}
} else {
f.fetches[hash].fetching = f.fetches[hash].fetching.Union(difference)
}
// Accumulate the hash and stop if the limit was reached
hashes = append(hashes, hash)
custodies = append(custodies, difference)
}
// Mark alternatives
if f.alternates[hash] == nil {
f.alternates[hash] = map[string]*types.CustodyBitmap{
peer: cells,
}
} else {
f.alternates[hash][peer] = cells
}
return len(hashes) < maxPayloadRetrievals
})
// If any hashes were allocated, request them from the peer
if len(hashes) > 0 {
// Group hashes by custody bitmap
requestByCustody := make(map[string]*cellRequest)
for i, hash := range hashes {
custody := custodies[i]
key := string(custody[:])
if _, ok := requestByCustody[key]; !ok {
requestByCustody[key] = &cellRequest{
txs: []common.Hash{},
cells: custody,
time: f.clock.Now(),
}
}
requestByCustody[key].txs = append(requestByCustody[key].txs, hash)
}
// construct request
var request []*cellRequest
for _, cr := range requestByCustody {
request = append(request, cr)
}
f.requests[peer] = request
go func(peer string, request []*cellRequest) {
for _, req := range request {
if err := f.fetchPayloads(peer, req.txs, req.cells); err != nil {
f.Drop(peer)
break
}
}
}(peer, request)
}
})
// If a new request was fired, schedule a timeout timer
if idle && len(f.requests) > 0 {
f.rescheduleTimeout(timer, timeout)
}
}
// forEachAnnounce loops over the given announcements in arrival order, invoking
// the do function for each until it returns false. We enforce an arrival
// ordering to minimize the chances of transaction nonce-gaps, which result in
// transactions being rejected by the txpool.
func (f *BlobFetcher) forEachAnnounce(announces map[common.Hash]*cellWithSeq, do func(hash common.Hash, cells *types.CustodyBitmap) bool) {
type announcement struct {
hash common.Hash
cells *types.CustodyBitmap
seq uint64
}
// Process announcements by their arrival order
list := make([]announcement, 0, len(announces))
for hash, entry := range announces {
list = append(list, announcement{hash: hash, cells: entry.cells, seq: entry.seq})
}
sort.Slice(list, func(i, j int) bool {
return list[i].seq < list[j].seq
})
for i := range list {
if !do(list[i].hash, list[i].cells) {
return
}
}
}
// forEachPeer does a range loop over a map of peers in production, but during
// testing it does a deterministic sorted random to allow reproducing issues.
func (f *BlobFetcher) forEachPeer(peers map[string]struct{}, do func(peer string)) {
// If we're running production(step == nil), use whatever Go's map gives us
if f.step == nil {
for peer := range peers {
do(peer)
}
return
}
// We're running the test suite, make iteration deterministic (sorted by peer id)
list := make([]string, 0, len(peers))
for peer := range peers {
list = append(list, peer)
}
sort.Strings(list)
for _, peer := range list {
do(peer)
}
}

View file

@ -0,0 +1,999 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package fetcher
import (
"slices"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
// makeTestBlobSidecar is a helper method to create random blob sidecar
// with certain number of blobs.
func makeTestCellSidecar(blobCount int) *types.BlobTxCellSidecar {
var (
blobs []kzg4844.Blob
commitments []kzg4844.Commitment
proofs []kzg4844.Proof
)
for i := 0; i < blobCount; i++ {
blob := &kzg4844.Blob{}
blob[0] = byte(i)
blobs = append(blobs, *blob)
commit, _ := kzg4844.BlobToCommitment(blob)
commitments = append(commitments, commit)
cellProofs, _ := kzg4844.ComputeCellProofs(blob)
proofs = append(proofs, cellProofs...)
}
sidecar, _ := types.NewBlobTxSidecar(types.BlobSidecarVersion1, blobs, commitments, proofs).ToBlobTxCellSidecar()
return sidecar
}
func selectCells(cells []kzg4844.Cell, custody *types.CustodyBitmap) []kzg4844.Cell {
custodyIndices := custody.Indices()
result := make([]kzg4844.Cell, 0)
for _, idx := range custodyIndices {
result = append(result, cells[idx])
}
return result
}
const (
testBlobAvailabilityTimeout = 500 * time.Millisecond
testBlobFetchTimeout = 5 * time.Second
)
var (
testBlobTxHashes = []common.Hash{
{0x01}, {0x02}, {0x03}, {0x04}, {0x05}, {0x06}, {0x07}, {0x08},
}
testBlobSidecars = []*types.BlobTxCellSidecar{
makeTestCellSidecar(1),
makeTestCellSidecar(2),
makeTestCellSidecar(3),
makeTestCellSidecar(4),
}
custody = types.NewCustodyBitmap([]uint64{0, 1, 2, 3, 4, 5, 6, 7})
fullCustody = *types.CustodyBitmapAll
halfCustody = *types.CustodyBitmapData
frontCustody = types.NewCustodyBitmap([]uint64{0, 1, 2, 3, 8, 9, 10, 11})
backCustody = types.NewCustodyBitmap([]uint64{4, 5, 6, 7, 8, 9, 10, 11})
differentCustody = types.NewCustodyBitmap([]uint64{8, 9, 10, 11, 12, 13, 14, 15})
)
type doBlobNotify struct {
peer string
hashes []common.Hash
custody types.CustodyBitmap
}
type doBlobEnqueue struct {
peer string
hashes []common.Hash
cells [][]kzg4844.Cell
custody types.CustodyBitmap
}
type blobDoFunc func(*BlobFetcher)
type isWaitingAvailability map[common.Hash]map[string]struct{}
type isDecidedFull map[common.Hash]struct{}
type isDecidedPartial map[common.Hash]struct{}
type blobAnnounce struct {
hash common.Hash
custody types.CustodyBitmap
}
type isBlobScheduled struct {
announces map[string][]blobAnnounce // announces에 있는 것들 (peer -> hash+custody)
fetching map[string][]blobAnnounce // requests에 있는 것들 (peer -> hash+custody)
}
type isCompleted []common.Hash
type isDropped []string
type isFetching struct {
hashes map[common.Hash]fetchInfo
}
type fetchInfo struct {
fetching *types.CustodyBitmap
fetched []uint64
}
type blobFetcherTest struct {
init func() *BlobFetcher
steps []interface{}
}
type mockRand struct {
value int
}
func (r *mockRand) Intn(n int) int {
return r.value
}
// TestBlobFetcherFullSchedule tests scheduling full payload decision
// Blob should be fetched immediately when its availability is announced
// by idle peer, if the client decided to pull the full payload
// Additional announcements should be recorded as alternates during the fetch
func TestBlobFetcherFullFetch(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 5}, // to force full requests (5 < 15)
)
},
steps: []interface{}{
// A announced full custody blob (should make full decision & start fetching)
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isDecidedFull{testBlobTxHashes[0]: struct{}{}},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// Same hash announced by another peer(B) -> should be added to alternatives
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
// Announce partial custody by C -> should be ignored
doBlobNotify{peer: "C", hashes: []common.Hash{testBlobTxHashes[1]}, custody: custody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
// Additional hashes announced by A -> should not be fetched
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[1]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}, {hash: testBlobTxHashes[1], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
// Announce of multiple transactions
doBlobNotify{peer: "D", hashes: []common.Hash{testBlobTxHashes[2], testBlobTxHashes[3]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}, {hash: testBlobTxHashes[1], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"D": {{hash: testBlobTxHashes[2], custody: halfCustody}, {hash: testBlobTxHashes[3], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"D": {{hash: testBlobTxHashes[2], custody: halfCustody}, {hash: testBlobTxHashes[3], custody: halfCustody}},
},
},
},
})
}
// TestBlobFetcherPartialFetching tests partial request decision and availability check flow
func TestBlobFetcherPartialFetch(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 20}, // Force partial requests (20 >= 15)
)
},
steps: []interface{}{
// First full announce for tx 0, 1 -> should make partial decision and go to waitlist
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0], testBlobTxHashes[1]}, custody: fullCustody},
isDecidedPartial{testBlobTxHashes[0]: struct{}{}, testBlobTxHashes[1]: struct{}{}},
isWaitingAvailability{testBlobTxHashes[0]: map[string]struct{}{"A": {}}, testBlobTxHashes[1]: map[string]struct{}{"A": {}}},
isBlobScheduled{announces: nil, fetching: nil},
// Partial announce for tx 0 (waitlist) -> should be dropped
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: custody},
isWaitingAvailability{testBlobTxHashes[0]: map[string]struct{}{"A": {}}, testBlobTxHashes[1]: map[string]struct{}{"A": {}}},
isBlobScheduled{announces: nil, fetching: nil},
// Second full announce for tx 0 -> should make tx 0 available & fetched
doBlobNotify{peer: "C", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isWaitingAvailability{testBlobTxHashes[1]: map[string]struct{}{"A": {}}},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"C": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &custody,
fetched: []uint64{},
},
},
},
// Partial announce for tx 0, overlapped custody -> overlapping part should be accepted
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: frontCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"B": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"C": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
// Partial announce for tx 0, with additional custody -> don't update
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: custody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"B": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"C": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
// Partial announce for tx 0, without any overlapped custody -> should be dropped
doBlobNotify{peer: "D", hashes: []common.Hash{testBlobTxHashes[0]}, custody: differentCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"B": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"C": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
},
})
}
// todo wait timeout
// todo drop
// TestBlobFetcherFullDelivery tests cell delivery and fetch completion logic (full fetch)
func TestBlobFetcherFullDelivery(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 5}, // Force full requests for simplicity
)
},
steps: []interface{}{
// Full announce by two peers (A, B) -> schedule fetch
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// All alternates should be clean up on delivery
doBlobEnqueue{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, cells: [][]kzg4844.Cell{selectCells(testBlobSidecars[0].Cells, &halfCustody)}, custody: halfCustody},
isBlobScheduled{announces: nil, fetching: nil},
isFetching{hashes: nil}, // fetches should be empty after completion
isCompleted{testBlobTxHashes[0]},
},
})
}
// TestBlobFetcherPartialDelivery tests cell delivery and fetch completion logic (partial fetch)
func TestBlobFetcherPartialDelivery(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 20},
)
},
steps: []interface{}{
// Full announce by two peers (A, B) -> schedule fetch
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isWaitingAvailability(nil),
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"B": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &custody,
fetched: []uint64{},
},
},
},
// Partial announce by C, D -> alternates
doBlobNotify{peer: "C", hashes: []common.Hash{testBlobTxHashes[0]}, custody: frontCustody},
doBlobNotify{peer: "D", hashes: []common.Hash{testBlobTxHashes[0]}, custody: backCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"B": {{hash: testBlobTxHashes[0], custody: custody}},
"C": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
// Drop A, B -> schedule fetch from C, D
doDrop("A"),
doDrop("B"),
isBlobScheduled{
announces: map[string][]blobAnnounce{
"C": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
fetching: map[string][]blobAnnounce{
"C": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
},
// Delivery from C -> wait for D
doBlobEnqueue{
peer: "C",
hashes: []common.Hash{testBlobTxHashes[0]},
cells: [][]kzg4844.Cell{selectCells(testBlobSidecars[0].Cells, frontCustody.Intersection(&custody))},
custody: *frontCustody.Intersection(&custody),
},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
fetching: map[string][]blobAnnounce{
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &custody,
fetched: frontCustody.Intersection(&custody).Indices(),
},
},
},
// Announce already delivered cells + fetching cells -> leave fetching cells only
doBlobNotify{peer: "E", hashes: []common.Hash{testBlobTxHashes[0]}, custody: custody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
"E": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
},
// Not delivered -> reschedule to E
doWait{time: blobFetchTimeout, step: true},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"E": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"E": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &custody,
fetched: frontCustody.Intersection(&custody).Indices(),
},
},
},
// Delivery from E -> complete
doWait{time: blobFetchTimeout / 2, step: true},
doBlobEnqueue{
peer: "E",
hashes: []common.Hash{testBlobTxHashes[0]},
cells: [][]kzg4844.Cell{selectCells(testBlobSidecars[0].Cells, backCustody.Intersection(&custody))},
custody: *backCustody.Intersection(&custody),
},
isCompleted{testBlobTxHashes[0]},
},
})
}
// TestBlobFetcherAvailabilityTimeout tests availability timeout for partial requests
func TestBlobFetcherAvailabilityTimeout(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 20},
)
},
steps: []interface{}{
// First full announce for tx 0 -> should make partial decision and go to waitlist
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isDecidedPartial{testBlobTxHashes[0]: struct{}{}},
isWaitingAvailability{testBlobTxHashes[0]: map[string]struct{}{"A": {}}},
isBlobScheduled{announces: nil, fetching: nil},
// Run clock for timeout
doWait{time: testBlobAvailabilityTimeout, step: true},
// After timeout, waitlist should be empty
isWaitingAvailability{},
isBlobScheduled{announces: nil, fetching: nil},
},
})
}
// TestBlobFetcherPeerDrop tests peer drop scenarios
func TestBlobFetcherPeerDrop(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 5},
)
},
steps: []interface{}{
// Full announce by peer A -> should schedule fetch
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isDecidedFull{testBlobTxHashes[0]: struct{}{}},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// Another peer B announces same hash -> should be added to alternates
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
// Drop peer A -> should reschedule fetch to peer B
doDrop("A"),
isBlobScheduled{
announces: map[string][]blobAnnounce{
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// Drop peer B -> should drop the transaction, remove all traces
doDrop("B"),
isBlobScheduled{announces: nil, fetching: nil},
isFetching{hashes: nil},
},
})
}
// TestBlobFetcherFetchTimeout tests fetch timeout and rescheduling, full request case
func TestBlobFetcherFetchTimeout(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 5},
)
},
steps: []interface{}{
// Full announce by peer A -> schedule fetch
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isDecidedFull{testBlobTxHashes[0]: struct{}{}},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// Another peer announces same hash -> should be added to alternates
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
// Wait for fetch timeout -> should reschedule to peer B
doWait{time: testBlobFetchTimeout, step: true},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// Wait for timeout -> should drop transaction
doWait{time: testBlobFetchTimeout, step: true},
isBlobScheduled{announces: nil, fetching: nil},
isFetching{hashes: nil},
},
})
}
// testBlobFetcher is the generic test runner for blob fetcher tests
func testBlobFetcher(t *testing.T, tt blobFetcherTest) {
clock := new(mclock.Simulated)
wait := make(chan struct{})
// Create a fetcher and boot it up
fetcher := tt.init()
fetcher.clock = clock
fetcher.step = wait
fetcher.Start()
defer fetcher.Stop()
defer func() {
for {
select {
case <-wait:
default:
return
}
}
}()
// Iterate through all the test steps and execute them
for i, step := range tt.steps {
// Clear the channel if anything is left over
for len(wait) > 0 {
<-wait
}
// Process the next step of the test
switch step := step.(type) {
case doBlobNotify:
if err := fetcher.Notify(step.peer, step.hashes, step.custody); err != nil {
t.Errorf("step %d: failed to notify fetcher: %v", i, err)
return
}
<-wait
case doBlobEnqueue:
if err := fetcher.Enqueue(step.peer, step.hashes, step.cells, step.custody); err != nil {
t.Errorf("step %d: failed to enqueue blobs: %v", i, err)
return
}
<-wait
case blobDoFunc:
step(fetcher)
case isWaitingAvailability:
// Check expected hashes and peers are present
for hash, peers := range step {
if waitPeers, ok := fetcher.waitlist[hash]; !ok {
t.Errorf("step %d: hash %x not in waitlist", i, hash)
return
} else {
// Check expected peers are present
for peer := range peers {
if _, ok := waitPeers[peer]; !ok {
t.Errorf("step %d: peer %s not waiting for hash %x", i, peer, hash)
return
}
}
// Check no unexpected peers are present
for peer := range waitPeers {
if _, ok := peers[peer]; !ok {
t.Errorf("step %d: unexpected peer %s waiting for hash %x", i, peer, hash)
return
}
}
}
}
// Check no unexpected hashes in waitlist
for hash := range fetcher.waitlist {
if _, ok := step[hash]; !ok {
t.Errorf("step %d: unexpected hash %x in waitlist", i, hash)
return
}
}
case isDecidedFull:
for hash := range step {
if _, ok := fetcher.full[hash]; !ok {
t.Errorf("step %d: hash %x not decided for full request", i, hash)
return
}
}
case isDecidedPartial:
for hash := range step {
if _, ok := fetcher.partial[hash]; !ok {
t.Errorf("step %d: hash %x not decided for partial request", i, hash)
return
}
}
case isBlobScheduled:
// todo fetches
// Check tracking (announces) - bidirectional verification
for peer, announces := range step.announces {
peerAnnounces := fetcher.announces[peer]
if peerAnnounces == nil {
t.Errorf("step %d: peer %s missing from announces", i, peer)
continue
}
// Check expected announces are present
for _, ann := range announces {
if cellWithSeq, ok := peerAnnounces[ann.hash]; !ok {
t.Errorf("step %d, peer %s: hash %x missing from announces", i, peer, ann.hash)
} else if *cellWithSeq.cells != ann.custody {
t.Errorf("step %d, peer %s, hash %x: custody mismatch in announces", i, peer, ann.hash)
}
}
// Check no unexpected announces are present
for hash := range peerAnnounces {
found := false
for _, ann := range announces {
if ann.hash == hash {
found = true
break
}
}
if !found {
t.Errorf("step %d, peer %s: unexpected hash %x in announces", i, peer, hash)
}
}
}
// Check no unexpected peers in announces
for peer := range fetcher.announces {
if _, ok := step.announces[peer]; !ok {
t.Errorf("step %d: unexpected peer %s in announces", i, peer)
}
}
// Check fetching (requests)
for peer, requests := range step.fetching {
peerRequests := fetcher.requests[peer]
if peerRequests == nil {
t.Errorf("step %d: peer %s missing from requests", i, peer)
continue
}
// Check expected requests are present
for _, req := range requests {
found := false
for _, cellReq := range peerRequests {
for _, hash := range cellReq.txs {
if hash == req.hash && *cellReq.cells == req.custody {
found = true
break
}
}
if found {
break
}
}
if !found {
t.Errorf("step %d, peer %s: hash %x with custody not found in requests", i, peer, req.hash)
}
}
// (bidirectional) Check no unexpected requests are present
for _, cellReq := range peerRequests {
for _, hash := range cellReq.txs {
found := false
for _, req := range requests {
if req.hash == hash && *cellReq.cells == req.custody {
found = true
break
}
}
if !found {
t.Errorf("step %d, peer %s: unexpected hash %x in requests", i, peer, hash)
}
}
}
}
// Check no unexpected peers in requests
for peer := range fetcher.requests {
if _, ok := step.fetching[peer]; !ok {
t.Errorf("step %d: unexpected peer %s in requests", i, peer)
}
}
// Check internal consistency: alternates should match announces
// For every hash being fetched, alternates should contain all peers who announced it
for _, announces := range step.fetching {
for _, announce := range announces {
hash := announce.hash
alternates := fetcher.alternates[hash]
if alternates == nil {
t.Errorf("step %d: hash %x missing from alternates", i, hash)
continue
}
// Check that all peers with this hash in announces are in alternates with matching custody
for peer, peerAnnounces := range fetcher.announces {
if cellWithSeq := peerAnnounces[hash]; cellWithSeq != nil {
if altCustody, ok := alternates[peer]; !ok {
t.Errorf("step %d, hash %x: peer %s missing from alternates", i, hash, peer)
} else if *altCustody != *cellWithSeq.cells {
t.Errorf("step %d, hash %x, peer %s: custody bitmap mismatch in alternates", i, hash, peer)
}
}
}
// Check that all peers in alternates actually have this hash announced with matching custody
for peer, altCustody := range alternates {
if fetcher.announces[peer] == nil || fetcher.announces[peer][hash] == nil {
t.Errorf("step %d, hash %x: peer %s extra in alternates", i, hash, peer)
} else if cellWithSeq := fetcher.announces[peer][hash]; *cellWithSeq.cells != *altCustody {
t.Errorf("step %d, hash %x, peer %s: custody bitmap mismatch between announces and alternates", i, hash, peer)
}
}
}
}
case isFetching:
// Check expected hashes are present in fetches
for hash, expected := range step.hashes {
if fetchStatus, ok := fetcher.fetches[hash]; !ok {
t.Errorf("step %d: hash %x missing from fetches", i, hash)
} else {
// Check fetching bitmap
if expected.fetching != nil {
if fetchStatus.fetching == nil {
t.Errorf("step %d, hash %x: fetching bitmap is nil", i, hash)
} else if *fetchStatus.fetching != *expected.fetching {
t.Errorf("step %d, hash %x: fetching bitmap mismatch", i, hash)
}
}
// Check fetched indices
if expected.fetched != nil {
if len(fetchStatus.fetched) != len(expected.fetched) {
t.Errorf("step %d, hash %x: fetched length mismatch, got %d, want %d", i, hash, len(fetchStatus.fetched), len(expected.fetched))
} else {
// Sort both slices before comparing
gotFetched := make([]uint64, len(fetchStatus.fetched))
copy(gotFetched, fetchStatus.fetched)
slices.Sort(gotFetched)
expectedFetched := make([]uint64, len(expected.fetched))
copy(expectedFetched, expected.fetched)
slices.Sort(expectedFetched)
if !slices.Equal(gotFetched, expectedFetched) {
t.Errorf("step %d, hash %x: fetched indices mismatch", i, hash)
}
}
}
}
}
// Check no unexpected hashes in fetches
for hash := range fetcher.fetches {
if _, ok := step.hashes[hash]; !ok {
t.Errorf("step %d: unexpected hash %x in fetches", i, hash)
}
}
case isCompleted:
for _, hash := range step {
if _, ok := fetcher.fetches[hash]; ok {
t.Errorf("step %d: hash %x still in fetches (should be completed)", i, hash)
return
}
}
case isDropped:
for _, peer := range step {
if _, ok := fetcher.announces[peer]; ok {
t.Errorf("step %d: peer %s still has announces (should be dropped)", i, peer)
return
}
}
case doWait:
clock.Run(step.time)
if step.step {
<-wait
}
case doDrop:
if err := fetcher.Drop(string(step)); err != nil {
t.Errorf("step %d: %v", i, err)
}
<-wait
default:
t.Errorf("step %d: unknown step type %T", i, step)
return
}
}
}

View file

@ -57,4 +57,25 @@ var (
// to become "unfrozen", either by eventually replying to the request // to become "unfrozen", either by eventually replying to the request
// or by being dropped, measuring from the moment the request was sent. // or by being dropped, measuring from the moment the request was sent.
txFetcherSlowWait = metrics.NewRegisteredHistogram("eth/fetcher/transaction/slow/wait", nil, metrics.NewExpDecaySample(1028, 0.015)) txFetcherSlowWait = metrics.NewRegisteredHistogram("eth/fetcher/transaction/slow/wait", nil, metrics.NewExpDecaySample(1028, 0.015))
blobAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/in", nil)
blobAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/dos", nil)
blobRequestOutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/out", nil)
blobRequestFailMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/fail", nil)
blobRequestDoneMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/done", nil)
blobRequestTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/timeout", nil)
blobReplyInMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/replies/in", nil)
blobReplyRejectMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/replies/reject", nil)
blobFetcherWaitingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/waiting/peers", nil)
blobFetcherWaitingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/waiting/hashes", nil)
blobFetcherQueueingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/queueing/peers", nil)
blobFetcherQueueingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/queueing/hashes", nil)
blobFetcherFetchingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/fetching/peers", nil)
blobFetcherFetchingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/fetching/hashes", nil)
blobFetcherWaitTime = metrics.NewRegisteredHistogram("eth/fetcher/blob/wait/time", nil, metrics.NewExpDecaySample(1028, 0.015))
blobFetcherFetchTime = metrics.NewRegisteredHistogram("eth/fetcher/blob/fetch/time", nil, metrics.NewExpDecaySample(1028, 0.015))
) )

View file

@ -232,7 +232,7 @@ func NewTxFetcherForTests(
// Notify announces the fetcher of the potential availability of a new batch of // Notify announces the fetcher of the potential availability of a new batch of
// transactions in the network. // transactions in the network.
func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []common.Hash) error { func (f *TxFetcher) Notify(peer string, kinds []byte, sizes []uint32, hashes []common.Hash) ([]common.Hash, error) {
// Keep track of all the announced transactions // Keep track of all the announced transactions
txAnnounceInMeter.Mark(int64(len(hashes))) txAnnounceInMeter.Mark(int64(len(hashes)))
@ -245,12 +245,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
unknownHashes = make([]common.Hash, 0, len(hashes)) unknownHashes = make([]common.Hash, 0, len(hashes))
unknownMetas = make([]txMetadata, 0, len(hashes)) unknownMetas = make([]txMetadata, 0, len(hashes))
blobFetchHashes = make([]common.Hash, 0, len(hashes))
duplicate int64 duplicate int64
onchain int64 onchain int64
underpriced int64 underpriced int64
) )
for i, hash := range hashes { for i, hash := range hashes {
err := f.validateMeta(hash, types[i]) err := f.validateMeta(hash, kinds[i])
if errors.Is(err, txpool.ErrAlreadyKnown) { if errors.Is(err, txpool.ErrAlreadyKnown) {
duplicate++ duplicate++
continue continue
@ -271,11 +273,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
} }
unknownHashes = append(unknownHashes, hash) unknownHashes = append(unknownHashes, hash)
if kinds[i] == types.BlobTxType {
blobFetchHashes = append(blobFetchHashes, hash)
}
// Transaction metadata has been available since eth68, and all // Transaction metadata has been available since eth68, and all
// legacy eth protocols (prior to eth68) have been deprecated. // legacy eth protocols (prior to eth68) have been deprecated.
// Therefore, metadata is always expected in the announcement. // Therefore, metadata is always expected in the announcement.
unknownMetas = append(unknownMetas, txMetadata{kind: types[i], size: sizes[i]}) unknownMetas = append(unknownMetas, txMetadata{kind: kinds[i], size: sizes[i]})
} }
txAnnounceKnownMeter.Mark(duplicate) txAnnounceKnownMeter.Mark(duplicate)
txAnnounceUnderpricedMeter.Mark(underpriced) txAnnounceUnderpricedMeter.Mark(underpriced)
@ -283,14 +288,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
// If anything's left to announce, push it into the internal loop // If anything's left to announce, push it into the internal loop
if len(unknownHashes) == 0 { if len(unknownHashes) == 0 {
return nil return blobFetchHashes, nil
} }
announce := &txAnnounce{origin: peer, hashes: unknownHashes, metas: unknownMetas} announce := &txAnnounce{origin: peer, hashes: unknownHashes, metas: unknownMetas}
select { select {
case f.notify <- announce: case f.notify <- announce:
return nil return blobFetchHashes, nil
case <-f.quit: case <-f.quit:
return errTerminated return nil, errTerminated
} }
} }

View file

@ -1888,7 +1888,7 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) {
// Process the original or expanded steps // Process the original or expanded steps
switch step := step.(type) { switch step := step.(type) {
case doTxNotify: case doTxNotify:
if err := fetcher.Notify(step.peer, step.types, step.sizes, step.hashes); err != nil { if _, err := fetcher.Notify(step.peer, step.types, step.sizes, step.hashes); err != nil {
t.Errorf("step %d: %v", i, err) t.Errorf("step %d: %v", i, err)
} }
<-wait // Fetcher needs to process this, wait until it's done <-wait // Fetcher needs to process this, wait until it's done

View file

@ -33,6 +33,7 @@ import (
"github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/fetcher" "github.com/ethereum/go-ethereum/eth/fetcher"
@ -75,7 +76,7 @@ type txPool interface {
// GetRLP retrieves the RLP-encoded transaction from local txpool // GetRLP retrieves the RLP-encoded transaction from local txpool
// with given tx hash. // with given tx hash.
GetRLP(hash common.Hash) []byte GetRLP(hash common.Hash, includeBlob bool) []byte
// GetMetadata returns the transaction type and transaction size with the // GetMetadata returns the transaction type and transaction size with the
// given transaction hash. // given transaction hash.
@ -97,6 +98,16 @@ type txPool interface {
FilterType(kind byte) bool FilterType(kind byte) bool
} }
// blobPool defines the methods needed from a blob pool implementation to
// support cell-based blob data availability.
type blobPool interface {
Has(hash common.Hash) bool
GetCells(hash common.Hash, mask types.CustodyBitmap) ([]kzg4844.Cell, error)
ValidateCells([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error
AddPayload([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error
GetCustody(hash common.Hash) *types.CustodyBitmap
}
// handlerConfig is the collection of initialization parameters to create a full // handlerConfig is the collection of initialization parameters to create a full
// node network handler. // node network handler.
type handlerConfig struct { type handlerConfig struct {
@ -104,11 +115,13 @@ type handlerConfig struct {
Database ethdb.Database // Database for direct sync insertions Database ethdb.Database // Database for direct sync insertions
Chain *core.BlockChain // Blockchain to serve data from Chain *core.BlockChain // Blockchain to serve data from
TxPool txPool // Transaction pool to propagate from TxPool txPool // Transaction pool to propagate from
BlobPool blobPool // Blob pool for cell-based blob data availability
Network uint64 // Network identifier to advertise Network uint64 // Network identifier to advertise
Sync ethconfig.SyncMode // Whether to snap or full sync Sync ethconfig.SyncMode // Whether to snap or full sync
BloomCache uint64 // Megabytes to alloc for snap sync bloom BloomCache uint64 // Megabytes to alloc for snap sync bloom
EventMux *event.TypeMux // Legacy event mux, deprecate for `feed` EventMux *event.TypeMux // Legacy event mux, deprecate for `feed`
RequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges RequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges
Custody types.CustodyBitmap
} }
type handler struct { type handler struct {
@ -118,11 +131,13 @@ type handler struct {
database ethdb.Database database ethdb.Database
txpool txPool txpool txPool
blobpool blobPool
chain *core.BlockChain chain *core.BlockChain
maxPeers int maxPeers int
downloader *downloader.Downloader downloader *downloader.Downloader
txFetcher *fetcher.TxFetcher txFetcher *fetcher.TxFetcher
blobFetcher *fetcher.BlobFetcher
peers *peerSet peers *peerSet
txBroadcastKey [16]byte txBroadcastKey [16]byte
@ -154,6 +169,7 @@ func newHandler(config *handlerConfig) (*handler, error) {
eventMux: config.EventMux, eventMux: config.EventMux,
database: config.Database, database: config.Database,
txpool: config.TxPool, txpool: config.TxPool,
blobpool: config.BlobPool,
chain: config.Chain, chain: config.Chain,
peers: newPeerSet(), peers: newPeerSet(),
txBroadcastKey: newBroadcastChoiceKey(), txBroadcastKey: newBroadcastChoiceKey(),
@ -189,6 +205,16 @@ func newHandler(config *handlerConfig) (*handler, error) {
return nil return nil
} }
h.txFetcher = fetcher.NewTxFetcher(h.chain, validateMeta, addTxs, fetchTx, h.removePeer) h.txFetcher = fetcher.NewTxFetcher(h.chain, validateMeta, addTxs, fetchTx, h.removePeer)
// Construct the blob fetcher for cell-based blob data availability
fetchPayloads := func(peer string, hashes []common.Hash, cells *types.CustodyBitmap) error {
p := h.peers.peer(peer)
if p == nil {
return errors.New("unknown peer")
}
return p.RequestPayload(hashes, cells)
}
h.blobFetcher = fetcher.NewBlobFetcher(h.blobpool.ValidateCells, h.blobpool.AddPayload, fetchPayloads, h.removePeer, &config.Custody, nil)
return h, nil return h, nil
} }
@ -403,6 +429,7 @@ func (h *handler) unregisterPeer(id string) {
} }
h.downloader.UnregisterPeer(id) h.downloader.UnregisterPeer(id)
h.txFetcher.Drop(id) h.txFetcher.Drop(id)
h.blobFetcher.Drop(id)
if err := h.peers.unregisterPeer(id); err != nil { if err := h.peers.unregisterPeer(id); err != nil {
logger.Error("Ethereum peer removal failed", "err", err) logger.Error("Ethereum peer removal failed", "err", err)
@ -425,6 +452,7 @@ func (h *handler) Start(maxPeers int) {
// start sync handlers // start sync handlers
h.txFetcher.Start() h.txFetcher.Start()
h.blobFetcher.Start()
// start peer handler tracker // start peer handler tracker
h.wg.Add(1) h.wg.Add(1)
@ -435,6 +463,7 @@ func (h *handler) Stop() {
h.txsSub.Unsubscribe() // quits txBroadcastLoop h.txsSub.Unsubscribe() // quits txBroadcastLoop
h.blockRange.stop() h.blockRange.stop()
h.txFetcher.Stop() h.txFetcher.Stop()
h.blobFetcher.Stop()
h.downloader.Terminate() h.downloader.Terminate()
// Quit chainSync and txsync64. // Quit chainSync and txsync64.

View file

@ -33,6 +33,7 @@ type ethHandler handler
func (h *ethHandler) Chain() *core.BlockChain { return h.chain } func (h *ethHandler) Chain() *core.BlockChain { return h.chain }
func (h *ethHandler) TxPool() eth.TxPool { return h.txpool } func (h *ethHandler) TxPool() eth.TxPool { return h.txpool }
func (h *ethHandler) BlobPool() eth.BlobPool { return h.blobpool }
// RunPeer is invoked when a peer joins on the `eth` protocol. // RunPeer is invoked when a peer joins on the `eth` protocol.
func (h *ethHandler) RunPeer(peer *eth.Peer, hand eth.Handler) error { func (h *ethHandler) RunPeer(peer *eth.Peer, hand eth.Handler) error {
@ -58,8 +59,19 @@ func (h *ethHandler) AcceptTxs() bool {
func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error { func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
// Consume any broadcasts and announces, forwarding the rest to the downloader // Consume any broadcasts and announces, forwarding the rest to the downloader
switch packet := packet.(type) { switch packet := packet.(type) {
case *eth.NewPooledTransactionHashesPacket: case *eth.NewPooledTransactionHashesPacket71:
return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes) hashes, err := h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
if err != nil {
return err
}
if len(hashes) != 0 {
return h.blobFetcher.Notify(peer.ID(), hashes, packet.Mask)
}
return nil
case *eth.NewPooledTransactionHashesPacket70:
_, err := h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
return err
case *eth.TransactionsPacket: case *eth.TransactionsPacket:
txs, err := packet.Items() txs, err := packet.Items()
@ -81,6 +93,9 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
} }
return h.txFetcher.Enqueue(peer.ID(), txs, true) return h.txFetcher.Enqueue(peer.ID(), txs, true)
case *eth.CellsResponse:
return h.blobFetcher.Enqueue(peer.ID(), packet.Hashes, packet.Cells, packet.Mask)
default: default:
return fmt.Errorf("unexpected eth packet type: %T", packet) return fmt.Errorf("unexpected eth packet type: %T", packet)
} }
@ -98,6 +113,11 @@ func handleTransactions(peer *eth.Peer, list []*types.Transaction, directBroadca
// If we receive any blob transactions missing sidecars, or with // If we receive any blob transactions missing sidecars, or with
// sidecars that don't correspond to the versioned hashes reported // sidecars that don't correspond to the versioned hashes reported
// in the header, disconnect from the sending peer. // in the header, disconnect from the sending peer.
if peer.Version() >= eth.ETH71 {
if tx.BlobTxSidecar() != nil && len(tx.BlobTxSidecar().Blobs) != 0 {
return fmt.Errorf("not allowed to respond with full-blob transaction under eth71")
}
} else {
if tx.BlobTxSidecar() == nil { if tx.BlobTxSidecar() == nil {
return errors.New("received sidecar-less blob transaction") return errors.New("received sidecar-less blob transaction")
} }
@ -106,6 +126,7 @@ func handleTransactions(peer *eth.Peer, list []*types.Transaction, directBroadca
} }
} }
} }
}
// Check for duplicates. // Check for duplicates.
hash := tx.Hash() hash := tx.Hash()

View file

@ -44,13 +44,14 @@ type testEthHandler struct {
func (h *testEthHandler) Chain() *core.BlockChain { panic("no backing chain") } func (h *testEthHandler) Chain() *core.BlockChain { panic("no backing chain") }
func (h *testEthHandler) TxPool() eth.TxPool { panic("no backing tx pool") } func (h *testEthHandler) TxPool() eth.TxPool { panic("no backing tx pool") }
func (h *testEthHandler) BlobPool() eth.BlobPool { return nil }
func (h *testEthHandler) AcceptTxs() bool { return true } func (h *testEthHandler) AcceptTxs() bool { return true }
func (h *testEthHandler) RunPeer(*eth.Peer, eth.Handler) error { panic("not used in tests") } func (h *testEthHandler) RunPeer(*eth.Peer, eth.Handler) error { panic("not used in tests") }
func (h *testEthHandler) PeerInfo(enode.ID) interface{} { panic("not used in tests") } func (h *testEthHandler) PeerInfo(enode.ID) interface{} { panic("not used in tests") }
func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
switch packet := packet.(type) { switch packet := packet.(type) {
case *eth.NewPooledTransactionHashesPacket: case *eth.NewPooledTransactionHashesPacket70:
h.txAnnounces.Send(packet.Hashes) h.txAnnounces.Send(packet.Hashes)
return nil return nil
@ -105,10 +106,12 @@ func testForkIDSplit(t *testing.T, protocol uint) {
_, blocksNoFork, _ = core.GenerateChainWithGenesis(gspecNoFork, engine, 2, nil) _, blocksNoFork, _ = core.GenerateChainWithGenesis(gspecNoFork, engine, 2, nil)
_, blocksProFork, _ = core.GenerateChainWithGenesis(gspecProFork, engine, 2, nil) _, blocksProFork, _ = core.GenerateChainWithGenesis(gspecProFork, engine, 2, nil)
txPool = newTestTxPool()
ethNoFork, _ = newHandler(&handlerConfig{ ethNoFork, _ = newHandler(&handlerConfig{
Database: dbNoFork, Database: dbNoFork,
Chain: chainNoFork, Chain: chainNoFork,
TxPool: newTestTxPool(), TxPool: txPool,
BlobPool: txPool,
Network: 1, Network: 1,
Sync: ethconfig.FullSync, Sync: ethconfig.FullSync,
BloomCache: 1, BloomCache: 1,
@ -116,7 +119,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
ethProFork, _ = newHandler(&handlerConfig{ ethProFork, _ = newHandler(&handlerConfig{
Database: dbProFork, Database: dbProFork,
Chain: chainProFork, Chain: chainProFork,
TxPool: newTestTxPool(), TxPool: txPool,
BlobPool: txPool,
Network: 1, Network: 1,
Sync: ethconfig.FullSync, Sync: ethconfig.FullSync,
BloomCache: 1, BloomCache: 1,
@ -137,8 +141,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close() defer p2pNoFork.Close()
defer p2pProFork.Close() defer p2pProFork.Close()
peerNoFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil) peerNoFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil)
peerProFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil) peerProFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil)
defer peerNoFork.Close() defer peerNoFork.Close()
defer peerProFork.Close() defer peerProFork.Close()
@ -168,8 +172,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close() defer p2pNoFork.Close()
defer p2pProFork.Close() defer p2pProFork.Close()
peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil, nil)
defer peerNoFork.Close() defer peerNoFork.Close()
defer peerProFork.Close() defer peerProFork.Close()
@ -199,8 +203,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close() defer p2pNoFork.Close()
defer p2pProFork.Close() defer p2pProFork.Close()
peerNoFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil) peerNoFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil) peerProFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil)
defer peerNoFork.Close() defer peerNoFork.Close()
defer peerProFork.Close() defer peerProFork.Close()
@ -249,8 +253,8 @@ func testRecvTransactions(t *testing.T, protocol uint) {
defer p2pSrc.Close() defer p2pSrc.Close()
defer p2pSink.Close() defer p2pSink.Close()
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool) src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, handler.txpool)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool) sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, handler.txpool)
defer src.Close() defer src.Close()
defer sink.Close() defer sink.Close()
@ -305,8 +309,8 @@ func testSendTransactions(t *testing.T, protocol uint) {
defer p2pSrc.Close() defer p2pSrc.Close()
defer p2pSink.Close() defer p2pSink.Close()
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool) src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, handler.blobpool)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool) sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, handler.blobpool)
defer src.Close() defer src.Close()
defer sink.Close() defer sink.Close()
@ -380,8 +384,8 @@ func testTransactionPropagation(t *testing.T, protocol uint) {
defer sourcePipe.Close() defer sourcePipe.Close()
defer sinkPipe.Close() defer sinkPipe.Close()
sourcePeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{byte(i + 1)}, "", nil, sourcePipe), sourcePipe, source.txpool) sourcePeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{byte(i + 1)}, "", nil, sourcePipe), sourcePipe, source.txpool, source.txpool)
sinkPeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, sink.txpool) sinkPeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, sink.txpool, sink.txpool)
defer sourcePeer.Close() defer sourcePeer.Close()
defer sinkPeer.Close() defer sinkPeer.Close()

View file

@ -17,6 +17,7 @@
package eth package eth
import ( import (
"errors"
"maps" "maps"
"math/big" "math/big"
"math/rand" "math/rand"
@ -31,6 +32,7 @@ import (
"github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
@ -54,7 +56,10 @@ var (
// Its goal is to get around setting up a valid statedb for the balance and nonce // Its goal is to get around setting up a valid statedb for the balance and nonce
// checks. // checks.
type testTxPool struct { type testTxPool struct {
pool map[common.Hash]*types.Transaction // Hash map of collected transactions txPool map[common.Hash]*types.Transaction // Hash map of collected transactions
cellPool map[common.Hash][]kzg4844.Cell
custody map[common.Hash]types.CustodyBitmap
txFeed event.Feed // Notification feed to allow waiting for inclusion txFeed event.Feed // Notification feed to allow waiting for inclusion
lock sync.RWMutex // Protects the transaction pool lock sync.RWMutex // Protects the transaction pool
@ -63,7 +68,9 @@ type testTxPool struct {
// newTestTxPool creates a mock transaction pool. // newTestTxPool creates a mock transaction pool.
func newTestTxPool() *testTxPool { func newTestTxPool() *testTxPool {
return &testTxPool{ return &testTxPool{
pool: make(map[common.Hash]*types.Transaction), txPool: make(map[common.Hash]*types.Transaction),
cellPool: make(map[common.Hash][]kzg4844.Cell),
custody: make(map[common.Hash]types.CustodyBitmap),
} }
} }
@ -73,7 +80,7 @@ func (p *testTxPool) Has(hash common.Hash) bool {
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()
return p.pool[hash] != nil return p.txPool[hash] != nil
} }
// Get retrieves the transaction from local txpool with given // Get retrieves the transaction from local txpool with given
@ -81,16 +88,16 @@ func (p *testTxPool) Has(hash common.Hash) bool {
func (p *testTxPool) Get(hash common.Hash) *types.Transaction { func (p *testTxPool) Get(hash common.Hash) *types.Transaction {
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()
return p.pool[hash] return p.txPool[hash]
} }
// Get retrieves the transaction from local txpool with given // Get retrieves the transaction from local txpool with given
// tx hash. // tx hash.
func (p *testTxPool) GetRLP(hash common.Hash) []byte { func (p *testTxPool) GetRLP(hash common.Hash, includeBlob bool) []byte {
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()
tx := p.pool[hash] tx := p.txPool[hash]
if tx != nil { if tx != nil {
blob, _ := rlp.EncodeToBytes(tx) blob, _ := rlp.EncodeToBytes(tx)
return blob return blob
@ -104,7 +111,7 @@ func (p *testTxPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
p.lock.Lock() p.lock.Lock()
defer p.lock.Unlock() defer p.lock.Unlock()
tx := p.pool[hash] tx := p.txPool[hash]
if tx != nil { if tx != nil {
return &txpool.TxMetadata{ return &txpool.TxMetadata{
Type: tx.Type(), Type: tx.Type(),
@ -121,7 +128,7 @@ func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error {
defer p.lock.Unlock() defer p.lock.Unlock()
for _, tx := range txs { for _, tx := range txs {
p.pool[tx.Hash()] = tx p.txPool[tx.Hash()] = tx
} }
p.txFeed.Send(core.NewTxsEvent{Txs: txs}) p.txFeed.Send(core.NewTxsEvent{Txs: txs})
return make([]error, len(txs)) return make([]error, len(txs))
@ -134,7 +141,7 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]
var count int var count int
batches := make(map[common.Address][]*types.Transaction) batches := make(map[common.Address][]*types.Transaction)
for _, tx := range p.pool { for _, tx := range p.txPool {
from, _ := types.Sender(types.HomesteadSigner{}, tx) from, _ := types.Sender(types.HomesteadSigner{}, tx)
batches[from] = append(batches[from], tx) batches[from] = append(batches[from], tx)
} }
@ -164,6 +171,68 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]
func (p *testTxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool) event.Subscription { func (p *testTxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool) event.Subscription {
return p.txFeed.Subscribe(ch) return p.txFeed.Subscribe(ch)
} }
func (p *testTxPool) GetCells(hash common.Hash, mask types.CustodyBitmap) ([]kzg4844.Cell, error) {
p.lock.RLock()
defer p.lock.RUnlock()
_, exists := p.txPool[hash]
if !exists {
return nil, errors.New("Requested tx does not exist")
}
var cells []kzg4844.Cell
if cells, exists = p.cellPool[hash]; !exists {
return nil, errors.New("Requested cells do not exist")
}
result := make([]kzg4844.Cell, 0, mask.OneCount())
for _, idx := range mask.Indices() {
if int(idx) < len(cells) {
result = append(result, cells[idx])
}
}
return result, nil
}
func (p *testTxPool) GetCustody(hash common.Hash) *types.CustodyBitmap {
p.lock.RLock()
defer p.lock.RUnlock()
mask, ok := p.custody[hash]
if !ok {
return nil
}
return &mask
}
// AddCells adds cells for a specific transaction hash (for testing)
func (p *testTxPool) AddCells(hash common.Hash, cells []kzg4844.Cell, mask types.CustodyBitmap) {
p.lock.Lock()
defer p.lock.Unlock()
p.cellPool[hash] = cells
p.custody[hash] = mask
}
func (p *testTxPool) AddPayload(txs []common.Hash, cells [][]kzg4844.Cell, custody *types.CustodyBitmap) []error {
p.lock.Lock()
defer p.lock.Unlock()
for i, tx := range txs {
p.cellPool[tx] = cells[i]
}
return nil
}
func (p *testTxPool) ValidateCells(txs []common.Hash, cells [][]kzg4844.Cell, custody *types.CustodyBitmap) []error {
p.lock.Lock()
defer p.lock.Unlock()
errors := make([]error, len(txs))
for i, tx := range txs {
errors[i] = kzg4844.VerifyCells(cells[i], p.txPool[tx].BlobTxSidecar().Commitments, p.txPool[tx].BlobTxSidecar().Proofs, custody.Indices())
}
return errors
}
// FilterType should check whether the pool supports the given type of transactions. // FilterType should check whether the pool supports the given type of transactions.
func (p *testTxPool) FilterType(kind byte) bool { func (p *testTxPool) FilterType(kind byte) bool {
@ -181,6 +250,7 @@ type testHandler struct {
db ethdb.Database db ethdb.Database
chain *core.BlockChain chain *core.BlockChain
txpool *testTxPool txpool *testTxPool
blobpool *testTxPool
handler *handler handler *handler
} }
@ -210,6 +280,7 @@ func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler
Database: db, Database: db,
Chain: chain, Chain: chain,
TxPool: txpool, TxPool: txpool,
BlobPool: txpool,
Network: 1, Network: 1,
Sync: mode, Sync: mode,
BloomCache: 1, BloomCache: 1,
@ -220,6 +291,7 @@ func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler
db: db, db: db,
chain: chain, chain: chain,
txpool: txpool, txpool: txpool,
blobpool: txpool,
handler: handler, handler: handler,
} }
} }
@ -317,7 +389,7 @@ func createTestPeers(rand *rand.Rand, n int) []*ethPeer {
var id enode.ID var id enode.ID
rand.Read(id[:]) rand.Read(id[:])
p2pPeer := p2p.NewPeer(id, "test", nil) p2pPeer := p2p.NewPeer(id, "test", nil)
ep := eth.NewPeer(eth.ETH69, p2pPeer, nil, nil) ep := eth.NewPeer(eth.ETH69, p2pPeer, nil, nil, nil)
peers[i] = &ethPeer{Peer: ep} peers[i] = &ethPeer{Peer: ep}
} }
return peers return peers

View file

@ -113,29 +113,51 @@ func (p *Peer) announceTransactions() {
pending []common.Hash pending []common.Hash
pendingTypes []byte pendingTypes []byte
pendingSizes []uint32 pendingSizes []uint32
mask types.CustodyBitmap
size common.StorageSize size common.StorageSize
processed = make(map[int]bool)
) )
for count = 0; count < len(queue) && size < maxTxPacketSize; count++ { for count = 0; count < len(queue) && size < maxTxPacketSize; count++ {
if meta := p.txpool.GetMetadata(queue[count]); meta != nil { if meta := p.txpool.GetMetadata(queue[count]); meta != nil {
custody := p.blobpool.GetCustody(queue[count])
if custody != nil {
// blob tx
if mask.OneCount() == 0 {
mask = *custody
} else {
if mask != *custody {
// group by mask
continue
}
}
}
pending = append(pending, queue[count]) pending = append(pending, queue[count])
pendingTypes = append(pendingTypes, meta.Type) pendingTypes = append(pendingTypes, meta.Type)
pendingSizes = append(pendingSizes, uint32(meta.Size)) pendingSizes = append(pendingSizes, uint32(meta.Size))
size += common.HashLength size += common.HashLength
processed[count] = true
} }
} }
// Shift and trim queue // Shift and trim queue using processed map
queue = queue[:copy(queue, queue[count:])] var remaining []common.Hash
for i, h := range queue {
if !processed[i] {
remaining = append(remaining, h)
}
}
queue = remaining
// If there's anything available to transfer, fire up an async writer // If there's anything available to transfer, fire up an async writer
if len(pending) > 0 { if len(pending) > 0 {
done = make(chan struct{}) done = make(chan struct{})
go func() { go func() {
if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes); err != nil { if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes, mask); err != nil {
fail <- err fail <- err
return return
} }
close(done) close(done)
p.Log().Trace("Sent transaction announcements", "count", len(pending)) p.Log().Trace("Sent transaction announcements", "count", len(pending), "mask", mask, "tx", pending)
}() }()
} }
} }

View file

@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enode"
@ -64,6 +65,9 @@ type Backend interface {
// TxPool retrieves the transaction pool object to serve data. // TxPool retrieves the transaction pool object to serve data.
TxPool() TxPool TxPool() TxPool
// BlobPool retrieves the blob pool object to serve cell requests.
BlobPool() BlobPool
// AcceptTxs retrieves whether transaction processing is enabled on the node // AcceptTxs retrieves whether transaction processing is enabled on the node
// or if inbound transactions should simply be dropped. // or if inbound transactions should simply be dropped.
AcceptTxs() bool AcceptTxs() bool
@ -83,6 +87,16 @@ type Backend interface {
Handle(peer *Peer, packet Packet) error Handle(peer *Peer, packet Packet) error
} }
// BlobPool defines the methods needed by the protocol handler to serve cell requests.
type BlobPool interface {
// GetCells retrieves cells for a given transaction hash filtered by the custody bitmap.
GetCells(hash common.Hash, mask types.CustodyBitmap) ([]kzg4844.Cell, error)
// GetCustody returns the custody bitmap for a given transaction hash.
GetCustody(hash common.Hash) *types.CustodyBitmap
// Has returns whether the blob pool contains a transaction with the given hash.
Has(hash common.Hash) bool
}
// TxPool defines the methods needed by the protocol handler to serve transactions. // TxPool defines the methods needed by the protocol handler to serve transactions.
type TxPool interface { type TxPool interface {
// Get retrieves the transaction from the local txpool with the given hash. // Get retrieves the transaction from the local txpool with the given hash.
@ -90,7 +104,7 @@ type TxPool interface {
// GetRLP retrieves the RLP-encoded transaction from the local txpool with // GetRLP retrieves the RLP-encoded transaction from the local txpool with
// the given hash. // the given hash.
GetRLP(hash common.Hash) []byte GetRLP(hash common.Hash, includeBlob bool) []byte
// GetMetadata returns the transaction type and transaction size with the // GetMetadata returns the transaction type and transaction size with the
// given transaction hash. // given transaction hash.
@ -106,7 +120,7 @@ func MakeProtocols(backend Backend, network uint64, disc enode.Iterator) []p2p.P
Version: version, Version: version,
Length: protocolLengths[version], Length: protocolLengths[version],
Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
peer := NewPeer(version, p, rw, backend.TxPool()) peer := NewPeer(version, p, rw, backend.TxPool(), backend.BlobPool())
defer peer.Close() defer peer.Close()
return backend.RunPeer(peer, func(peer *Peer) error { return backend.RunPeer(peer, func(peer *Peer) error {
@ -180,6 +194,22 @@ var eth69 = map[uint64]msgHandler{
BlockRangeUpdateMsg: handleBlockRangeUpdate, BlockRangeUpdateMsg: handleBlockRangeUpdate,
} }
var eth71 = map[uint64]msgHandler{
TransactionsMsg: handleTransactions,
NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes71,
GetBlockHeadersMsg: handleGetBlockHeaders,
BlockHeadersMsg: handleBlockHeaders,
GetBlockBodiesMsg: handleGetBlockBodies,
BlockBodiesMsg: handleBlockBodies,
GetReceiptsMsg: handleGetReceipts,
ReceiptsMsg: handleReceipts,
GetPooledTransactionsMsg: handleGetPooledTransactions,
PooledTransactionsMsg: handlePooledTransactions,
BlockRangeUpdateMsg: handleBlockRangeUpdate,
GetCellsMsg: handleGetCells,
CellsMsg: handleCells,
}
// handleMessage is invoked whenever an inbound message is received from a remote // handleMessage is invoked whenever an inbound message is received from a remote
// peer. The remote connection is torn down upon returning any error. // peer. The remote connection is torn down upon returning any error.
func handleMessage(backend Backend, peer *Peer) error { func handleMessage(backend Backend, peer *Peer) error {
@ -194,9 +224,12 @@ func handleMessage(backend Backend, peer *Peer) error {
defer msg.Discard() defer msg.Discard()
var handlers map[uint64]msgHandler var handlers map[uint64]msgHandler
if peer.version == ETH69 { switch peer.version {
case ETH69:
handlers = eth69 handlers = eth69
} else { case ETH71:
handlers = eth71
default:
return fmt.Errorf("unknown eth protocol version: %v", peer.version) return fmt.Errorf("unknown eth protocol version: %v", peer.version)
} }

View file

@ -65,6 +65,7 @@ type testBackend struct {
db ethdb.Database db ethdb.Database
chain *core.BlockChain chain *core.BlockChain
txpool *txpool.TxPool txpool *txpool.TxPool
blobpool *blobpool.BlobPool
} }
// newTestBackend creates an empty chain and wraps it into a mock backend. // newTestBackend creates an empty chain and wraps it into a mock backend.
@ -145,6 +146,7 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, cancun bool, generat
db: db, db: db,
chain: chain, chain: chain,
txpool: txpool, txpool: txpool,
blobpool: blobPool,
} }
} }
@ -156,6 +158,7 @@ func (b *testBackend) close() {
func (b *testBackend) Chain() *core.BlockChain { return b.chain } func (b *testBackend) Chain() *core.BlockChain { return b.chain }
func (b *testBackend) TxPool() TxPool { return b.txpool } func (b *testBackend) TxPool() TxPool { return b.txpool }
func (b *testBackend) BlobPool() BlobPool { return b.blobpool }
func (b *testBackend) RunPeer(peer *Peer, handler Handler) error { func (b *testBackend) RunPeer(peer *Peer, handler Handler) error {
// Normally the backend would do peer maintenance and handshakes. All that // Normally the backend would do peer maintenance and handshakes. All that

View file

@ -26,6 +26,7 @@ import (
"github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/tracker" "github.com/ethereum/go-ethereum/p2p/tracker"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
@ -482,7 +483,27 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer)
if !backend.AcceptTxs() { if !backend.AcceptTxs() {
return nil return nil
} }
ann := new(NewPooledTransactionHashesPacket) ann := new(NewPooledTransactionHashesPacket70)
if err := msg.Decode(ann); err != nil {
return err
}
if len(ann.Hashes) != len(ann.Types) || len(ann.Hashes) != len(ann.Sizes) {
return fmt.Errorf("NewPooledTransactionHashes: invalid len of fields in %v %v %v", len(ann.Hashes), len(ann.Types), len(ann.Sizes))
}
// Schedule all the unknown hashes for retrieval
for _, hash := range ann.Hashes {
peer.MarkTransaction(hash)
}
return backend.Handle(peer, ann)
}
func handleNewPooledTransactionHashes71(backend Backend, msg Decoder, peer *Peer) error {
// New transaction announcement arrived, make sure we have
// a valid and fresh chain to handle them
if !backend.AcceptTxs() {
return nil
}
ann := new(NewPooledTransactionHashesPacket71)
if err := msg.Decode(ann); err != nil { if err := msg.Decode(ann); err != nil {
return err return err
} }
@ -502,11 +523,11 @@ func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error
if err := msg.Decode(&query); err != nil { if err := msg.Decode(&query); err != nil {
return err return err
} }
hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsRequest) hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsRequest, peer.version < ETH71)
return peer.ReplyPooledTransactionsRLP(query.RequestId, hashes, txs) return peer.ReplyPooledTransactionsRLP(query.RequestId, hashes, txs)
} }
func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsRequest) ([]common.Hash, []rlp.RawValue) { func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsRequest, includeBlob bool) ([]common.Hash, []rlp.RawValue) {
// Gather transactions until the fetch or network limits is reached // Gather transactions until the fetch or network limits is reached
var ( var (
bytes int bytes int
@ -518,7 +539,7 @@ func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsReq
break break
} }
// Retrieve the requested transaction, skipping if unknown to us // Retrieve the requested transaction, skipping if unknown to us
encoded := backend.TxPool().GetRLP(hash) encoded := backend.TxPool().GetRLP(hash, includeBlob)
if len(encoded) == 0 { if len(encoded) == 0 {
continue continue
} }
@ -580,3 +601,52 @@ func handleBlockRangeUpdate(backend Backend, msg Decoder, peer *Peer) error {
peer.lastRange.Store(&update) peer.lastRange.Store(&update)
return nil return nil
} }
func handleGetCells(backend Backend, msg Decoder, peer *Peer) error {
// Decode the cell retrieval message
var query GetCellsRequestPacket
if err := msg.Decode(&query); err != nil {
return err
}
hashes, cells, custody := answerGetCells(backend, query.GetCellsRequest)
return peer.ReplyCells(query.RequestId, hashes, cells, custody)
}
func answerGetCells(backend Backend, query GetCellsRequest) ([]common.Hash, [][]kzg4844.Cell, types.CustodyBitmap) {
var (
cellCounts int
hashes []common.Hash
cells [][]kzg4844.Cell
)
maxCells := softResponseLimit / 2048
for _, hash := range query.Hashes {
if cellCounts >= maxCells {
break
}
cell, _ := backend.BlobPool().GetCells(hash, query.Mask)
if len(cell) == 0 {
// skip this tx
continue
}
hashes = append(hashes, hash)
cells = append(cells, cell)
cellCounts += len(cell)
}
return hashes, cells, query.Mask
}
func handleCells(backend Backend, msg Decoder, peer *Peer) error {
var cellsResponse CellsPacket
if err := msg.Decode(&cellsResponse); err != nil {
return err
}
tresp := tracker.Response{
ID: cellsResponse.RequestId,
MsgCode: CellsMsg,
Size: len(cellsResponse.CellsResponse.Hashes),
}
if err := peer.tracker.Fulfil(tresp); err != nil {
return fmt.Errorf("Cells: %w", err)
}
return backend.Handle(peer, &cellsResponse.CellsResponse)
}

View file

@ -77,7 +77,7 @@ func testHandshake(t *testing.T, protocol uint) {
defer app.Close() defer app.Close()
defer net.Close() defer net.Close()
peer := NewPeer(protocol, p2p.NewPeer(enode.ID{}, "peer", nil), net, nil) peer := NewPeer(protocol, p2p.NewPeer(enode.ID{}, "peer", nil), net, nil, nil)
defer peer.Close() defer peer.Close()
// Send the junk test with one peer, check the handshake failure // Send the junk test with one peer, check the handshake failure

View file

@ -24,6 +24,7 @@ import (
mapset "github.com/deckarep/golang-set/v2" mapset "github.com/deckarep/golang-set/v2"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/tracker" "github.com/ethereum/go-ethereum/p2p/tracker"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
@ -54,6 +55,7 @@ type Peer struct {
lastRange atomic.Pointer[BlockRangeUpdatePacket] lastRange atomic.Pointer[BlockRangeUpdatePacket]
txpool TxPool // Transaction pool used by the broadcasters for liveness checks txpool TxPool // Transaction pool used by the broadcasters for liveness checks
blobpool BlobPool
knownTxs *knownCache // Set of transaction hashes known to be known by this peer knownTxs *knownCache // Set of transaction hashes known to be known by this peer
txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests
txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests
@ -68,7 +70,7 @@ type Peer struct {
// NewPeer creates a wrapper for a network connection and negotiated protocol // NewPeer creates a wrapper for a network connection and negotiated protocol
// version. // version.
func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Peer { func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool, blobpool BlobPool) *Peer {
cap := p2p.Cap{Name: ProtocolName, Version: version} cap := p2p.Cap{Name: ProtocolName, Version: version}
id := p.ID().String() id := p.ID().String()
peer := &Peer{ peer := &Peer{
@ -84,6 +86,7 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe
reqCancel: make(chan *cancel), reqCancel: make(chan *cancel),
resDispatch: make(chan *response), resDispatch: make(chan *response),
txpool: txpool, txpool: txpool,
blobpool: blobpool,
term: make(chan struct{}), term: make(chan struct{}),
} }
// Start up all the broadcasters // Start up all the broadcasters
@ -166,10 +169,13 @@ func (p *Peer) AsyncSendTransactions(hashes []common.Hash) {
// This method is a helper used by the async transaction announcer. Don't call it // This method is a helper used by the async transaction announcer. Don't call it
// directly as the queueing (memory) and transmission (bandwidth) costs should // directly as the queueing (memory) and transmission (bandwidth) costs should
// not be managed directly. // not be managed directly.
func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32) error { func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32, cells types.CustodyBitmap) error {
// Mark all the transactions as known, but ensure we don't overflow our limits // Mark all the transactions as known, but ensure we don't overflow our limits
p.knownTxs.Add(hashes...) p.knownTxs.Add(hashes...)
return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket{Types: types, Sizes: sizes, Hashes: hashes}) if p.version >= ETH71 {
return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket71{Types: types, Sizes: sizes, Hashes: hashes, Mask: cells})
}
return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket70{Types: types, Sizes: sizes, Hashes: hashes})
} }
// AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually // AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually
@ -222,6 +228,41 @@ func (p *Peer) ReplyReceiptsRLP(id uint64, receipts rlp.RawList[*ReceiptList]) e
}) })
} }
// ReplyCells is the response to GetCells.
func (p *Peer) ReplyCells(id uint64, hashes []common.Hash, cells [][]kzg4844.Cell, mask types.CustodyBitmap) error {
return p2p.Send(p.rw, CellsMsg, &CellsPacket{
RequestId: id,
CellsResponse: CellsResponse{
Hashes: hashes,
Cells: cells,
Mask: mask,
},
})
}
// RequestPayload fetches a batch of cells from a remote node.
func (p *Peer) RequestPayload(hashes []common.Hash, cell *types.CustodyBitmap) error {
p.Log().Debug("Fetching batch of cells", "txcount", len(hashes), "cellcount", cell.OneCount())
id := rand.Uint64()
err := p.tracker.Track(tracker.Request{
ID: id,
ReqCode: GetCellsMsg,
RespCode: CellsMsg,
Size: len(hashes),
})
if err != nil {
return err
}
return p2p.Send(p.rw, GetCellsMsg, &GetCellsRequestPacket{
RequestId: id,
GetCellsRequest: GetCellsRequest{
Hashes: hashes,
Mask: *cell,
},
})
}
// RequestOneHeader is a wrapper around the header query functions to fetch a // RequestOneHeader is a wrapper around the header query functions to fetch a
// single header. It is used solely by the fetcher. // single header. It is used solely by the fetcher.
func (p *Peer) RequestOneHeader(hash common.Hash, sink chan *Response) (*Request, error) { func (p *Peer) RequestOneHeader(hash common.Hash, sink chan *Response) (*Request, error) {

View file

@ -45,7 +45,7 @@ func newTestPeer(name string, version uint, backend Backend) (*testPeer, <-chan
var id enode.ID var id enode.ID
rand.Read(id[:]) rand.Read(id[:])
peer := NewPeer(version, p2p.NewPeer(id, name, nil), net, backend.TxPool()) peer := NewPeer(version, p2p.NewPeer(id, name, nil), net, backend.TxPool(), backend.BlobPool())
errc := make(chan error, 1) errc := make(chan error, 1)
go func() { go func() {
defer app.Close() defer app.Close()

View file

@ -24,12 +24,14 @@ import (
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/forkid"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rlp"
) )
// Constants to match up protocol versions and messages // Constants to match up protocol versions and messages
const ( const (
ETH69 = 69 ETH69 = 69
ETH71 = 71
) )
// ProtocolName is the official short name of the `eth` protocol used during // ProtocolName is the official short name of the `eth` protocol used during
@ -38,11 +40,11 @@ const ProtocolName = "eth"
// ProtocolVersions are the supported versions of the `eth` protocol (first // ProtocolVersions are the supported versions of the `eth` protocol (first
// is primary). // is primary).
var ProtocolVersions = []uint{ETH69} var ProtocolVersions = []uint{ETH71, ETH69}
// protocolLengths are the number of implemented message corresponding to // protocolLengths are the number of implemented message corresponding to
// different protocol versions. // different protocol versions.
var protocolLengths = map[uint]uint64{ETH69: 18} var protocolLengths = map[uint]uint64{ETH69: 18, ETH71: 20}
// maxMessageSize is the maximum cap on the size of a protocol message. // maxMessageSize is the maximum cap on the size of a protocol message.
const maxMessageSize = 10 * 1024 * 1024 const maxMessageSize = 10 * 1024 * 1024
@ -65,6 +67,8 @@ const (
GetReceiptsMsg = 0x0f GetReceiptsMsg = 0x0f
ReceiptsMsg = 0x10 ReceiptsMsg = 0x10
BlockRangeUpdateMsg = 0x11 BlockRangeUpdateMsg = 0x11
GetCellsMsg = 0x12
CellsMsg = 0x13
) )
var ( var (
@ -230,13 +234,22 @@ type ReceiptsPacket struct {
// ReceiptsRLPResponse is used for receipts, when we already have it encoded // ReceiptsRLPResponse is used for receipts, when we already have it encoded
type ReceiptsRLPResponse []rlp.RawValue type ReceiptsRLPResponse []rlp.RawValue
// NewPooledTransactionHashesPacket represents a transaction announcement packet on eth/68 and newer. // NewPooledTransactionHashesPacket70 represents a transaction announcement packet on eth/69.
type NewPooledTransactionHashesPacket struct { type NewPooledTransactionHashesPacket70 struct {
Types []byte Types []byte
Sizes []uint32 Sizes []uint32
Hashes []common.Hash Hashes []common.Hash
} }
// NewPooledTransactionHashesPacket71 represents a transaction announcement packet on eth/71
// with an additional custody bitmap field for cell-based blob data availability.
type NewPooledTransactionHashesPacket71 struct {
Types []byte
Sizes []uint32
Hashes []common.Hash
Mask types.CustodyBitmap
}
// GetPooledTransactionsRequest represents a transaction query. // GetPooledTransactionsRequest represents a transaction query.
type GetPooledTransactionsRequest []common.Hash type GetPooledTransactionsRequest []common.Hash
@ -273,6 +286,31 @@ type BlockRangeUpdatePacket struct {
LatestBlockHash common.Hash LatestBlockHash common.Hash
} }
// GetCellsRequest represents a request for cells of blob transactions.
type GetCellsRequest struct {
Hashes []common.Hash
Mask types.CustodyBitmap
}
// GetCellsRequestPacket represents a cell request with request ID wrapping.
type GetCellsRequestPacket struct {
RequestId uint64
GetCellsRequest
}
// CellsResponse represents a response containing cells for blob transactions.
type CellsResponse struct {
Hashes []common.Hash
Cells [][]kzg4844.Cell
Mask types.CustodyBitmap
}
// CellsPacket represents a cells response with request ID wrapping.
type CellsPacket struct {
RequestId uint64
CellsResponse
}
func (*StatusPacket) Name() string { return "Status" } func (*StatusPacket) Name() string { return "Status" }
func (*StatusPacket) Kind() byte { return StatusMsg } func (*StatusPacket) Kind() byte { return StatusMsg }
@ -291,8 +329,11 @@ func (*GetBlockBodiesRequest) Kind() byte { return GetBlockBodiesMsg }
func (*BlockBodiesResponse) Name() string { return "BlockBodies" } func (*BlockBodiesResponse) Name() string { return "BlockBodies" }
func (*BlockBodiesResponse) Kind() byte { return BlockBodiesMsg } func (*BlockBodiesResponse) Kind() byte { return BlockBodiesMsg }
func (*NewPooledTransactionHashesPacket) Name() string { return "NewPooledTransactionHashes" } func (*NewPooledTransactionHashesPacket70) Name() string { return "NewPooledTransactionHashes" }
func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransactionHashesMsg } func (*NewPooledTransactionHashesPacket70) Kind() byte { return NewPooledTransactionHashesMsg }
func (*NewPooledTransactionHashesPacket71) Name() string { return "NewPooledTransactionHashes" }
func (*NewPooledTransactionHashesPacket71) Kind() byte { return NewPooledTransactionHashesMsg }
func (*GetPooledTransactionsRequest) Name() string { return "GetPooledTransactions" } func (*GetPooledTransactionsRequest) Name() string { return "GetPooledTransactions" }
func (*GetPooledTransactionsRequest) Kind() byte { return GetPooledTransactionsMsg } func (*GetPooledTransactionsRequest) Kind() byte { return GetPooledTransactionsMsg }
@ -311,3 +352,9 @@ func (*ReceiptsRLPResponse) Kind() byte { return ReceiptsMsg }
func (*BlockRangeUpdatePacket) Name() string { return "BlockRangeUpdate" } func (*BlockRangeUpdatePacket) Name() string { return "BlockRangeUpdate" }
func (*BlockRangeUpdatePacket) Kind() byte { return BlockRangeUpdateMsg } func (*BlockRangeUpdatePacket) Kind() byte { return BlockRangeUpdateMsg }
func (*GetCellsRequest) Name() string { return "GetCells" }
func (*GetCellsRequest) Kind() byte { return GetCellsMsg }
func (*CellsResponse) Name() string { return "Cells" }
func (*CellsResponse) Kind() byte { return CellsMsg }

View file

@ -50,8 +50,8 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) {
defer emptyPipeEth.Close() defer emptyPipeEth.Close()
defer fullPipeEth.Close() defer fullPipeEth.Close()
emptyPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{1}, "", caps), emptyPipeEth, empty.txpool) emptyPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{1}, "", caps), emptyPipeEth, empty.txpool, empty.blobpool)
fullPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{2}, "", caps), fullPipeEth, full.txpool) fullPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{2}, "", caps), fullPipeEth, full.txpool, full.blobpool)
defer emptyPeerEth.Close() defer emptyPeerEth.Close()
defer fullPeerEth.Close() defer fullPeerEth.Close()

View file

@ -139,7 +139,7 @@ func fuzz(input []byte) int {
if verbose { if verbose {
fmt.Println("Notify", peer, announceIdxs) fmt.Println("Notify", peer, announceIdxs)
} }
if err := f.Notify(peer, types, sizes, announces); err != nil { if _, err := f.Notify(peer, types, sizes, announces); err != nil {
panic(err) panic(err)
} }