forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-05-24 16:59:26 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge 81117d9270 into 1149f76dca

Browse source
This commit is contained in:
Bosul Mun 2026-05-19 14:45:51 +05:30 committed by GitHub
commit 8370e2e991
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
40 changed files with 4342 additions and 402 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
beacon/engine/types.go
View file

@ -157,6 +157,11 @@ type BlobAndProofV2 struct {
CellProofs []hexutil.Bytes `json:"proofs"` // proofs MUST contain exactly CELLS_PER_EXT_BLOB cell proofs.
}
type BlobCellsAndProofsV1 struct {
BlobCells []hexutil.Bytes `json:"blob_cells"`
Proofs []hexutil.Bytes `json:"proofs"`
}
// JSON type overrides for ExecutionPayloadEnvelope.
type executionPayloadEnvelopeMarshaling struct {
BlockValue *hexutil.Big

13
cmd/devp2p/internal/ethtest/conn.go
View file

@ -66,10 +66,11 @@ func (s *Suite) dialAs(key *ecdsa.PrivateKey) (*Conn, error) {
return nil, err
}
conn.caps = []p2p.Cap{
{Name: "eth", Version: 72},
{Name: "eth", Version: 70},
{Name: "eth", Version: 69},
}
conn.ourHighestProtoVersion = 70
conn.ourHighestProtoVersion = 72
return &conn, nil
}
@ -93,6 +94,10 @@ type Conn struct {
ourHighestProtoVersion uint
ourHighestSnapProtoVersion uint
caps []p2p.Cap
// pending holds messages received by readUntil that did not match the
// caller's expected type.
pending []any
}
// Read reads a packet from the connection.
@ -168,11 +173,15 @@ func (c *Conn) ReadEth() (any, error) {
case eth.TransactionsMsg:
msg = new(eth.TransactionsPacket)
case eth.NewPooledTransactionHashesMsg:
msg = new(eth.NewPooledTransactionHashesPacket)
msg = new(eth.NewPooledTransactionHashesPacket72)
case eth.GetPooledTransactionsMsg:
msg = new(eth.GetPooledTransactionsPacket)
case eth.PooledTransactionsMsg:
msg = new(eth.PooledTransactionsPacket)
case eth.GetCellsMsg:
msg = new(eth.GetCellsRequestPacket)
case eth.CellsMsg:
msg = new(eth.CellsPacket)
default:
panic(fmt.Sprintf("unhandled eth msg code %d", code))
}

2
cmd/devp2p/internal/ethtest/protocol.go
View file

@ -32,7 +32,7 @@ const (
// Unexported devp2p protocol lengths from p2p package.
const (
baseProtoLen = 16
ethProtoLen = 18
ethProtoLen = 22
snapProtoLen = 8
)

390
cmd/devp2p/internal/ethtest/suite.go
View file

@ -21,6 +21,7 @@ import (
"crypto/rand"
"errors"
"fmt"
"os"
"reflect"
"sync"
"time"
@ -93,6 +94,10 @@ func (s *Suite) EthTests() []utesting.Test {
{Name: "BlobViolations", Fn: s.TestBlobViolations},
{Name: "TestBlobTxWithoutSidecar", Fn: s.TestBlobTxWithoutSidecar},
{Name: "TestBlobTxWithMismatchedSidecar", Fn: s.TestBlobTxWithMismatchedSidecar},
// test eth/72 blob txs
{Name: "BlobTxAvailabilityFailure", Fn: s.TestBlobTxAvailabilityFailure},
{Name: "GetCells", Fn: s.TestGetCells},
{Name: "BlobTxWithInvalidCells", Fn: s.TestBlobTxWithInvalidCells},
}
}
@ -966,7 +971,7 @@ the transactions using a GetPooledTransactions request.`)
}
// Send announcement.
ann := eth.NewPooledTransactionHashesPacket{Types: txTypes, Sizes: sizes, Hashes: hashes}
ann := eth.NewPooledTransactionHashesPacket72{Types: txTypes, Sizes: sizes, Hashes: hashes}
err = conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann)
if err != nil {
t.Fatalf("failed to write to connection: %v", err)
@ -984,7 +989,7 @@ the transactions using a GetPooledTransactions request.`)
t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsRequest))
}
return
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket72:
continue
case *eth.TransactionsPacket:
continue
@ -1003,22 +1008,23 @@ func makeSidecar(data ...byte) *types.BlobTxSidecar {
for i := range blobs {
blobs[i][0] = data[i]
c, _ := kzg4844.BlobToCommitment(&blobs[i])
p, _ := kzg4844.ComputeBlobProof(&blobs[i], c)
cellProofs, _ := kzg4844.ComputeCellProofs(&blobs[i])
commitments = append(commitments, c)
proofs = append(proofs, p)
proofs = append(proofs, cellProofs...)
}
return types.NewBlobTxSidecar(types.BlobSidecarVersion0, blobs, commitments, proofs)
return types.NewBlobTxSidecar(types.BlobSidecarVersion1, blobs, commitments, proofs)
}
func (s *Suite) makeBlobTxs(count, blobs int, discriminator byte) (txs types.Transactions) {
func (s *Suite) makeBlobTxs(txCount, blobCount int, discriminator byte) (txs types.Transactions, blobs [][]kzg4844.Blob) {
from, nonce := s.chain.GetSender(5)
for i := 0; i < count; i++ {
for i := 0; i < txCount; i++ {
// Make blob data, max of 2 blobs per tx.
blobdata := make([]byte, min(blobs, 2))
blobdata := make([]byte, min(blobCount, 2))
for i := range blobdata {
blobdata[i] = discriminator
blobs -= 1
blobCount -= 1
}
sidecar := makeSidecar(blobdata...)
inner := &types.BlobTx{
ChainID: uint256.MustFromBig(s.chain.config.ChainID),
Nonce: nonce + uint64(i),
@ -1026,16 +1032,17 @@ func (s *Suite) makeBlobTxs(count, blobs int, discriminator byte) (txs types.Tra
GasFeeCap: uint256.MustFromBig(s.chain.Head().BaseFee()),
Gas: 100000,
BlobFeeCap: uint256.MustFromBig(eip4844.CalcBlobFee(s.chain.config, s.chain.Head().Header())),
BlobHashes: makeSidecar(blobdata...).BlobHashes(),
Sidecar: makeSidecar(blobdata...),
BlobHashes: sidecar.BlobHashes(),
Sidecar: sidecar,
}
tx, err := s.chain.SignTx(from, types.NewTx(inner))
if err != nil {
panic("blob tx signing failed")
}
txs = append(txs, tx)
blobs = append(blobs, sidecar.Blobs)
txs = append(txs, tx.WithoutBlob())
}
return txs
return txs, blobs
}
func (s *Suite) TestBlobViolations(t *utesting.T) {
@ -1046,28 +1053,30 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
}
// Create blob txs for each tests with unique tx hashes.
var (
t1 = s.makeBlobTxs(2, 3, 0x1)
t2 = s.makeBlobTxs(2, 3, 0x2)
t1, _ = s.makeBlobTxs(2, 3, 0x1)
t2, _ = s.makeBlobTxs(2, 3, 0x2)
)
for _, test := range []struct {
ann eth.NewPooledTransactionHashesPacket
ann eth.NewPooledTransactionHashesPacket72
resp eth.PooledTransactionsResponse
}{
// Invalid tx size.
{
ann: eth.NewPooledTransactionHashesPacket{
ann: eth.NewPooledTransactionHashesPacket72{
Types: []byte{types.BlobTxType, types.BlobTxType},
Sizes: []uint32{uint32(t1[0].Size()), uint32(t1[1].Size() + 10)},
Hashes: []common.Hash{t1[0].Hash(), t1[1].Hash()},
Mask: *types.CustodyBitmapAll,
},
resp: eth.PooledTransactionsResponse(t1),
},
// Wrong tx type.
{
ann: eth.NewPooledTransactionHashesPacket{
ann: eth.NewPooledTransactionHashesPacket72{
Types: []byte{types.DynamicFeeTxType, types.BlobTxType},
Sizes: []uint32{uint32(t2[0].Size()), uint32(t2[1].Size())},
Hashes: []common.Hash{t2[0].Hash(), t2[1].Hash()},
Mask: *types.CustodyBitmapAll,
},
resp: eth.PooledTransactionsResponse(t2),
},
@ -1095,15 +1104,21 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
if code, _, err := conn.Read(); err != nil {
t.Fatalf("expected disconnect on blob violation, got err: %v", err)
} else if code != discMsg {
if code == protoOffset(ethProto)+eth.NewPooledTransactionHashesMsg {
// sometimes we'll get a blob transaction hashes announcement before the disconnect
// because blob transactions are scheduled to be fetched right away.
if code, _, err = conn.Read(); err != nil {
t.Fatalf("expected disconnect on blob violation, got err on second read: %v", err)
for {
code, _, err := conn.Read()
if err != nil {
t.Fatalf("expected disconnect on blob violation, got err: %v", err)
}
if code == discMsg {
break
}
switch code {
case protoOffset(ethProto) + eth.NewPooledTransactionHashesMsg,
protoOffset(ethProto) + eth.GetCellsMsg:
continue
default:
t.Fatalf("expected disconnect on blob violation, got msg code: %d", code)
}
}
if code != discMsg {
t.Fatalf("expected disconnect on blob violation, got msg code: %d", code)
}
}
conn.Close()
@ -1122,22 +1137,29 @@ func mangleSidecar(tx *types.Transaction) *types.Transaction {
func (s *Suite) TestBlobTxWithoutSidecar(t *utesting.T) {
t.Log(`This test checks that a blob transaction first advertised/transmitted without blobs will result in the sending peer being disconnected, and the full transaction should be successfully retrieved from another peer.`)
tx := s.makeBlobTxs(1, 2, 42)[0]
badTx := tx.WithoutBlobTxSidecar()
s.testBadBlobTx(t, tx, badTx)
tx, _ := s.makeBlobTxs(1, 2, 42)
badTx := tx[0].WithoutBlobTxSidecar()
s.testBadBlobTx(t, tx[0], badTx)
}
func (s *Suite) TestBlobTxWithMismatchedSidecar(t *utesting.T) {
t.Log(`This test checks that a blob transaction first advertised/transmitted without blobs, whose commitment don't correspond to the blob_versioned_hashes in the transaction, will result in the sending peer being disconnected, and the full transaction should be successfully retrieved from another peer.`)
tx := s.makeBlobTxs(1, 2, 43)[0]
badTx := mangleSidecar(tx)
s.testBadBlobTx(t, tx, badTx)
tx, _ := s.makeBlobTxs(1, 2, 43)
badTx := mangleSidecar(tx[0])
s.testBadBlobTx(t, tx[0], badTx)
}
// readUntil reads eth protocol messages until a message of the target type is
// received. It returns an error if there is a disconnect, or if the context
// is cancelled before a message of the desired type can be read.
func readUntil[T any](ctx context.Context, conn *Conn) (*T, error) {
// First check the buffer for a previously-stashed match.
for i, msg := range conn.pending {
if t, ok := msg.(*T); ok {
conn.pending = append(conn.pending[:i], conn.pending[i+1:]...)
return t, nil
}
}
for {
select {
case <-ctx.Done():
@ -1151,11 +1173,10 @@ func readUntil[T any](ctx context.Context, conn *Conn) (*T, error) {
}
continue
}
switch res := received.(type) {
case *T:
return res, nil
if t, ok := received.(*T); ok {
return t, nil
}
conn.pending = append(conn.pending, received)
}
}
@ -1193,10 +1214,11 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types
return
}
ann := eth.NewPooledTransactionHashesPacket{
ann := eth.NewPooledTransactionHashesPacket72{
Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(badTx.Size())},
Hashes: []common.Hash{badTx.Hash()},
Mask: *types.CustodyBitmapAll,
}
if err := conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann); err != nil {
@ -1244,14 +1266,15 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types
return
}
ann := eth.NewPooledTransactionHashesPacket{
ann := eth.NewPooledTransactionHashesPacket72{
Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(tx.Size())},
Hashes: []common.Hash{tx.Hash()},
Mask: *types.CustodyBitmapAll,
}
if err := conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann); err != nil {
errc <- fmt.Errorf("sending announcement failed: %v", err)
errc <- fmt.Errorf("sending first announcement failed: %v", err)
return
}
@ -1301,3 +1324,292 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types
t.Fatalf("%v", err)
}
}
func (s *Suite) TestBlobTxAvailabilityFailure(t *utesting.T) {
t.Log(`This test announces 4 blob txs from a single peer. With fetchProbability 0.15,
there will be at least one partial fetch (1-0.15^4). When only 1 peer announced availability,
partial fetch GetCells should never arrive. Any GetCells that does arrive must be a full fetch.`)
if err := s.engine.sendForkchoiceUpdated(); err != nil {
t.Fatalf("send fcu failed: %v", err)
}
txs, _ := s.makeBlobTxs(4, 4, 0x30)
conn, err := s.dial()
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn.Close()
if err := conn.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
// Announce all 4 txs from a single peer.
hashes := make([]common.Hash, len(txs))
txTypes := make([]byte, len(txs))
sizes := make([]uint32, len(txs))
for i, tx := range txs {
hashes[i] = tx.Hash()
txTypes[i] = types.BlobTxType
sizes[i] = uint32(tx.WithoutBlob().Size())
}
ann := eth.NewPooledTransactionHashesPacket72{
Types: txTypes,
Sizes: sizes,
Hashes: hashes,
Mask: *types.CustodyBitmapAll,
}
if err := conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann); err != nil {
t.Fatalf("announce failed: %v", err)
}
// Read messages for a short period. Any GetCells that arrives must be
// a full fetch request (mask >= DataPerBlob), not a partial fetch.
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
for {
select {
case <-ctx.Done():
return
default:
}
msg, err := conn.ReadEth()
if err != nil {
if errors.Is(err, os.ErrDeadlineExceeded) {
return // timeout, test passed
}
t.Fatalf("unexpected error: %v", err)
}
switch req := msg.(type) {
case *eth.GetCellsRequestPacket:
if req.Mask.OneCount() < kzg4844.DataPerBlob {
t.Fatalf("received partial GetCells request with only %d cells from single peer announcement", req.Mask.OneCount())
}
case *eth.GetPooledTransactionsPacket:
encTxs, _ := rlp.EncodeToRawList(txs)
conn.Write(ethProto, eth.PooledTransactionsMsg, eth.PooledTransactionsPacket{
RequestId: req.RequestId,
List: encTxs,
})
}
}
}
// buildCells extracts cells at mask indices from the original tx's blobs
func buildCells(blobs []kzg4844.Blob, mask types.CustodyBitmap) []kzg4844.Cell {
allCells, _ := kzg4844.ComputeCells(blobs)
indices := mask.Indices()
result := make([]kzg4844.Cell, 0, len(blobs)*len(indices))
for b := 0; b < len(blobs); b++ {
for _, idx := range indices {
result = append(result, allCells[b*kzg4844.CellsPerBlob+int(idx)])
}
}
return result
}
// readAnyFrom waits for a message of type T on any of the given conns
// and returns the packet and the conn it came from.
func readAnyFrom[T any](conns ...*Conn) (*T, *Conn, error) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
type result struct {
pkt *T
c *Conn
}
ch := make(chan result, len(conns))
errCh := make(chan error, len(conns))
for _, c := range conns {
go func(c *Conn) {
pkt, err := readUntil[T](ctx, c)
if err != nil {
if !errors.Is(err, context.Canceled) {
errCh <- err
}
return
}
ch <- result{pkt, c}
}(c)
}
select {
case r := <-ch:
return r.pkt, r.c, nil
case err := <-errCh:
return nil, nil, err
}
}
func (s *Suite) TestGetCells(t *utesting.T) {
t.Log(`This test checks that blob tx announcements trigger GetCells requests,
and that providing valid cells causes the tx to enter the pool.`)
if err := s.engine.sendForkchoiceUpdated(); err != nil {
t.Fatalf("send fcu failed: %v", err)
}
txs, blobs := s.makeBlobTxs(1, 1, 0x31)
tx := txs[0]
blob := blobs[0]
// Two peers ensure GetCells arrives regardless of full/partial fetch path.
conn1, err := s.dial()
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn1.Close()
if err := conn1.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
conn2, err := s.dial()
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn2.Close()
if err := conn2.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
ann := eth.NewPooledTransactionHashesPacket72{
Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(tx.Size())},
Hashes: []common.Hash{tx.Hash()},
Mask: *types.CustodyBitmapAll,
}
if err := conn1.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann); err != nil {
t.Fatalf("conn1 announce failed: %v", err)
}
if err := conn2.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann); err != nil {
t.Fatalf("conn2 announce failed: %v", err)
}
// Wait for GetPooledTransactions on either conn, respond with tx (without blobs).
pooledReq, pc, err := readAnyFrom[eth.GetPooledTransactionsPacket](conn1, conn2)
if err != nil {
t.Fatalf("failed to read GetPooledTransactions: %v", err)
}
encTxs, _ := rlp.EncodeToRawList([]*types.Transaction{tx})
resp := eth.PooledTransactionsPacket{RequestId: pooledReq.RequestId, List: encTxs}
if err := pc.Write(ethProto, eth.PooledTransactionsMsg, resp); err != nil {
t.Fatalf("writing pooled tx response failed: %v", err)
}
// Wait for GetCells request on either conn.
cellsReq, cc, err := readAnyFrom[eth.GetCellsRequestPacket](conn1, conn2)
if err != nil {
t.Fatalf("failed to read GetCells: %v", err)
}
if len(cellsReq.Hashes) == 0 || cellsReq.Hashes[0] != tx.Hash() {
t.Fatalf("GetCells for wrong hash: %v", cellsReq.Hashes)
}
// Respond with valid cells matching the requested mask.
cells := buildCells(blob, cellsReq.Mask)
cellsResp := eth.CellsPacket{
RequestId: cellsReq.RequestId,
CellsResponse: eth.CellsResponse{
Hashes: []common.Hash{tx.Hash()},
Cells: [][]kzg4844.Cell{cells},
Mask: cellsReq.Mask,
},
}
if err := cc.Write(ethProto, eth.CellsMsg, cellsResp); err != nil {
t.Fatalf("writing cells response failed: %v", err)
}
// Either peer should not be disconnected after providing valid data.
if readUntilDisconnect(cc) {
t.Fatalf("unexpected disconnect on cells-providing peer")
}
}
func (s *Suite) TestBlobTxWithInvalidCells(t *utesting.T) {
t.Log(`This test checks that a peer responding to GetCells with invalid cells is disconnected,
while the other peer is not.`)
if err := s.engine.sendForkchoiceUpdated(); err != nil {
t.Fatalf("send fcu failed: %v", err)
}
txs, blobs := s.makeBlobTxs(1, 1, 0x32)
tx := txs[0]
blob := blobs[0]
conn1, err := s.dial()
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn1.Close()
if err := conn1.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
conn2, err := s.dial()
if err != nil {
t.Fatalf("dial failed: %v", err)
}
defer conn2.Close()
if err := conn2.peer(s.chain, nil); err != nil {
t.Fatalf("peering failed: %v", err)
}
ann := eth.NewPooledTransactionHashesPacket72{
Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(tx.Size())},
Hashes: []common.Hash{tx.Hash()},
Mask: *types.CustodyBitmapAll,
}
if err := conn1.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann); err != nil {
t.Fatalf("conn1 announce failed: %v", err)
}
if err := conn2.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann); err != nil {
t.Fatalf("conn2 announce failed: %v", err)
}
pooledReq, pc, err := readAnyFrom[eth.GetPooledTransactionsPacket](conn1, conn2)
if err != nil {
t.Fatalf("failed to read GetPooledTransactions: %v", err)
}
encTxs, _ := rlp.EncodeToRawList([]*types.Transaction{tx})
if err := pc.Write(ethProto, eth.PooledTransactionsMsg,
eth.PooledTransactionsPacket{RequestId: pooledReq.RequestId, List: encTxs}); err != nil {
t.Fatalf("writing pooled tx response failed: %v", err)
}
cellsReq, cc, err := readAnyFrom[eth.GetCellsRequestPacket](conn1, conn2)
if err != nil {
t.Fatalf("failed to read GetCells: %v", err)
}
// Respond with corrupted cells (all zero bytes).
blobCount := len(blob)
corrupted := make([]kzg4844.Cell, blobCount*cellsReq.Mask.OneCount())
badResp := eth.CellsPacket{
RequestId: cellsReq.RequestId,
CellsResponse: eth.CellsResponse{
Hashes: []common.Hash{tx.Hash()},
Cells: [][]kzg4844.Cell{corrupted},
Mask: cellsReq.Mask,
},
}
if err := cc.Write(ethProto, eth.CellsMsg, badResp); err != nil {
t.Fatalf("writing bad cells response failed: %v", err)
}
// The peer that sent corrupted cells must be disconnected.
if !readUntilDisconnect(cc) {
t.Fatalf("expected peer to be disconnected after invalid cells")
}
// The innocent peer must stay connected.
otherConn := conn1
if cc == conn1 {
otherConn = conn2
}
if readUntilDisconnect(otherConn) {
t.Fatalf("innocent peer should not be disconnected")
}
}

4
cmd/devp2p/internal/ethtest/transaction.go
View file

@ -74,7 +74,7 @@ func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error {
for _, tx := range txs {
got[tx.Hash()] = true
}
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket72:
for _, hash := range msg.Hashes {
got[hash] = true
}
@ -160,7 +160,7 @@ func (s *Suite) sendInvalidTxs(t *utesting.T, txs []*types.Transaction) error {
return fmt.Errorf("received bad tx: %s", tx.Hash())
}
}
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket72:
for _, hash := range msg.Hashes {
if _, ok := invalids[hash]; ok {
return fmt.Errorf("received bad tx: %s", hash)

443
core/txpool/blobpool/blobpool.go
View file

@ -129,9 +129,12 @@ type blobTxMeta struct {
announced bool // Whether the tx has been announced to listeners
id uint64 // Storage ID in the pool's persistent store
storageSize uint32 // Byte size in the pool's persistent store
size uint64 // RLP-encoded size of transaction including the attached blob
id uint64 // Storage ID in the pool's persistent store
storageSize uint32 // Byte size in the pool's persistent store
size uint64 // RLP-encoded size of transaction including the attached blob
sizeWithoutBlob uint64 // RLP-encoded size of transaction without blob data (for ETH/72)
custody *types.CustodyBitmap
nonce uint64 // Needed to prioritize inclusion order within an account
costCap *uint256.Int // Needed to validate cumulative balance sufficiency
@ -149,78 +152,125 @@ type blobTxMeta struct {
evictionBlobFeeJumps float64 // Worse blob fee (converted to fee jumps) across all previous nonces
}
// blobTxForPool is the storage representation of a blob transaction in the
// BlobTxForPool is the storage representation of a blob transaction in the
// blobpool.
type blobTxForPool struct {
type BlobTxForPool struct {
Tx *types.Transaction // tx without sidecar
Version byte
Commitments []kzg4844.Commitment
Proofs []kzg4844.Proof
Blobs []kzg4844.Blob
Cells []kzg4844.Cell
Custody types.CustodyBitmap
}
// Sidecar returns BlobTxSidecar of ptx.
func (ptx *blobTxForPool) Sidecar() *types.BlobTxSidecar {
return types.NewBlobTxSidecar(ptx.Version, ptx.Blobs, ptx.Commitments, ptx.Proofs)
// Sidecar returns BlobTxSidecar of pooled transaction. Since this function
// recovers the blob field in sidecar, it is expansive and needs to be
// avoided if possible. Returns error if recovery fails (e.g. insufficient cells).
func (ptx *BlobTxForPool) sidecar() (*types.BlobTxSidecar, error) {
blobs, err := kzg4844.RecoverBlobs(ptx.Cells, ptx.Custody.Indices())
if err != nil {
return nil, err
}
return types.NewBlobTxSidecar(ptx.Version, blobs, ptx.Commitments, ptx.Proofs), nil
}
// ApplySidecar copies the sidecar's fields into the flat fields.
func (ptx *blobTxForPool) ApplySidecar(sc *types.BlobTxSidecar) {
ptx.Version = sc.Version
ptx.Commitments = sc.Commitments
ptx.Proofs = sc.Proofs
ptx.Blobs = sc.Blobs
func (ptx *BlobTxForPool) toV1() error {
// todo: If we have a function to compute proofs from cells,
// we can avoid blob recovery here
blobs, err := kzg4844.RecoverBlobs(ptx.Cells, ptx.Custody.Indices())
if err != nil {
return err
}
proofs := make([]kzg4844.Proof, 0)
for _, blob := range blobs {
proof, err := kzg4844.ComputeCellProofs(&blob)
if err != nil {
return err
}
proofs = append(proofs, proof...)
}
ptx.Proofs = proofs
ptx.Version = types.BlobSidecarVersion1
return nil
}
// TxSize returns the transaction size on the network without
// reconstructing the transaction.
func (ptx *blobTxForPool) TxSize() uint64 {
var blobs, commitments, proofs uint64
for i := range ptx.Blobs {
blobs += rlp.BytesSize(ptx.Blobs[i][:])
}
func (ptx *BlobTxForPool) txSize() uint64 {
var commitments, proofs uint64
for i := range ptx.Commitments {
commitments += rlp.BytesSize(ptx.Commitments[i][:])
}
for i := range ptx.Proofs {
proofs += rlp.BytesSize(ptx.Proofs[i][:])
}
var blob kzg4844.Blob
blobs := uint64(len(ptx.Commitments)) * rlp.BytesSize(blob[:])
return ptx.Tx.Size() + rlp.ListSize(rlp.ListSize(blobs)+rlp.ListSize(commitments)+rlp.ListSize(proofs))
}
func (ptx *BlobTxForPool) txSizeWithoutBlob() uint64 {
var commitments, proofs uint64
for i := range ptx.Commitments {
commitments += rlp.BytesSize(ptx.Commitments[i][:])
}
for i := range ptx.Proofs {
proofs += rlp.BytesSize(ptx.Proofs[i][:])
}
return ptx.Tx.Size() + rlp.ListSize(rlp.ListSize(0)+rlp.ListSize(commitments)+rlp.ListSize(proofs))
}
// ToTx reconstructs a full Transaction with the sidecar attached.
func (ptx *blobTxForPool) ToTx() *types.Transaction {
return ptx.Tx.WithBlobTxSidecar(ptx.Sidecar())
func (ptx *BlobTxForPool) toTx() (*types.Transaction, error) {
sc, err := ptx.sidecar()
if err != nil {
return nil, err
}
return ptx.Tx.WithBlobTxSidecar(sc), nil
}
// newBlobTxForPool decomposes a blob transaction into blobTxForPool type.
func newBlobTxForPool(tx *types.Transaction) *blobTxForPool {
func newBlobTxForPool(tx *types.Transaction) (*BlobTxForPool, error) {
sc := tx.BlobTxSidecar()
if sc == nil {
panic("missing blob tx sidecar")
return nil, errors.New("missing blob tx sidecar")
}
return &blobTxForPool{
cells, err := kzg4844.ComputeCells(sc.Blobs)
if err != nil {
return nil, err
}
return &BlobTxForPool{
Tx: tx.WithoutBlobTxSidecar(),
Version: sc.Version,
Commitments: sc.Commitments,
Proofs: sc.Proofs,
Blobs: sc.Blobs,
}
Cells: cells,
Custody: *types.CustodyBitmapAll,
}, nil
}
// encodeForNetwork transforms stored blobTxForPool RLP into the standard
// network transaction encoding. This is used for getRLP.
// encodeForNetwork transforms stored BlobTxForPool RLP into the network
// transaction encoding for the given eth protocol version. Used for getRLP.
//
// Stored RLP: [type_byte || tx_fields, version, [comms], [proofs], [blobs]]
// V0: type_byte || rlp([tx_fields, [blobs], [comms], [proofs]])
// V1: type_byte || rlp([tx_fields, version, [blobs], [comms], [proofs]])
func encodeForNetwork(storedRLP []byte) ([]byte, error) {
// Stored RLP: [type_byte || tx_fields, version, [comms], [proofs], [cells], custody]
//
// eth/69, eth/70: [blobs] is recovered from stored cells via kzg.
//
// V0: type_byte || rlp([tx_fields, [blobs], [comms], [proofs]])
// V1: type_byte || rlp([tx_fields, version, [blobs], [comms], [proofs]])
//
// eth/72: [blobs] is replaced by an empty list (cells are fetched separately
//
// via GetCells).
// V0: type_byte || rlp([tx_fields, [], [comms], [proofs]])
// V1: type_byte || rlp([tx_fields, version, [], [comms], [proofs]])
func encodeForNetwork(storedRLP []byte, version uint) ([]byte, error) {
elems, err := rlp.SplitListValues(storedRLP)
if err != nil {
return nil, fmt.Errorf("invalid blobTxForPool RLP: %w", err)
return nil, fmt.Errorf("invalid BlobTxForPool RLP: %w", err)
}
if len(elems) < 5 {
return nil, fmt.Errorf("blobTxForPool has %d elements, need at least 5", len(elems))
if len(elems) < 6 {
return nil, fmt.Errorf("BlobTxForPool has %d elements, need at least 6", len(elems))
}
// 1. Extract tx byte and other tx fields
@ -235,22 +285,48 @@ func encodeForNetwork(storedRLP []byte) ([]byte, error) {
txRLP := txBytes[1:]
// 2. Find the version of sidecar.
version, _, err := rlp.SplitUint64(elems[1])
if err != nil || version > 255 {
sidecarVersion, _, err := rlp.SplitUint64(elems[1])
if err != nil || sidecarVersion > 255 {
return nil, fmt.Errorf("invalid version: %w", err)
}
versionByte := byte(version)
sidecarVersionByte := byte(sidecarVersion)
// 3. Extract sidecar elements.
commitmentsRLP := elems[2]
proofsRLP := elems[3]
blobsRLP := elems[4]
// 4. Reconstruct into the network format.
var outer [][]byte
if versionByte == types.BlobSidecarVersion0 {
outer = [][]byte{txRLP, blobsRLP, commitmentsRLP, proofsRLP}
// 4. Build the [blobs] field for the wire format.
var blobsField []byte
// todo: should we use eth.ETH72 here
if version >= 72 {
// eth/72 omits the blob payload; peers fetch cells separately via GetCells.
blobsField = []byte{0xc0} // RLP-encoded empty list
} else {
outer = [][]byte{txRLP, elems[1], blobsRLP, commitmentsRLP, proofsRLP}
// eth/69, eth/70 need actual blobs: recover them from stored cells.
var cells []kzg4844.Cell
if err := rlp.DecodeBytes(elems[4], &cells); err != nil {
return nil, fmt.Errorf("invalid cells RLP: %w", err)
}
var custody types.CustodyBitmap
if err := rlp.DecodeBytes(elems[5], &custody); err != nil {
return nil, fmt.Errorf("invalid custody RLP: %w", err)
}
blobs, err := kzg4844.RecoverBlobs(cells, custody.Indices())
if err != nil {
return nil, fmt.Errorf("failed to recover blobs: %w", err)
}
blobsField, err = rlp.EncodeToBytes(blobs)
if err != nil {
return nil, fmt.Errorf("failed to encode blobs: %w", err)
}
}
// 5. Reconstruct into the network format.
var outer [][]byte
if sidecarVersionByte == types.BlobSidecarVersion0 {
outer = [][]byte{txRLP, blobsField, commitmentsRLP, proofsRLP}
} else {
outer = [][]byte{txRLP, elems[1], blobsField, commitmentsRLP, proofsRLP}
}
body, err := rlp.MergeListValues(outer)
if err != nil {
@ -265,21 +341,23 @@ func encodeForNetwork(storedRLP []byte) ([]byte, error) {
// newBlobTxMeta retrieves the indexed metadata fields from a pooled blob
// transaction and assembles a helper struct to track in memory.
func newBlobTxMeta(id uint64, size uint64, storageSize uint32, ptx *blobTxForPool) *blobTxMeta {
func newBlobTxMeta(id uint64, storageSize uint32, ptx *BlobTxForPool) *blobTxMeta {
meta := &blobTxMeta{
hash: ptx.Tx.Hash(),
vhashes: ptx.Tx.BlobHashes(),
version: ptx.Version,
id: id,
storageSize: storageSize,
size: size,
nonce: ptx.Tx.Nonce(),
costCap: uint256.MustFromBig(ptx.Tx.Cost()),
execTipCap: uint256.MustFromBig(ptx.Tx.GasTipCap()),
execFeeCap: uint256.MustFromBig(ptx.Tx.GasFeeCap()),
blobFeeCap: uint256.MustFromBig(ptx.Tx.BlobGasFeeCap()),
execGas: ptx.Tx.Gas(),
blobGas: ptx.Tx.BlobGas(),
hash: ptx.Tx.Hash(),
vhashes: ptx.Tx.BlobHashes(),
version: ptx.Version,
id: id,
storageSize: storageSize,
size: ptx.txSize(),
sizeWithoutBlob: ptx.txSizeWithoutBlob(),
nonce: ptx.Tx.Nonce(),
costCap: uint256.MustFromBig(ptx.Tx.Cost()),
execTipCap: uint256.MustFromBig(ptx.Tx.GasTipCap()),
execFeeCap: uint256.MustFromBig(ptx.Tx.GasFeeCap()),
blobFeeCap: uint256.MustFromBig(ptx.Tx.BlobGasFeeCap()),
execGas: ptx.Tx.Gas(),
blobGas: ptx.Tx.BlobGas(),
custody: &ptx.Custody,
}
meta.basefeeJumps = dynamicFeeJumps(meta.execFeeCap)
meta.blobfeeJumps = dynamicBlobFeeJumps(meta.blobFeeCap)
@ -478,8 +556,8 @@ type BlobPool struct {
stored uint64 // Useful data size of all transactions on disk
limbo *limbo // Persistent data store for the non-finalized blobs
gapped map[common.Address][]*types.Transaction // Transactions that are currently gapped (nonce too high)
gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion
gapped map[common.Address][]*BlobTxForPool // Transactions that are currently gapped (nonce too high)
gappedSource map[common.Hash]common.Address // Source of gapped transactions to allow rechecking on inclusion
signer types.Signer // Transaction signer to use for sender recovery
chain BlockChain // Chain object to access the state through
@ -514,7 +592,7 @@ func New(config Config, chain BlockChain, hasPendingAuth func(common.Address) bo
lookup: newLookup(),
index: make(map[common.Address][]*blobTxMeta),
spent: make(map[common.Address]*uint256.Int),
gapped: make(map[common.Address][]*types.Transaction),
gapped: make(map[common.Address][]*BlobTxForPool),
gappedSource: make(map[common.Hash]common.Address),
}
}
@ -605,7 +683,14 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
if tx.BlobTxSidecar() == nil {
continue
}
ptx := newBlobTxForPool(&tx)
ptx, err := newBlobTxForPool(&tx)
// Note that we skip errors here.
// Just like parseTransaction failure does not abort the blobpool creation,
// conversion process also cannot abort the entire process.
if err != nil {
log.Error("Failed to convert legacy tx to pooledBlobTx", "hash", tx.Hash(), "err", err)
continue
}
blob, err := rlp.EncodeToBytes(ptx)
if err != nil {
continue
@ -614,7 +699,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reser
if err != nil {
continue
}
meta := newBlobTxMeta(id, ptx.TxSize(), p.store.Size(id), ptx)
meta := newBlobTxMeta(id, p.store.Size(id), ptx)
// If the newly inserted transaction fails to be tracked,
// it should also be removed with those in `toDelete`
@ -717,20 +802,29 @@ func (p *BlobPool) Close() error {
// parseTransaction is a callback method on pool creation that gets called for
// each transaction on disk to create the in-memory metadata index.
// Return value `bool` is set to true when the entry has old Transaction type.
// Announced state is not initialized here, it needs to be initialized separately.
//
// If a legacy types.Transaction is found on disk, it is returned for migration
// in Init (the old ID will be deleted and a new pooledBlobTx written).
// If a pooledBlobTx is found, it is indexed directly and nil is returned.
func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error {
var ptx blobTxForPool
var ptx BlobTxForPool
if err := rlp.DecodeBytes(blob, &ptx); err != nil {
kind, content, _, splitErr := rlp.Split(blob)
// check whether it is legacy tx type
if splitErr == nil && kind == rlp.String && len(content) > 1 && content[0] == 3 {
return errLegacyTx
}
log.Error("Failed to decode blob pool entry", "id", id, "err", err)
return err
}
meta := newBlobTxMeta(id, ptx.TxSize(), size, &ptx)
meta := newBlobTxMeta(id, size, &ptx)
sender, err := types.Sender(p.signer, ptx.Tx)
if err != nil {
// This path is impossible unless the signature validity changes across
// restarts. For that ever improbable case, recover gracefully by ignoring
// this data entry.
log.Error("Failed to recover blob tx sender", "id", id, "hash", ptx.Tx.Hash(), "err", err)
return err
}
return p.trackTransaction(meta, sender)
@ -1019,7 +1113,7 @@ func (p *BlobPool) offload(addr common.Address, nonce uint64, id uint64, inclusi
log.Error("Blobs missing for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err)
return
}
var ptx blobTxForPool
var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
log.Error("Blobs corrupted for included transaction", "from", addr, "nonce", nonce, "id", id, "err", err)
return
@ -1107,7 +1201,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) {
log.Error("Blobs missing for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err)
continue
}
var ptx blobTxForPool
var ptx BlobTxForPool
if err = rlp.DecodeBytes(data, &ptx); err != nil {
log.Error("Blobs corrupted for announcable transaction", "from", addr, "nonce", meta.nonce, "id", meta.id, "err", err)
continue
@ -1281,12 +1375,10 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error {
// this attack is financially inefficient to execute.
head := p.head.Load()
if p.chain.Config().IsOsaka(head.Number, head.Time) && ptx.Version == types.BlobSidecarVersion0 {
sc := ptx.Sidecar()
if err := sc.ToV1(); err != nil {
if err := ptx.toV1(); err != nil {
log.Error("Failed to convert the legacy sidecar", "err", err)
return err
}
ptx.ApplySidecar(sc)
log.Info("Legacy blob transaction is reorged", "hash", ptx.Tx.Hash())
}
blob, err := rlp.EncodeToBytes(ptx)
@ -1299,7 +1391,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error {
log.Error("Failed to write transaction into storage", "hash", ptx.Tx.Hash(), "err", err)
return err
}
meta := newBlobTxMeta(id, ptx.TxSize(), p.store.Size(id), ptx)
meta := newBlobTxMeta(id, p.store.Size(id), ptx)
if _, ok := p.index[addr]; !ok {
if err := p.reserver.Hold(addr); err != nil {
log.Warn("Failed to reserve account for blob pool", "tx", ptx.Tx.Hash(), "from", addr, "err", err)
@ -1393,7 +1485,7 @@ func (p *BlobPool) ValidateTxBasics(tx *types.Transaction) error {
Accept: 1 << types.BlobTxType,
MaxSize: txMaxSize,
MinTip: p.gasTip.Load().ToBig(),
MaxBlobCount: maxBlobsPerTx,
MaxBlobCount: maxBlobsPerTx, //todo this field is currently not being used
}
return txpool.ValidateTransaction(tx, p.head.Load(), p.signer, opts)
}
@ -1528,6 +1620,7 @@ func (p *BlobPool) Has(hash common.Hash) bool {
return poolHas || gapped
}
// getRLP returns the raw RLP-encoded pooledBlobTx data from the store.
func (p *BlobPool) getRLP(hash common.Hash) []byte {
// Track the amount of time waiting to retrieve a fully resolved blob tx from
// the pool and the amount of time actually spent on pulling the data from disk.
@ -1554,32 +1647,38 @@ func (p *BlobPool) getRLP(hash common.Hash) []byte {
}
// Get returns a transaction if it is contained in the pool, or nil otherwise.
// Note that this function always try to recover full blobs
func (p *BlobPool) Get(hash common.Hash) *types.Transaction {
data := p.getRLP(hash)
if len(data) == 0 {
return nil
}
var ptx blobTxForPool
var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
id, _ := p.lookup.storeidOfTx(hash)
log.Error("Blobs corrupted for traced transaction",
"hash", hash, "id", id, "err", err)
log.Error("Blobs corrupted for traced transaction", "hash", hash, "id", id, "err", err)
return nil
}
return ptx.ToTx()
tx, err := ptx.toTx()
if err != nil {
log.Error("Failed to recover transaction in blobpool", "hash", hash, "err", err)
return nil
}
return tx
}
// GetRLP returns a RLP-encoded transaction for network if it is contained in the pool.
// It converts the pool's internal type to the RLP format used by the eth protocol:
// e.g. type_byte || [..., version, [blobs], [comms], [proofs]]
func (p *BlobPool) GetRLP(hash common.Hash) []byte {
// GetRLP returns an RLP-encoded transaction if it is contained in the pool.
// TODO: The pool internally stores pooledBlobTx (cell sidecar format), but callers expect
// types.Transaction RLP. This requires an additional decode-encode step, which is inefficient
// and contradicts the original purpose of this function.
// Possible improvements: Drop eth70 and store the cell and transaction separately.
func (p *BlobPool) GetRLP(hash common.Hash, version uint) []byte {
data := p.getRLP(hash)
if len(data) == 0 {
// Not in this pool, do not log.
return nil
}
rlp, err := encodeForNetwork(data)
rlp, err := encodeForNetwork(data, version)
if err != nil {
log.Error("Failed to encode pooled tx into the network type", "hash", hash, "err", err)
return nil
@ -1596,13 +1695,14 @@ func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
p.lock.RLock()
defer p.lock.RUnlock()
size, ok := p.lookup.sizeOfTx(hash)
meta, ok := p.lookup.txIndex[hash]
if !ok {
return nil
}
return &txpool.TxMetadata{
Type: types.BlobTxType,
Size: size,
Type: types.BlobTxType,
Size: meta.size,
SizeWithoutBlob: meta.sizeWithoutBlob,
}
}
@ -1653,12 +1753,16 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte) ([]*kzg4844.Blo
}
// Decode the blob transaction
var ptx blobTxForPool
var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
log.Error("Blobs corrupted for traced transaction", "id", txID, "err", err)
continue
}
sidecar := ptx.Sidecar()
sidecar, err := ptx.sidecar()
if err != nil {
log.Error("Failed to recover sidecar in blobpool", "id", txID, "err", err)
continue
}
// Traverse the blobs in the transaction
for i, hash := range ptx.Tx.BlobHashes() {
list, ok := indices[hash]
@ -1695,7 +1799,92 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash, version byte) ([]*kzg4844.Blo
return blobs, commitments, proofs, nil
}
// AvailableBlobs returns the number of blobs that are available in the subpool.
// GetBlobHashes returns the blob versioned hashes for a given transaction hash.
func (p *BlobPool) GetBlobHashes(txHash common.Hash) []common.Hash {
p.lock.RLock()
defer p.lock.RUnlock()
vhashes, ok := p.lookup.blobHashesOfTx(txHash)
if !ok {
return nil
}
return vhashes
}
// GetBlobCells returns cells for the given versioned blob hashes,
// filtered by the requested cell indices(mask).
// Each entry in the result corresponds to one vhash. Nil entries mean the blob
// was not available.
func (p *BlobPool) GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error) {
var (
cells = make([][]*kzg4844.Cell, len(vhashes))
proofs = make([][]*kzg4844.Proof, len(vhashes))
vindex = make(map[common.Hash][]int) // Indices of versioned hashes in the request
filled = make(map[common.Hash]struct{})
)
for i, h := range vhashes {
vindex[h] = append(vindex[h], i)
}
requestedIndices := mask.Indices()
for _, vhash := range vhashes {
if _, ok := filled[vhash]; ok {
continue
}
p.lock.RLock()
txID, exists := p.lookup.storeidOfBlob(vhash)
p.lock.RUnlock()
if !exists {
continue
}
data, err := p.store.Get(txID)
if err != nil {
continue
}
var ptx BlobTxForPool
if err := rlp.DecodeBytes(data, &ptx); err != nil {
continue
}
tx := ptx.Tx
cellsPerBlob := ptx.Custody.OneCount()
storedIndices := ptx.Custody.Indices()
for blobIdx, hash := range tx.BlobHashes() {
indices, ok := vindex[hash]
if !ok {
continue
}
filled[hash] = struct{}{}
blobCells := make([]*kzg4844.Cell, len(requestedIndices))
blobProofs := make([]*kzg4844.Proof, len(requestedIndices))
for i, cellIdx := range requestedIndices {
pos := -1
for k, storedIdx := range storedIndices {
if storedIdx == cellIdx {
pos = k
break
}
}
if pos >= 0 {
cell := ptx.Cells[blobIdx*cellsPerBlob+pos]
blobCells[i] = &cell
proofIdx := blobIdx*kzg4844.CellProofsPerBlob + int(cellIdx)
if proofIdx < len(ptx.Proofs) {
proof := ptx.Proofs[proofIdx]
blobProofs[i] = &proof
}
}
}
for _, idx := range indices {
cells[idx] = blobCells
proofs[idx] = blobProofs
}
}
}
return cells, proofs, nil
}
func (p *BlobPool) AvailableBlobs(vhashes []common.Hash) int {
available := 0
for _, vhash := range vhashes {
@ -1718,14 +1907,19 @@ func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error {
if errs[i] = p.ValidateTxBasics(tx); errs[i] != nil {
continue
}
errs[i] = p.add(tx)
ptx, err := newBlobTxForPool(tx)
if err != nil {
errs[i] = err
continue
}
errs[i] = p.AddPooledTx(ptx)
}
return errs
}
// add inserts a new blob transaction into the pool if it passes validation (both
// consensus validity and pool restrictions).
func (p *BlobPool) add(tx *types.Transaction) (err error) {
func (p *BlobPool) AddPooledTx(ptx *BlobTxForPool) (err error) {
// The blob pool blocks on adding a transaction. This is because blob txs are
// only even pulled from the network, so this method will act as the overload
// protection for fetches.
@ -1738,13 +1932,23 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) {
addtimeHist.Update(time.Since(start).Nanoseconds())
}(time.Now())
return p.addLocked(tx, true)
return p.addLocked(ptx, true)
}
// addLocked inserts a new blob transaction into the pool if it passes validation (both
// consensus validity and pool restrictions). It must be called with the pool lock held.
// Only for internal use.
func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error) {
func (p *BlobPool) addLocked(ptx *BlobTxForPool, checkGapped bool) (err error) {
tx := ptx.Tx
//todo: remove this type and also ToBlobTxCellSidecar function
cellSidecar := &types.BlobTxCellSidecar{
Version: ptx.Version,
Cells: ptx.Cells,
Commitments: ptx.Commitments,
Proofs: ptx.Proofs,
Custody: ptx.Custody,
}
// Ensure the transaction is valid from all perspectives
if err := p.validateTx(tx); err != nil {
log.Trace("Transaction validation failed", "hash", tx.Hash(), "err", err)
@ -1761,7 +1965,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
from, _ := types.Sender(p.signer, tx)
allowance := p.gappedAllowance(from)
if allowance >= 1 && len(p.gappedSource) < maxGapped {
p.gapped[from] = append(p.gapped[from], tx)
p.gapped[from] = append(p.gapped[from], ptx)
p.gappedSource[tx.Hash()] = from
gappedGauge.Update(int64(len(p.gappedSource)))
log.Trace("added tx to gapped blob queue", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
@ -1785,6 +1989,12 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
}
return err
}
if err := txpool.ValidateBlobSidecar(tx, cellSidecar, p.head.Load(), &txpool.ValidationOptions{
Config: p.chain.Config(),
MaxBlobCount: maxBlobsPerTx,
}); err != nil {
return err
}
// If the address is not yet known, request exclusivity to track the account
// only by this subpool until all transactions are evicted
from, _ := types.Sender(p.signer, tx) // already validated above
@ -1807,7 +2017,6 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
}
// Transaction permitted into the pool from a nonce and cost perspective,
// insert it into the database and update the indices
ptx := newBlobTxForPool(tx)
blob, err := rlp.EncodeToBytes(ptx)
if err != nil {
log.Error("Failed to encode transaction for storage", "hash", tx.Hash(), "err", err)
@ -1817,7 +2026,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
if err != nil {
return err
}
meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), ptx)
meta := newBlobTxMeta(id, p.store.Size(id), ptx)
var (
next = p.state.GetNonce(from)
@ -1928,13 +2137,13 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
// We have to add in nonce order, but we want to stable sort to cater for situations
// where transactions are replaced, keeping the original receive order for same nonce
sort.SliceStable(gtxs, func(i, j int) bool {
return gtxs[i].Nonce() < gtxs[j].Nonce()
return gtxs[i].Tx.Nonce() < gtxs[j].Tx.Nonce()
})
for len(gtxs) > 0 {
stateNonce := p.state.GetNonce(from)
firstgap := stateNonce + uint64(len(p.index[from]))
if gtxs[0].Nonce() > firstgap {
if gtxs[0].Tx.Nonce() > firstgap {
// Anything beyond the first gap is not addable yet
break
}
@ -1942,26 +2151,26 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
// Drop any buffered transactions that became stale in the meantime (included in chain or replaced)
// If we arrive to the transaction in the pending range (between the state Nonce and first gap, we
// try to add them now while removing from here.
tx := gtxs[0]
ptx := gtxs[0]
gtxs[0] = nil
gtxs = gtxs[1:]
delete(p.gappedSource, tx.Hash())
delete(p.gappedSource, ptx.Tx.Hash())
if tx.Nonce() < stateNonce {
if ptx.Tx.Nonce() < stateNonce {
// Stale, drop it. Eventually we could add to limbo here if hash matches.
log.Trace("Gapped blob transaction became stale", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from]))
log.Trace("Gapped blob transaction became stale", "hash", ptx.Tx.Hash(), "from", from, "nonce", ptx.Tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from]))
continue
}
if tx.Nonce() <= firstgap {
if ptx.Tx.Nonce() <= firstgap {
// If we hit the pending range, including the first gap, add it and continue to try to add more.
// We do not recurse here, but continue to loop instead.
// We are under lock, so we can add the transaction directly.
if err := p.addLocked(tx, false); err == nil {
if err := p.addLocked(ptx, false); err == nil {
gappedPromotedMeter.Mark(1)
log.Trace("Gapped blob transaction added to pool", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
log.Trace("Gapped blob transaction added to pool", "hash", ptx.Tx.Hash(), "from", from, "nonce", ptx.Tx.Nonce(), "qlen", len(p.gapped[from]))
} else {
log.Trace("Gapped blob transaction not accepted", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "err", err)
log.Trace("Gapped blob transaction not accepted", "hash", ptx.Tx.Hash(), "from", from, "nonce", ptx.Tx.Nonce(), "err", err)
}
}
}
@ -2087,6 +2296,10 @@ func (p *BlobPool) Pending(filter txpool.PendingFilter) (map[common.Address][]*t
break // execution gas limit is too high
}
}
// Skip transactions without enough cells to recover blobs
if tx.custody != nil && tx.custody.OneCount() < kzg4844.DataPerBlob {
break // not enough cells to build a full payload, discard rest of txs from the account
}
// Transaction was accepted according to the filter, append to the pending list
lazies = append(lazies, &txpool.LazyTransaction{
Pool: p,
@ -2229,10 +2442,10 @@ func (p *BlobPool) evictGapped() {
// and we overwrite the slice for this account after filtering.
keep := txs[:0]
for i, gtx := range txs {
if gtx.Time().Before(cutoff) || gtx.Nonce() < nonce {
if gtx.Tx.Time().Before(cutoff) || gtx.Tx.Nonce() < nonce {
// Evict old or stale transactions
// Should we add stale to limbo here if it would belong?
delete(p.gappedSource, gtx.Hash())
delete(p.gappedSource, gtx.Tx.Hash())
txs[i] = nil // Explicitly nil out evicted element
} else {
keep = append(keep, gtx)
@ -2351,7 +2564,7 @@ func (p *BlobPool) Clear() {
// Reset counters and the gapped buffer
p.stored = 0
p.gapped = make(map[common.Address][]*types.Transaction)
p.gapped = make(map[common.Address][]*BlobTxForPool)
p.gappedSource = make(map[common.Hash]common.Address)
var (
@ -2360,3 +2573,13 @@ func (p *BlobPool) Clear() {
)
p.evict = newPriceHeap(basefee, blobfee, p.index)
}
// GetCustody returns the custody bitmap for a given transaction hash.
func (p *BlobPool) GetCustody(hash common.Hash) *types.CustodyBitmap {
p.lock.RLock()
defer p.lock.RUnlock()
if meta := p.lookup.txIndex[hash]; meta != nil {
return &meta.custody
}
return nil
}

346
core/txpool/blobpool/blobpool_test.go
View file

@ -237,7 +237,11 @@ func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64,
// encodeForPool encodes a blob transaction in the blobTxForPool storage format.
func encodeForPool(tx *types.Transaction) []byte {
blob, _ := rlp.EncodeToBytes(newBlobTxForPool(tx))
ptx, err := newBlobTxForPool(tx)
if err != nil {
panic(err)
}
blob, _ := rlp.EncodeToBytes(ptx)
return blob
}
@ -497,7 +501,7 @@ func verifyBlobRetrievals(t *testing.T, pool *BlobPool) {
// - 8. Fully duplicate transactions (matching hash) must be dropped
// - 9. Duplicate nonces from the same account must be dropped
func TestOpenDrops(t *testing.T) {
//log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true)))
// log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true)))
// Create a temporary folder for the persistent backend
storage := t.TempDir()
@ -524,75 +528,76 @@ func TestOpenDrops(t *testing.T) {
S: new(uint256.Int),
})
blob, _ := rlp.EncodeToBytes(tx)
badsig, _ := store.Put(blob)
badsig := tx.Hash()
store.Put(blob)
// Insert a sequence of transactions with a nonce gap in between to verify
// that anything gapped will get evicted (case 3).
var (
gapper, _ = crypto.GenerateKey()
valids = make(map[uint64]struct{})
gapped = make(map[uint64]struct{})
valids = make(map[common.Hash]struct{})
gapped = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 3, 4, 6, 7} { // first gap at #2, another at #5
tx := makeTx(nonce, 1, 1, 1, gapper)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce < 2 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
gapped[id] = struct{}{}
gapped[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions with a gapped starting nonce to verify
// that the entire set will get dropped (case 3).
var (
dangler, _ = crypto.GenerateKey()
dangling = make(map[uint64]struct{})
dangling = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{1, 2, 3} { // first gap at #0, all set dangling
tx := makeTx(nonce, 1, 1, 1, dangler)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
dangling[id] = struct{}{}
store.Put(blob)
dangling[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with already passed nonces to veirfy
// that the entire set will get dropped (case 4).
var (
filler, _ = crypto.GenerateKey()
filled = make(map[uint64]struct{})
filled = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // account nonce at 3, all set filled
tx := makeTx(nonce, 1, 1, 1, filler)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
filled[id] = struct{}{}
store.Put(blob)
filled[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with partially passed nonces to verify
// that the included part of the set will get dropped (case 4).
var (
overlapper, _ = crypto.GenerateKey()
overlapped = make(map[uint64]struct{})
overlapped = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2, 3} { // account nonce at 2, half filled
tx := makeTx(nonce, 1, 1, 1, overlapper)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce >= 2 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
overlapped[id] = struct{}{}
overlapped[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions with an underpriced first to verify that
// the entire set will get dropped (case 5).
var (
underpayer, _ = crypto.GenerateKey()
underpaid = make(map[uint64]struct{})
underpaid = make(map[common.Hash]struct{})
)
for i := 0; i < 5; i++ { // make #0 underpriced
var tx *types.Transaction
@ -603,15 +608,15 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
id, _ := store.Put(blob)
underpaid[id] = struct{}{}
store.Put(blob)
underpaid[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with an underpriced in between to verify
// that it and anything newly gapped will get evicted (case 5).
var (
outpricer, _ = crypto.GenerateKey()
outpriced = make(map[uint64]struct{})
outpriced = make(map[common.Hash]struct{})
)
for i := 0; i < 5; i++ { // make #2 underpriced
var tx *types.Transaction
@ -622,18 +627,18 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if i < 2 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
outpriced[id] = struct{}{}
outpriced[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions fully overdrafted to verify that the
// entire set will get invalidated (case 6).
var (
exceeder, _ = crypto.GenerateKey()
exceeded = make(map[uint64]struct{})
exceeded = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // nonce 0 overdrafts the account
var tx *types.Transaction
@ -644,14 +649,14 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
id, _ := store.Put(blob)
exceeded[id] = struct{}{}
store.Put(blob)
exceeded[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions partially overdrafted to verify that part
// of the set will get invalidated (case 6).
var (
overdrafter, _ = crypto.GenerateKey()
overdrafted = make(map[uint64]struct{})
overdrafted = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // nonce 1 overdrafts the account
var tx *types.Transaction
@ -662,44 +667,46 @@ func TestOpenDrops(t *testing.T) {
}
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce < 1 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
overdrafted[id] = struct{}{}
overdrafted[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions overflowing the account cap to verify
// that part of the set will get invalidated (case 7).
var (
overcapper, _ = crypto.GenerateKey()
overcapped = make(map[uint64]struct{})
overcapped = make(map[common.Hash]struct{})
)
for nonce := uint64(0); nonce < maxTxsPerAccount+3; nonce++ {
blob := encodeForPool(makeTx(nonce, 1, 1, 1, overcapper))
tx := makeTx(nonce, 1, 1, 1, overcapper)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce < maxTxsPerAccount {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
overcapped[id] = struct{}{}
overcapped[tx.Hash()] = struct{}{}
}
}
// Insert a batch of duplicated transactions to verify that only one of each
// version will remain (case 8).
var (
duplicater, _ = crypto.GenerateKey()
duplicated = make(map[uint64]struct{})
duplicated = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} {
blob := encodeForPool(makeTx(nonce, 1, 1, 1, duplicater))
tx := makeTx(nonce, 1, 1, 1, duplicater)
blob := encodeForPool(tx)
for i := 0; i < int(nonce)+1; i++ {
id, _ := store.Put(blob)
store.Put(blob)
if i == 0 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
duplicated[id] = struct{}{}
duplicated[tx.Hash()] = struct{}{}
}
}
}
@ -707,17 +714,18 @@ func TestOpenDrops(t *testing.T) {
// remain (case 9).
var (
repeater, _ = crypto.GenerateKey()
repeated = make(map[uint64]struct{})
repeated = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} {
for i := 0; i < int(nonce)+1; i++ {
blob := encodeForPool(makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater))
tx := makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater)
blob := encodeForPool(tx)
id, _ := store.Put(blob)
store.Put(blob)
if i == 0 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
repeated[id] = struct{}{}
repeated[tx.Hash()] = struct{}{}
}
}
}
@ -754,39 +762,41 @@ func TestOpenDrops(t *testing.T) {
// Verify that the malformed (case 1), badly signed (case 2) and gapped (case
// 3) txs have been deleted from the pool
alive := make(map[uint64]struct{})
alive := make(map[common.Hash]struct{})
for _, txs := range pool.index {
for _, tx := range txs {
switch tx.id {
case malformed:
t.Errorf("malformed RLP transaction remained in storage")
case badsig:
t.Errorf("invalidly signed transaction remained in storage")
default:
if _, ok := dangling[tx.id]; ok {
if badsig == tx.hash {
t.Errorf("invalidly signed transaction remained in storage")
}
if _, ok := dangling[tx.hash]; ok {
t.Errorf("dangling transaction remained in storage: %d", tx.id)
} else if _, ok := filled[tx.id]; ok {
} else if _, ok := filled[tx.hash]; ok {
t.Errorf("filled transaction remained in storage: %d", tx.id)
} else if _, ok := overlapped[tx.id]; ok {
} else if _, ok := overlapped[tx.hash]; ok {
t.Errorf("overlapped transaction remained in storage: %d", tx.id)
} else if _, ok := gapped[tx.id]; ok {
} else if _, ok := gapped[tx.hash]; ok {
t.Errorf("gapped transaction remained in storage: %d", tx.id)
} else if _, ok := underpaid[tx.id]; ok {
} else if _, ok := underpaid[tx.hash]; ok {
t.Errorf("underpaid transaction remained in storage: %d", tx.id)
} else if _, ok := outpriced[tx.id]; ok {
} else if _, ok := outpriced[tx.hash]; ok {
t.Errorf("outpriced transaction remained in storage: %d", tx.id)
} else if _, ok := exceeded[tx.id]; ok {
} else if _, ok := exceeded[tx.hash]; ok {
t.Errorf("fully overdrafted transaction remained in storage: %d", tx.id)
} else if _, ok := overdrafted[tx.id]; ok {
} else if _, ok := overdrafted[tx.hash]; ok {
t.Errorf("partially overdrafted transaction remained in storage: %d", tx.id)
} else if _, ok := overcapped[tx.id]; ok {
} else if _, ok := overcapped[tx.hash]; ok {
t.Errorf("overcapped transaction remained in storage: %d", tx.id)
} else if _, ok := duplicated[tx.id]; ok {
t.Errorf("duplicated transaction remained in storage: %d", tx.id)
} else if _, ok := repeated[tx.id]; ok {
} else if _, ok := repeated[tx.hash]; ok {
t.Errorf("repeated nonce transaction remained in storage: %d", tx.id)
} else {
alive[tx.id] = struct{}{}
if _, ok := alive[tx.hash]; ok {
t.Errorf("duplicated transaction remained in storage: %d", tx.id)
}
alive[tx.hash] = struct{}{}
}
}
}
@ -795,14 +805,14 @@ func TestOpenDrops(t *testing.T) {
if len(alive) != len(valids) {
t.Errorf("valid transaction count mismatch: have %d, want %d", len(alive), len(valids))
}
for id := range alive {
if _, ok := valids[id]; !ok {
t.Errorf("extra transaction %d", id)
for hash := range alive {
if _, ok := valids[hash]; !ok {
t.Errorf("extra transaction %s", hash)
}
}
for id := range valids {
if _, ok := alive[id]; !ok {
t.Errorf("missing transaction %d", id)
for hash := range valids {
if _, ok := alive[hash]; !ok {
t.Errorf("missing transaction %s", hash)
}
}
// Verify all the calculated pool internals. Interestingly, this is **not**
@ -1021,7 +1031,10 @@ func TestOpenCap(t *testing.T) {
keep = []common.Address{addr1, addr3}
drop = []common.Address{addr2}
size = 2 * (txAvgSize + blobSize + uint64(txBlobOverhead))
// After migration to pooledBlobTx, cells (128 x 2048 = 2*blobSize) replace blobs.
// The actual billy slot size for pooledBlobTx is 2*(blobSize+txBlobOverhead)+txAvgSize.
pooledSlotSize uint64 = 2*(blobSize+uint64(txBlobOverhead)) + txAvgSize
size = 2 * pooledSlotSize
)
store.Put(blob1)
store.Put(blob2)
@ -1030,7 +1043,7 @@ func TestOpenCap(t *testing.T) {
// Verify pool capping twice: first by reducing the data cap, then restarting
// with a high cap to ensure everything was persisted previously
for _, datacap := range []uint64{2 * (txAvgSize + blobSize + uint64(txBlobOverhead)), 1000 * (txAvgSize + blobSize + uint64(txBlobOverhead))} {
for _, datacap := range []uint64{size, 1000 * pooledSlotSize} {
// Create a blob pool out of the pre-seeded data, but cap it to 2 blob transaction
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
@ -1357,7 +1370,7 @@ func TestLegacyTxConversion(t *testing.T) {
// Legacy formats should not exist on pool.store
pool.store.Iterate(func(id uint64, size uint32, blob []byte) {
var ptx blobTxForPool
var ptx BlobTxForPool
if err := rlp.DecodeBytes(blob, &ptx); err != nil {
t.Errorf("entry %d not in new blobTxForPool format: %v", id, err)
}
@ -1415,7 +1428,7 @@ func TestBlobCountLimit(t *testing.T) {
// Check that first succeeds second fails.
if errs[0] != nil {
t.Fatalf("expected tx with 7 blobs to succeed, got %v", errs[0])
t.Fatalf("expected tx with 7 blobs to succeed, got: %v", errs[0])
}
if !errors.Is(errs[1], txpool.ErrTxBlobLimitExceeded) {
t.Fatalf("expected tx with 8 blobs to fail, got: %v", errs[1])
@ -2140,32 +2153,6 @@ func TestGetBlobs(t *testing.T) {
pool.Close()
}
// TestEncodeForNetwork verifies that encodeForNetwork produces output identical
// to rlp.EncodeToBytes on the original transaction, for both V0 and V1 sidecars.
func TestEncodeForNetwork(t *testing.T) {
t.Run("v0", func(t *testing.T) { testEncodeForNetwork(t, types.BlobSidecarVersion0) })
t.Run("v1", func(t *testing.T) { testEncodeForNetwork(t, types.BlobSidecarVersion1) })
}
func testEncodeForNetwork(t *testing.T, version byte) {
key, _ := crypto.GenerateKey()
tx := makeMultiBlobTx(0, 1, 1, 1, 1, 0, key, version)
wantRLP, err := rlp.EncodeToBytes(tx)
if err != nil {
t.Fatalf("failed to encode tx: %v", err)
}
storedRLP := encodeForPool(tx)
gotRLP, err := encodeForNetwork(storedRLP)
if err != nil {
t.Fatalf("encodeForNetwork failed: %v", err)
}
if !bytes.Equal(gotRLP, wantRLP) {
t.Fatalf("network encoding mismatch (version %d): got %d bytes, want %d bytes", version, len(gotRLP), len(wantRLP))
}
}
// fakeBilly is a billy.Database implementation which just drops data on the floor.
type fakeBilly struct {
billy.Database
@ -2228,7 +2215,8 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) {
b.Fatal(err)
}
statedb.AddBalance(addr, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
pool.add(tx)
pooledTx, _ := newBlobTxForPool(tx)
pool.AddPooledTx(pooledTx)
}
statedb.Commit(0, true, false)
defer pool.Close()
@ -2249,3 +2237,161 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) {
}
}
}
func TestGetCells(t *testing.T) {
storage := t.TempDir()
os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700)
store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(params.BlobTxMaxBlobs), nil)
var (
key1, _ = crypto.GenerateKey()
addr1 = crypto.PubkeyToAddress(key1.PublicKey)
tx1 = makeMultiBlobTx(0, 1, 1000, 100, 3, 0, key1, types.BlobSidecarVersion1) // blobs [0, 3)
pooledTx1, _ = newBlobTxForPool(tx1)
blob1, _ = rlp.EncodeToBytes(pooledTx1)
)
store.Put(blob1)
store.Close()
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
statedb.Commit(0, true, false)
chain := &testBlockChain{
config: params.MainnetChainConfig,
basefee: uint256.NewInt(params.InitialBaseFee),
blobfee: uint256.NewInt(params.BlobTxMinBlobGasprice),
statedb: statedb,
}
pool := New(Config{Datadir: storage}, chain, nil)
if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil {
t.Fatalf("failed to create blob pool: %v", err)
}
defer pool.Close()
tests := []struct {
name string
hash common.Hash
mask types.CustodyBitmap
expectedLen int
shouldFail bool
}{
{
name: "Get cells with single index",
hash: tx1.Hash(),
mask: types.NewCustodyBitmap([]uint64{5}),
expectedLen: 3,
shouldFail: false,
},
{
name: "Get cells with all indices",
hash: tx1.Hash(),
mask: *types.CustodyBitmapAll,
expectedLen: 384,
shouldFail: false,
},
{
name: "Get cells with no indices",
hash: tx1.Hash(),
mask: types.CustodyBitmap{},
expectedLen: 0,
shouldFail: false,
},
{
name: "Get cells for non-existent transaction",
hash: common.Hash{0x01, 0x02, 0x03},
mask: types.NewCustodyBitmap([]uint64{0, 1}),
expectedLen: 0,
shouldFail: true,
},
{
name: "Get cells with random indices",
hash: tx1.Hash(),
mask: types.NewRandomCustodyBitmap(8),
expectedLen: 24,
shouldFail: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
vhashes := pool.GetBlobHashes(tt.hash)
if tt.shouldFail {
if vhashes != nil {
t.Errorf("expected nil vhashes for non-existent tx")
}
return
}
if vhashes == nil {
t.Fatalf("expected vhashes, got nil")
}
blobCells, _, err := pool.GetBlobCells(vhashes, tt.mask)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Count total non-nil cells across all blobs
totalCells := 0
for _, bc := range blobCells {
for _, c := range bc {
if c != nil {
totalCells++
}
}
}
if totalCells != tt.expectedLen {
t.Errorf("expected %d cells, got %d", tt.expectedLen, totalCells)
}
})
}
}
// TestEncodeForNetwork verifies that encodeForNetwork produces the correct wire
// encoding for each (sidecar version, eth protocol version) combination.
// - eth/69, eth/70: blobs recovered from cells, output matches the original tx
// - eth/72: blob payload omitted, output matches tx.WithoutBlob()
func TestEncodeForNetwork(t *testing.T) {
cases := []struct {
name string
sidecarVer byte
ethVer uint
}{
{"v0/eth70", types.BlobSidecarVersion0, 70},
{"v1/eth70", types.BlobSidecarVersion1, 70},
{"v0/eth72", types.BlobSidecarVersion0, 72},
{"v1/eth72", types.BlobSidecarVersion1, 72},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
testEncodeForNetwork(t, tc.sidecarVer, tc.ethVer)
})
}
}
func testEncodeForNetwork(t *testing.T, sidecarVer byte, ethVer uint) {
key, _ := crypto.GenerateKey()
tx := makeMultiBlobTx(0, 1, 1, 1, 1, 0, key, sidecarVer)
wantTx := tx
if ethVer >= 72 {
wantTx = tx.WithoutBlob()
}
wantRLP, err := rlp.EncodeToBytes(wantTx)
if err != nil {
t.Fatalf("failed to encode tx: %v", err)
}
storedRLP := encodeForPool(tx)
gotRLP, err := encodeForNetwork(storedRLP, ethVer)
if err != nil {
t.Fatalf("encodeForNetwork failed: %v", err)
}
if !bytes.Equal(gotRLP, wantRLP) {
t.Fatalf("network encoding mismatch: got %d bytes, want %d bytes", len(gotRLP), len(wantRLP))
}
}

295
core/txpool/blobpool/buffer.go Normal file
View file

@ -0,0 +1,295 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package blobpool
import (
"cmp"
"fmt"
"slices"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
)
// todo: per-peer size limit
var (
blobBufferTxFirstCounter = metrics.NewRegisteredCounter("blobpool/buffer/txfirst", nil)
blobBufferCellsFirstCounter = metrics.NewRegisteredCounter("blobpool/buffer/cellsfirst", nil)
blobBufferTotalTx = metrics.NewRegisteredGauge("blobpool/buffer/txcount", nil)
blobBufferTotalCells = metrics.NewRegisteredGauge("blobpool/buffer/cellcount", nil)
)
const (
bufferLifetime = 2 * time.Minute
)
// PeerDelivery holds cells delivered by a single peer, in blob-major order.
type PeerDelivery struct {
Cells []kzg4844.Cell
Indices []uint64
}
type txEntry struct {
tx *types.Transaction
// Technically it is not required to store peer information to drop properly.
// This is mainly for per peer size limit check.
peer string
added time.Time
}
type cellEntry struct {
deliveries map[string]*PeerDelivery
custody *types.CustodyBitmap
added time.Time
}
type BlobBuffer struct {
txs map[common.Hash]*txEntry
cells map[common.Hash]*cellEntry
addToPool func(*BlobTxForPool) error
validateTx func(*types.Transaction) error
dropPeer func(string)
}
func NewBlobBuffer(validateTx func(*types.Transaction) error, addToPool func(*BlobTxForPool) error, dropPeer func(string)) *BlobBuffer {
return &BlobBuffer{
txs: make(map[common.Hash]*txEntry),
cells: make(map[common.Hash]*cellEntry),
validateTx: validateTx,
addToPool: addToPool,
dropPeer: dropPeer,
}
}
// AddTx buffers a blob transaction (without blobs) from an ETH/72 peer.
// If cells are already buffered, verification and pool insertion are attempted.
func (b *BlobBuffer) AddTx(tx *types.Transaction, peer string) error {
defer b.updateMetrics()()
// First remove any timed-out entries.
b.evict()
hash := tx.Hash()
sidecar := tx.BlobTxSidecar()
if sidecar == nil {
return fmt.Errorf("blob transaction without sidecar")
}
// tx validation
if err := b.validateTx(tx); err != nil {
log.Warn("Transaction validation failed, dropping peer", "peer", peer, "err", err)
b.dropPeer(peer)
return err
}
// vhash check
if err := sidecar.ValidateBlobCommitmentHashes(tx.BlobHashes()); err != nil {
log.Warn("Commitment hash mismatch, dropping peer", "peer", peer, "err", err)
b.dropPeer(peer)
return err
}
// proof count check
if len(sidecar.Proofs) < len(sidecar.Commitments)*kzg4844.CellProofsPerBlob {
b.dropPeer(peer)
return fmt.Errorf("insufficient proofs in sidecar")
}
if entry, ok := b.cells[hash]; ok {
return b.add(hash, tx, entry)
}
blobBufferTxFirstCounter.Inc(1)
b.txs[hash] = &txEntry{tx: tx, peer: peer, added: time.Now()}
return nil
}
// AddCells buffers per-peer cell deliveries from the blob fetcher.
// If the transaction is already buffered, verification and pool insertion are attempted.
func (b *BlobBuffer) AddCells(hash common.Hash, deliveries map[string]*PeerDelivery, custody *types.CustodyBitmap) error {
defer b.updateMetrics()()
// First remove any timed-out entries.
b.evict()
b.cells[hash] = &cellEntry{
deliveries: deliveries,
custody: custody,
added: time.Now(),
}
if txe, ok := b.txs[hash]; ok {
return b.add(hash, txe.tx, b.cells[hash])
}
blobBufferCellsFirstCounter.Inc(1)
return nil
}
// todo returning error here is strange
// add verifies cells per-peer, sorts them, and adds to the pool.
func (b *BlobBuffer) add(hash common.Hash, tx *types.Transaction, cells *cellEntry) error {
sidecar := tx.BlobTxSidecar()
// Per-peer cell verification
if badPeers := b.verifyCells(cells, sidecar); len(badPeers) > 0 {
b.dropPeers(badPeers)
delete(b.cells, hash)
delete(b.txs, hash)
return fmt.Errorf("cell verification failed")
}
blobCount := len(tx.BlobHashes())
sorted, custody := sortCells(cells, blobCount)
pooledTx := &BlobTxForPool{
Tx: tx.WithoutBlobTxSidecar(),
Version: sidecar.Version,
Commitments: sidecar.Commitments,
Proofs: sidecar.Proofs,
Cells: sorted,
Custody: *custody,
}
err := b.addToPool(pooledTx)
delete(b.cells, hash)
delete(b.txs, hash)
return err
}
func (b *BlobBuffer) HasTx(hash common.Hash) bool {
_, ok := b.txs[hash]
return ok
}
func (b *BlobBuffer) HasCells(hash common.Hash) bool {
_, ok := b.cells[hash]
return ok
}
func (b *BlobBuffer) dropPeers(peers []string) {
if b.dropPeer == nil {
return
}
for _, p := range peers {
b.dropPeer(p)
}
}
func (b *BlobBuffer) evict() {
now := time.Now()
for hash, entry := range b.txs {
if now.Sub(entry.added) > bufferLifetime {
delete(b.txs, hash)
}
}
for hash, entry := range b.cells {
if now.Sub(entry.added) > bufferLifetime {
delete(b.cells, hash)
}
}
}
// updateMetrics updates the metrics gauges.
// This should be called at the start of any operation that changes the buffer
// content. The returned function is to be called at the end of the operation,
// usually with defer.
func (b *BlobBuffer) updateMetrics() func() {
preTxCount := len(b.txs)
preCellsCount := len(b.cells)
return func() {
if len(b.txs) != preTxCount {
blobBufferTotalTx.Update(int64(len(b.txs)))
}
if len(b.cells) != preCellsCount {
blobBufferTotalCells.Update(int64(len(b.cells)))
}
}
}
// verifyCells verifies each peer's cells against the sidecar.
// Returns the list of peers whose cells failed verification.
func (b *BlobBuffer) verifyCells(entry *cellEntry, sidecar *types.BlobTxSidecar) []string {
var badPeers []string
for peer, delivery := range entry.deliveries {
if err := verifyPeerCells(delivery, sidecar); err != nil {
log.Debug("Cell verification failed", "peer", peer, "err", err)
badPeers = append(badPeers, peer)
}
}
return badPeers
}
// verifyPeerCells verifies a single peer's cells against the sidecar proofs.
// delivery.Cells is blob-major: [blob0_cell0..blob0_cellN, blob1_cell0..blob1_cellN, ...]
func verifyPeerCells(delivery *PeerDelivery, sidecar *types.BlobTxSidecar) error {
cellsPerBlob := len(delivery.Indices)
blobCount := len(delivery.Cells) / cellsPerBlob
if blobCount == 0 || blobCount != len(sidecar.Commitments) {
return fmt.Errorf("blob count mismatch: delivery %d, commitments %d", blobCount, len(sidecar.Commitments))
}
// Extract proofs corresponding to this peer's cell indices
var proofs []kzg4844.Proof
for blobIdx := 0; blobIdx < blobCount; blobIdx++ {
for _, cellIdx := range delivery.Indices {
proofIdx := blobIdx*kzg4844.CellProofsPerBlob + int(cellIdx)
if proofIdx >= len(sidecar.Proofs) {
return fmt.Errorf("proof index out of range: %d", proofIdx)
}
proofs = append(proofs, sidecar.Proofs[proofIdx])
}
}
return kzg4844.VerifyCells(delivery.Cells, sidecar.Commitments, proofs, delivery.Indices)
}
// sortCells merges all per-peer deliveries into a single flat cell array
// sorted by custody index.
//
// e.g.
// peer A: cells = [blob0_cell5, blob0_cell3, blob1_cell5, blob1_cell3]
// peer B: cells = [blob0_cell1, blob0_cell7, blob1_cell1, blob1_cell7]
// -> [blob0_cell1, blob0_cell3, blob0_cell5, blob0_cell7, blob1_cell1, blob1_cell3, blob1_cell5, blob1_cell7]
func sortCells(entry *cellEntry, blobCount int) ([]kzg4844.Cell, *types.CustodyBitmap) {
// indices per delivery
var indices []uint64
// 1. compose per blob cells
blob := make([][]kzg4844.Cell, blobCount)
for _, d := range entry.deliveries {
n := len(d.Indices)
indices = append(indices, d.Indices...)
for b := range blobCount {
blob[b] = append(blob[b], d.Cells[b*n:(b+1)*n]...)
}
}
// 2. sort
perm := make([]int, len(indices))
for i := range perm {
perm[i] = i
}
// perm represents the position of cells in sorted array
slices.SortFunc(perm, func(a, b int) int {
return cmp.Compare(indices[a], indices[b])
})
// reorder cells
var res []kzg4844.Cell
for b := range blobCount {
for _, p := range perm {
res = append(res, blob[b][p])
}
}
custody := types.NewCustodyBitmap(indices)
return res, &custody
}

257
core/txpool/blobpool/buffer_test.go Normal file
View file

@ -0,0 +1,257 @@
package blobpool
import (
"crypto/ecdsa"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/params"
"github.com/holiman/uint256"
)
// makeV1Tx creates a V1 blob transaction with cell proofs, then strips blobs
// (simulating what ETH/72 peers send).
func makeV1Tx(t *testing.T, nonce uint64, blobCount int, blobOffset int, key *ecdsa.PrivateKey) *types.Transaction {
t.Helper()
tx := makeMultiBlobTx(nonce, 1, 1, 1, blobCount, blobOffset, key, types.BlobSidecarVersion1)
return tx.WithoutBlob()
}
// makePeerDelivery creates a PeerDelivery for given cell indices from a set of blobs.
func makePeerDelivery(t *testing.T, blobOffset, blobCount int, indices []uint64) *PeerDelivery {
t.Helper()
var allCells []kzg4844.Cell
for i := 0; i < blobCount; i++ {
cells, err := kzg4844.ComputeCells([]kzg4844.Blob{*testBlobs[blobOffset+i]})
if err != nil {
t.Fatal(err)
}
allCells = append(allCells, cells...)
}
var deliveryCells []kzg4844.Cell
for b := 0; b < blobCount; b++ {
for _, idx := range indices {
deliveryCells = append(deliveryCells, allCells[b*kzg4844.CellsPerBlob+int(idx)])
}
}
return &PeerDelivery{Cells: deliveryCells, Indices: indices}
}
func newTestBuffer(t *testing.T) *BlobBuffer {
t.Helper()
return NewBlobBuffer(
func(tx *types.Transaction) error { return nil },
func(ptx *BlobTxForPool) error { return nil },
func(peer string) {},
)
}
func TestSortCells(t *testing.T) {
blobCount := 2
blobOffset := 0
peerA := makePeerDelivery(t, blobOffset, blobCount, []uint64{5, 3})
peerB := makePeerDelivery(t, blobOffset, blobCount, []uint64{1, 7})
custody := types.NewCustodyBitmap([]uint64{1, 3, 5, 7})
entry := &cellEntry{
deliveries: map[string]*PeerDelivery{
"peerA": peerA,
"peerB": peerB,
},
custody: &custody,
}
sorted, resultCustody := sortCells(entry, blobCount)
resultIndices := resultCustody.Indices()
if len(resultIndices) != 4 {
t.Fatalf("expected 4 indices, got %d", len(resultIndices))
}
for i, expected := range []uint64{1, 3, 5, 7} {
if resultIndices[i] != expected {
t.Errorf("index %d: expected %d, got %d", i, expected, resultIndices[i])
}
}
expected := makePeerDelivery(t, blobOffset, blobCount, []uint64{1, 3, 5, 7})
if len(sorted) != len(expected.Cells) {
t.Fatalf("sorted length %d != expected %d", len(sorted), len(expected.Cells))
}
for i := range sorted {
if sorted[i] != expected.Cells[i] {
t.Errorf("cell %d mismatch", i)
}
}
}
func TestAddTxThenCells(t *testing.T) {
key, _ := crypto.GenerateKey()
blobCount := 2
buf := newTestBuffer(t)
tx := makeV1Tx(t, 0, blobCount, 0, key)
hash := tx.Hash()
if err := buf.AddTx(tx, "peerA"); err != nil {
t.Fatal(err)
}
if !buf.HasTx(hash) {
t.Fatal("tx should be buffered")
}
dataIndices := make([]uint64, kzg4844.DataPerBlob)
for i := range dataIndices {
dataIndices[i] = uint64(i)
}
delivery := makePeerDelivery(t, 0, blobCount, dataIndices)
custody := types.NewCustodyBitmap(dataIndices)
if err := buf.AddCells(hash, map[string]*PeerDelivery{"peerB": delivery}, &custody); err != nil {
t.Fatal(err)
}
if buf.HasTx(hash) || buf.HasCells(hash) {
t.Fatal("buffer should be empty after add")
}
}
func TestAddCellsThenTx(t *testing.T) {
key, _ := crypto.GenerateKey()
blobCount := 2
buf := newTestBuffer(t)
tx := makeV1Tx(t, 0, blobCount, 0, key)
hash := tx.Hash()
dataIndices := make([]uint64, kzg4844.DataPerBlob)
for i := range dataIndices {
dataIndices[i] = uint64(i)
}
delivery := makePeerDelivery(t, 0, blobCount, dataIndices)
custody := types.NewCustodyBitmap(dataIndices)
if err := buf.AddCells(hash, map[string]*PeerDelivery{"peerB": delivery}, &custody); err != nil {
t.Fatal(err)
}
if !buf.HasCells(hash) {
t.Fatal("cells should be buffered")
}
if err := buf.AddTx(tx, "peerA"); err != nil {
t.Fatal(err)
}
if buf.HasTx(hash) || buf.HasCells(hash) {
t.Fatal("buffer should be empty after add")
}
}
func TestMultiPeerDelivery(t *testing.T) {
key, _ := crypto.GenerateKey()
blobCount := 2
buf := newTestBuffer(t)
tx := makeV1Tx(t, 0, blobCount, 0, key)
hash := tx.Hash()
buf.AddTx(tx, "peerA")
indicesA := []uint64{0, 2, 4, 6}
indicesB := []uint64{1, 3, 5, 7}
deliveryA := makePeerDelivery(t, 0, blobCount, indicesA)
deliveryB := makePeerDelivery(t, 0, blobCount, indicesB)
allIndices := append(indicesA, indicesB...)
custody := types.NewCustodyBitmap(allIndices)
if err := buf.AddCells(hash, map[string]*PeerDelivery{
"peerB": deliveryA,
"peerC": deliveryB,
}, &custody); err != nil {
t.Fatal(err)
}
if buf.HasTx(hash) || buf.HasCells(hash) {
t.Fatal("buffer should be empty after add")
}
}
func TestBadCell(t *testing.T) {
key, _ := crypto.GenerateKey()
blobCount := 1
var dropped []string
buf := NewBlobBuffer(
func(tx *types.Transaction) error { return nil },
func(ptx *BlobTxForPool) error { return nil },
func(peer string) { dropped = append(dropped, peer) },
)
tx := makeV1Tx(t, 0, blobCount, 0, key)
hash := tx.Hash()
buf.AddTx(tx, "peerA")
goodDelivery := makePeerDelivery(t, 0, blobCount, []uint64{0, 1, 2, 3})
badDelivery := makePeerDelivery(t, 0, blobCount, []uint64{4, 5, 6, 7})
for i := range badDelivery.Cells {
for j := range badDelivery.Cells[i] {
badDelivery.Cells[i][j] ^= 0xFF
}
}
allIndices := []uint64{0, 1, 2, 3, 4, 5, 6, 7}
custody := types.NewCustodyBitmap(allIndices)
err := buf.AddCells(hash, map[string]*PeerDelivery{
"peerB": goodDelivery,
"peerC": badDelivery,
}, &custody)
if err == nil {
t.Fatal("expected error from bad cells")
}
if len(dropped) != 1 || dropped[0] != "peerC" {
t.Fatalf("only peerC should have been dropped, got: %v", dropped)
}
if buf.HasTx(hash) || buf.HasCells(hash) {
t.Fatal("buffer should be empty after bad cell drop")
}
}
func TestBadTx(t *testing.T) {
key, _ := crypto.GenerateKey()
var dropped []string
buf := NewBlobBuffer(
func(tx *types.Transaction) error { return nil },
func(ptx *BlobTxForPool) error { return nil },
func(peer string) { dropped = append(dropped, peer) },
)
blobtx := &types.BlobTx{
ChainID: uint256.MustFromBig(params.MainnetChainConfig.ChainID),
Nonce: 0,
GasTipCap: uint256.NewInt(1),
GasFeeCap: uint256.NewInt(1),
Gas: 21000,
BlobFeeCap: uint256.NewInt(1),
BlobHashes: []common.Hash{testBlobVHashes[0]},
Value: uint256.NewInt(100),
Sidecar: types.NewBlobTxSidecar(types.BlobSidecarVersion1,
nil,
[]kzg4844.Commitment{testBlobCommits[1]},
testBlobCellProofs[1],
),
}
tx := types.MustSignNewTx(key, types.LatestSigner(params.MainnetChainConfig), blobtx)
err := buf.AddTx(tx, "peerA")
if err == nil {
t.Fatal("expected error from commitment mismatch")
}
if len(dropped) != 1 || dropped[0] != "peerA" {
t.Fatalf("only peerA should have been dropped, got: %v", dropped)
}
if buf.HasTx(tx.Hash()) {
t.Fatal("tx should not be buffered")
}
}

10
core/txpool/blobpool/limbo.go
View file

@ -33,7 +33,7 @@ import (
type limboBlob struct {
TxHash common.Hash // Owner transaction's hash to support resurrecting reorged txs
Block uint64 // Block in which the blob transaction was included
Ptx *blobTxForPool
Ptx *BlobTxForPool
}
// limbo is a light, indexed database to temporarily store recently included
@ -146,7 +146,9 @@ func (l *limbo) finalize(final *types.Header) {
// push stores a new blob transaction into the limbo, waiting until finality for
// it to be automatically evicted.
func (l *limbo) push(ptx *blobTxForPool, block uint64) error {
func (l *limbo) push(ptx *BlobTxForPool, block uint64) error {
// If the blobs are already tracked by the limbo, consider it a programming
// error. There's not much to do against it, but be loud.
hash := ptx.Tx.Hash()
if _, ok := l.index[hash]; ok {
log.Error("Limbo cannot push already tracked blobs", "tx", hash)
@ -162,7 +164,7 @@ func (l *limbo) push(ptx *blobTxForPool, block uint64) error {
// pull retrieves a previously pushed set of blobs back from the limbo, removing
// it at the same time. This method should be used when a previously included blob
// transaction gets reorged out.
func (l *limbo) pull(tx common.Hash) (*blobTxForPool, error) {
func (l *limbo) pull(tx common.Hash) (*BlobTxForPool, error) {
// If the blobs are not tracked by the limbo, there's not much to do. This
// can happen for example if a blob transaction is mined without pushing it
// into the network first.
@ -239,7 +241,7 @@ func (l *limbo) getAndDrop(id uint64) (*limboBlob, error) {
// setAndIndex assembles a limbo blob database entry and stores it, also updating
// the in-memory indices.
func (l *limbo) setAndIndex(ptx *blobTxForPool, block uint64) error {
func (l *limbo) setAndIndex(ptx *BlobTxForPool, block uint64) error {
txhash := ptx.Tx.Hash()
item := &limboBlob{
TxHash: txhash,

24
core/txpool/blobpool/lookup.go
View file

@ -18,11 +18,15 @@ package blobpool
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
type txMetadata struct {
id uint64 // the billy id of transction
size uint64 // the RLP encoded size of transaction (blobs are included)
id uint64 // the billy id of transction
size uint64 // the RLP encoded size of transaction (blobs are included)
sizeWithoutBlob uint64 // the RLP encoded size without blob data (for ETH/72 announcements)
custody types.CustodyBitmap
vhashes []common.Hash // blob versioned hashes for the transaction
}
// lookup maps blob versioned hashes to transaction hashes that include them,
@ -56,6 +60,15 @@ func (l *lookup) storeidOfTx(txhash common.Hash) (uint64, bool) {
return meta.id, true
}
// blobHashesOfTx returns the blob versioned hashes for a transaction.
func (l *lookup) blobHashesOfTx(txhash common.Hash) ([]common.Hash, bool) {
meta, ok := l.txIndex[txhash]
if !ok {
return nil, false
}
return meta.vhashes, true
}
// storeidOfBlob returns the datastore storage item id of a blob.
func (l *lookup) storeidOfBlob(vhash common.Hash) (uint64, bool) {
// If the blob is unknown, return a miss
@ -91,8 +104,11 @@ func (l *lookup) track(tx *blobTxMeta) {
}
// Map the transaction hash to the datastore id and RLP-encoded transaction size
l.txIndex[tx.hash] = &txMetadata{
id: tx.id,
size: tx.size,
id: tx.id,
size: tx.size,
sizeWithoutBlob: tx.sizeWithoutBlob,
custody: *tx.custody,
vhashes: tx.vhashes,
}
}

2
core/txpool/legacypool/legacypool.go
View file

@ -1010,7 +1010,7 @@ func (pool *LegacyPool) get(hash common.Hash) *types.Transaction {
}
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
func (pool *LegacyPool) GetRLP(hash common.Hash) []byte {
func (pool *LegacyPool) GetRLP(hash common.Hash, _ uint) []byte {
tx := pool.all.Get(hash)
if tx == nil {
return nil

7
core/txpool/subpool.go
View file

@ -86,8 +86,9 @@ type PendingFilter struct {
// TxMetadata denotes the metadata of a transaction.
type TxMetadata struct {
Type uint8 // The type of the transaction
Size uint64 // The length of the 'rlp encoding' of a transaction
Type uint8 // The type of the transaction
Size uint64 // The length of the 'rlp encoding' of a transaction (including blobs)
SizeWithoutBlob uint64 // The length without blob data (for ETH/72 announcements)
}
// SubPool represents a specialized transaction pool that lives on its own (e.g.
@ -132,7 +133,7 @@ type SubPool interface {
Get(hash common.Hash) *types.Transaction
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
GetRLP(hash common.Hash) []byte
GetRLP(hash common.Hash, version uint) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.

4
core/txpool/txpool.go
View file

@ -287,9 +287,9 @@ func (p *TxPool) Get(hash common.Hash) *types.Transaction {
}
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
func (p *TxPool) GetRLP(hash common.Hash) []byte {
func (p *TxPool) GetRLP(hash common.Hash, version uint) []byte {
for _, subpool := range p.subpools {
encoded := subpool.GetRLP(hash)
encoded := subpool.GetRLP(hash, version)
if len(encoded) != 0 {
return encoded
}

85
core/txpool/validation.go
View file

@ -64,9 +64,6 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
if opts.Accept&(1<<tx.Type()) == 0 {
return fmt.Errorf("%w: tx type %v not supported by this pool", core.ErrTxTypeNotSupported, tx.Type())
}
if blobCount := len(tx.BlobHashes()); blobCount > opts.MaxBlobCount {
return fmt.Errorf("%w: blob count %v, limit %v", ErrTxBlobLimitExceeded, blobCount, opts.MaxBlobCount)
}
// Before performing any expensive validations, sanity check that the tx is
// smaller than the maximum limit the pool can meaningfully handle
if tx.Size() > opts.MaxSize {
@ -146,9 +143,6 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
if tx.GasTipCapIntCmp(opts.MinTip) < 0 {
return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.GasTipCap(), opts.MinTip)
}
if tx.Type() == types.BlobTxType {
return validateBlobTx(tx, head, opts)
}
if tx.Type() == types.SetCodeTxType {
if len(tx.SetCodeAuthorizations()) == 0 {
return errors.New("set code tx must have at least one authorization tuple")
@ -157,14 +151,33 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
return nil
}
// validateBlobTx implements the blob-transaction specific validations.
func validateBlobTx(tx *types.Transaction, head *types.Header, opts *ValidationOptions) error {
sidecar := tx.BlobTxSidecar()
if sidecar == nil {
return errors.New("missing sidecar in blob transaction")
func ValidateBlobSidecar(tx *types.Transaction, sidecar *types.BlobTxCellSidecar, head *types.Header, opts *ValidationOptions) error {
if sidecar.Custody.OneCount() == 0 {
return errors.New("blobless blob transaction")
}
// Ensure the sidecar is constructed with the correct version, consistent
// with the current fork.
// Ensure the blob fee cap satisfies the minimum blob gas price
if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
}
// Verify whether the blob count is consistent with other parts of the sidecar and the transaction
blobCount := len(sidecar.Cells) / sidecar.Custody.OneCount()
hashes := tx.BlobHashes()
if blobCount == 0 {
return errors.New("blobless blob transaction")
}
if blobCount != len(sidecar.Commitments) || blobCount != len(hashes) {
return fmt.Errorf("invalid number of %d blobs compared to %d commitments and %d blob hashes", blobCount, len(sidecar.Commitments), len(tx.BlobHashes()))
}
// Check whether the blob count does not exceed the max blob count
if blobCount > opts.MaxBlobCount {
return fmt.Errorf("%w: blob count %v, limit %v", ErrTxBlobLimitExceeded, blobCount, opts.MaxBlobCount)
}
if err := sidecar.ValidateBlobCommitmentHashes(hashes); err != nil {
return err
}
// Ensure the sidecar version is correct for the current fork (master: bd77b77ed)
version := types.BlobSidecarVersion0
if opts.Config.IsOsaka(head.Number, head.Time) {
version = types.BlobSidecarVersion1
@ -172,50 +185,42 @@ func validateBlobTx(tx *types.Transaction, head *types.Header, opts *ValidationO
if sidecar.Version != version {
return fmt.Errorf("unexpected sidecar version, want: %d, got: %d", version, sidecar.Version)
}
// Ensure the blob fee cap satisfies the minimum blob gas price
if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
}
// Ensure the number of items in the blob transaction and various side
// data match up before doing any expensive validations
hashes := tx.BlobHashes()
if len(hashes) == 0 {
return errors.New("blobless blob transaction")
}
if len(hashes) > params.BlobTxMaxBlobs {
return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.BlobTxMaxBlobs)
}
if len(sidecar.Blobs) != len(hashes) {
return fmt.Errorf("invalid number of %d blobs compared to %d blob hashes", len(sidecar.Blobs), len(hashes))
}
if err := sidecar.ValidateBlobCommitmentHashes(hashes); err != nil {
return err
}
// Fork-specific sidecar checks, including proof verification.
if sidecar.Version == types.BlobSidecarVersion1 {
return validateBlobSidecarOsaka(sidecar, hashes)
} else {
return validateBlobSidecarLegacy(sidecar, hashes)
}
return validateBlobSidecarLegacy(sidecar, hashes)
}
func validateBlobSidecarLegacy(sidecar *types.BlobTxSidecar, hashes []common.Hash) error {
func validateBlobSidecarLegacy(sidecar *types.BlobTxCellSidecar, hashes []common.Hash) error {
if len(sidecar.Proofs) != len(hashes) {
return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes))
}
for i := range sidecar.Blobs {
if err := kzg4844.VerifyBlobProof(&sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil {
return fmt.Errorf("%w: invalid blob proof: %v", ErrKZGVerificationError, err)
blobs, err := kzg4844.RecoverBlobs(sidecar.Cells, sidecar.Custody.Indices())
if err != nil {
return fmt.Errorf("%w: %v", ErrKZGVerificationError, err)
}
for i := range blobs {
if err := kzg4844.VerifyBlobProof(&blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil {
return fmt.Errorf("%w: invalid blob %d: %v", ErrKZGVerificationError, i, err)
}
}
return nil
}
func validateBlobSidecarOsaka(sidecar *types.BlobTxSidecar, hashes []common.Hash) error {
func validateBlobSidecarOsaka(sidecar *types.BlobTxCellSidecar, hashes []common.Hash) error {
if len(sidecar.Proofs) != len(hashes)*kzg4844.CellProofsPerBlob {
return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes)*kzg4844.CellProofsPerBlob)
}
if err := kzg4844.VerifyCellProofs(sidecar.Blobs, sidecar.Commitments, sidecar.Proofs); err != nil {
indices := sidecar.Custody.Indices()
cellProofs := make([]kzg4844.Proof, 0)
for blobIdx := range len(sidecar.Commitments) {
for _, proofIdx := range indices {
idx := blobIdx*kzg4844.CellProofsPerBlob + int(proofIdx)
cellProofs = append(cellProofs, sidecar.Proofs[idx])
}
}
if err := kzg4844.VerifyCells(sidecar.Cells, sidecar.Commitments, cellProofs, sidecar.Custody.Indices()); err != nil {
return fmt.Errorf("%w: %v", ErrKZGVerificationError, err)
}
return nil

152
core/types/custody_bitmap.go Normal file
View file

@ -0,0 +1,152 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package types
import (
"fmt"
"math/bits"
"math/rand"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
// CustodyBitmap is a bitmap to represent which custody index to store (little endian).
type CustodyBitmap [16]byte
// MarshalText implements encoding.TextMarshaler.
func (b CustodyBitmap) MarshalText() ([]byte, error) {
return []byte(hexutil.Encode(b[:])), nil
}
// UnmarshalText implements encoding.TextUnmarshaler.
func (b *CustodyBitmap) UnmarshalText(input []byte) error {
decoded, err := hexutil.Decode(string(input))
if err != nil {
return fmt.Errorf("custody bitmap: %v", err)
}
if len(decoded) != len(b) {
return fmt.Errorf("custody bitmap: invalid length %d, want %d", len(decoded), len(b))
}
copy(b[:], decoded)
return nil
}
var (
CustodyBitmapAll = func() *CustodyBitmap {
var result CustodyBitmap
for i := range result {
result[i] = 0xFF
}
return &result
}()
CustodyBitmapData = func() *CustodyBitmap {
var result CustodyBitmap
for i := 0; i < kzg4844.DataPerBlob/8; i++ {
result[i] = 0xFF
}
return &result
}()
)
func NewCustodyBitmap(indices []uint64) CustodyBitmap {
var result CustodyBitmap
for _, i := range indices {
if i >= uint64(kzg4844.CellsPerBlob) {
panic("CustodyBitmap: bit index out of range")
}
result[i/8] |= 1 << (i % 8)
}
return result
}
// NewRandomCustodyBitmap creates a CustodyBitmap with n randomly selected indices.
// This should be used only for tests.
func NewRandomCustodyBitmap(n int) CustodyBitmap {
if n <= 0 || n > kzg4844.CellsPerBlob {
panic("CustodyBitmap: invalid number of indices")
}
indices := make([]uint64, 0, n)
used := make(map[uint64]bool)
for len(indices) < n {
idx := uint64(rand.Intn(kzg4844.CellsPerBlob))
if !used[idx] {
used[idx] = true
indices = append(indices, idx)
}
}
return NewCustodyBitmap(indices)
}
// IsSet returns whether bit i is set.
func (b CustodyBitmap) IsSet(i uint64) bool {
if i >= uint64(kzg4844.CellsPerBlob) {
return false
}
return (b[i/8]>>(i%8))&1 == 1
}
// OneCount returns the number of bits set to 1.
func (b CustodyBitmap) OneCount() int {
total := 0
for _, v := range b {
total += bits.OnesCount8(v)
}
return total
}
// Indices returns the bit positions set to 1, in ascending order.
func (b CustodyBitmap) Indices() []uint64 {
out := make([]uint64, 0, b.OneCount())
for byteIdx, val := range b {
v := val
for v != 0 {
tz := bits.TrailingZeros8(v)
out = append(out, uint64(byteIdx*8+tz))
v &^= 1 << tz
}
}
return out
}
// Difference returns b AND NOT set (bits in b but not in set).
func (b CustodyBitmap) Difference(set *CustodyBitmap) *CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] &^ set[i]
}
return &out
}
// Intersection returns b AND set.
func (b CustodyBitmap) Intersection(set *CustodyBitmap) *CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] & set[i]
}
return &out
}
// Union returns b OR set.
func (b CustodyBitmap) Union(set *CustodyBitmap) *CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] | set[i]
}
return &out
}

13
core/types/transaction.go
View file

@ -510,6 +510,19 @@ func (tx *Transaction) WithoutBlobTxSidecar() *Transaction {
return cpy
}
// todo: remove
// WithoutBlob returns a copy of tx with the blob data removed from the sidecar,
// keeping commitments, proofs and other metadata intact.
func (tx *Transaction) WithoutBlob() *Transaction {
blobtx, ok := tx.inner.(*BlobTx)
if !ok || blobtx.Sidecar == nil {
return tx
}
sidecarWithoutBlob := blobtx.Sidecar.Copy()
sidecarWithoutBlob.Blobs = nil
return tx.WithBlobTxSidecar(sidecarWithoutBlob)
}
// WithBlobTxSidecar returns a copy of tx with the blob sidecar added.
func (tx *Transaction) WithBlobTxSidecar(sideCar *BlobTxSidecar) *Transaction {
blobtx, ok := tx.inner.(*BlobTx)

31
core/types/tx_blob.go
View file

@ -176,6 +176,37 @@ func (sc *BlobTxSidecar) Copy() *BlobTxSidecar {
}
}
func (sc *BlobTxSidecar) ToBlobTxCellSidecar() (*BlobTxCellSidecar, error) {
cells, err := kzg4844.ComputeCells(sc.Blobs)
if err != nil {
return nil, err
}
return &BlobTxCellSidecar{
Version: sc.Version,
Cells: cells,
Commitments: sc.Commitments,
Proofs: sc.Proofs,
Custody: *CustodyBitmapAll,
}, nil
}
type BlobTxCellSidecar struct {
Version byte
Cells []kzg4844.Cell
Commitments []kzg4844.Commitment
Proofs []kzg4844.Proof
Custody CustodyBitmap
}
// ValidateBlobCommitmentHashes checks whether the given hashes correspond to the
// commitments in the sidecar
func (c *BlobTxCellSidecar) ValidateBlobCommitmentHashes(hashes []common.Hash) error {
sc := BlobTxSidecar{
Commitments: c.Commitments,
}
return sc.ValidateBlobCommitmentHashes(hashes)
}
// blobTxWithBlobs represents blob tx with its corresponding sidecar.
// This is an interface because sidecars are versioned.
type blobTxWithBlobs interface {

4
eth/backend.go
View file

@ -44,6 +44,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/fetcher"
"github.com/ethereum/go-ethereum/eth/gasprice"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/eth/protocols/snap"
@ -338,10 +339,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
Database: chainDb,
Chain: eth.blockchain,
TxPool: eth.txPool,
BlobPool: eth.blobTxPool,
Network: networkID,
Sync: config.SyncMode,
BloomCache: uint64(cacheLimit),
RequiredBlocks: config.RequiredBlocks,
Custody: *types.CustodyBitmapAll,
}); err != nil {
return nil, err
}
@ -425,6 +428,7 @@ func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager
func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain }
func (s *Ethereum) TxPool() *txpool.TxPool { return s.txPool }
func (s *Ethereum) BlobTxPool() *blobpool.BlobPool { return s.blobTxPool }
func (s *Ethereum) BlobFetcher() *fetcher.BlobFetcher { return s.handler.blobFetcher }
func (s *Ethereum) Engine() consensus.Engine { return s.engine }
func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb }
func (s *Ethereum) IsListening() bool { return true } // Always listening

60
eth/catalyst/api.go
View file

@ -217,7 +217,7 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV3(ctx context.Context, update engine.
// ForkchoiceUpdatedV4 is equivalent to V3 with the addition of slot number
// in the payload attributes. It supports only PayloadAttributesV4.
func (api *ConsensusAPI) ForkchoiceUpdatedV4(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) {
func (api *ConsensusAPI) ForkchoiceUpdatedV4(ctx context.Context, update engine.ForkchoiceStateV1, params *engine.PayloadAttributes, custodyColumns *types.CustodyBitmap) (engine.ForkChoiceResponse, error) {
if params != nil {
switch {
case params.Withdrawals == nil:
@ -230,6 +230,9 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV4(ctx context.Context, update engine.
return engine.STATUS_INVALID, unsupportedForkErr("fcuV4 must only be called for amsterdam payloads")
}
}
if custodyColumns != nil {
api.eth.BlobFetcher().UpdateCustody(*custodyColumns)
}
// TODO(matt): the spec requires that fcu is applied when called on a valid
// hash, even if params are wrong. To do this we need to split up
// forkchoiceUpdate into a function that only updates the head and then a
@ -679,6 +682,61 @@ func (api *ConsensusAPI) getBlobs(hashes []common.Hash, v2 bool) ([]*engine.Blob
return res, nil
}
// GetBlobsV4 returns cell-level blob data from the transaction pool.
// V4 returns only the requested cells as specified by the indices_bitarray.
func (api *ConsensusAPI) GetBlobsV4(hashes []common.Hash, indicesBitarray types.CustodyBitmap) ([]*engine.BlobCellsAndProofsV1, error) {
head := api.eth.BlockChain().CurrentHeader()
// Sparse blobpool is not necessarily coupled with the Amsterdam fork and
// can technically be supported after the Osaka fork
// (where cell proofs are introduced).
if api.config().LatestFork(head.Time) < forks.Osaka {
return nil, nil
}
if len(hashes) > 128 {
return nil, engine.TooLargeRequest.With(fmt.Errorf("requested blob count too large: %v", len(hashes)))
}
cells, proofs, err := api.eth.BlobTxPool().GetBlobCells(hashes, indicesBitarray)
if err != nil {
return nil, engine.InvalidParams.With(err)
}
var (
res = make([]*engine.BlobCellsAndProofsV1, len(hashes))
hitCount int
)
getBlobsRequestedCounter.Inc(int64(len(hashes)))
for i := range hashes {
if cells[i] == nil || proofs[i] == nil {
continue
}
hitCount++
blobCells := make([]hexutil.Bytes, len(cells[i]))
for j, cell := range cells[i] {
if cell != nil {
blobCells[j] = cell[:]
}
}
blobProofs := make([]hexutil.Bytes, len(proofs[i]))
for j, proof := range proofs[i] {
if proof != nil {
blobProofs[j] = proof[:]
}
}
res[i] = &engine.BlobCellsAndProofsV1{
BlobCells: blobCells,
Proofs: blobProofs,
}
}
getBlobsAvailableCounter.Inc(int64(hitCount))
if hitCount == len(hashes) {
getBlobsRequestCompleteHit.Inc(1)
} else if hitCount > 0 {
getBlobsRequestPartialHit.Inc(1)
} else {
getBlobsRequestMiss.Inc(1)
}
return res, nil
}
// Helper for NewPayload* methods.
var invalidStatus = engine.PayloadStatusV1{Status: engine.INVALID}

16
eth/catalyst/api_test.go
View file

@ -1911,7 +1911,12 @@ func newGetBlobEnv(t testing.TB, version byte) (*node.Node, *ConsensusAPI) {
tx1 := makeMultiBlobTx(&config, 0, 2, 0, key1, version) // blob[0, 2)
tx2 := makeMultiBlobTx(&config, 0, 2, 2, key2, version) // blob[2, 4)
tx3 := makeMultiBlobTx(&config, 0, 2, 4, key3, version) // blob[4, 6)
ethServ.TxPool().Add([]*types.Transaction{tx1, tx2, tx3}, true)
errs := ethServ.TxPool().Add([]*types.Transaction{tx1, tx2, tx3}, true)
for i, err := range errs {
if err != nil {
t.Logf("Add tx %d failed: %v", i, err)
}
}
api := newConsensusAPIWithoutHeartbeat(ethServ)
return n, api
@ -2108,6 +2113,15 @@ func runGetBlobs(t testing.TB, getBlobs getBlobsFn, start, limit int, fillRandom
}
}
if !reflect.DeepEqual(result, expect) {
t.Logf("result len=%d, expect len=%d", len(result), len(expect))
if len(result) > 0 && result[0] != nil && len(expect) > 0 && expect[0] != nil {
t.Logf("result[0].Blob len=%d, expect[0].Blob len=%d", len(result[0].Blob), len(expect[0].Blob))
t.Logf("result[0].CellProofs len=%d, expect[0].CellProofs len=%d", len(result[0].CellProofs), len(expect[0].CellProofs))
t.Logf("result[0].Blob == expect[0].Blob: %v", reflect.DeepEqual(result[0].Blob, expect[0].Blob))
t.Logf("result[0].CellProofs == expect[0].CellProofs: %v", reflect.DeepEqual(result[0].CellProofs, expect[0].CellProofs))
} else {
t.Logf("result[0]=%v, expect[0]=%v", result, expect)
}
t.Fatalf("Unexpected result for case %s", name)
}
}

828
eth/fetcher/blob_fetcher.go Normal file
View file

@ -0,0 +1,828 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package fetcher
import (
"math/rand"
"slices"
"sort"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
// todo remove partial / full
type random interface {
Intn(n int) int
}
// BlobFetcher fetches blobs of new type-3 transactions with probability p,
// and for the remaining (1-p) transactions, it performs availability checks.
// For availability checks, it fetches cells from each blob in the transaction
// according to the custody cell indices provided by the consensus client
// connected to this execution client.
// todo
var blobFetchTimeout = 5 * time.Second
var blobAvailabilityTimeout = 2 * time.Second
const (
availabilityThreshold = 2
maxPayloadRetrievals = 128
maxPayloadAnnounces = 4096
fetchProbability = 15
MAX_CELLS_PER_PARTIAL_REQUEST = 8
)
type blobTxAnnounce struct {
origin string // Identifier of the peer that sent the announcement
txs []common.Hash // Hashes of transactions announced
cells types.CustodyBitmap // Custody information of transactions being announced
}
type cellRequest struct {
txs []common.Hash // Transactions that have been requested for their cells
cells *types.CustodyBitmap // Requested cell indices
time mclock.AbsTime // Timestamp when the request was made
}
type payloadDelivery struct {
origin string // Peer from which the payloads were delivered
txs []common.Hash // Hashes of transactions that were delivered
cells [][]kzg4844.Cell
cellBitmap *types.CustodyBitmap
}
type cellWithSeq struct {
seq uint64
cells *types.CustodyBitmap
}
// PeerCellDelivery holds cells delivered by a single peer.
type PeerCellDelivery struct {
Cells []kzg4844.Cell // blob-major order as received
Indices []uint64 // custody indices provided by this peer
}
type fetchStatus struct {
fetching *types.CustodyBitmap // To avoid fetching cells which had already been fetched / currently being fetched
fetched []uint64 // Custody indices that have been fetched (per-blob, same for all blobs)
deliveries map[string]*PeerCellDelivery // Per-peer cell deliveries
blobCount int // Number of blobs in this tx (set on first delivery)
}
type BlobFetcherFunctions struct {
HasPayload func(common.Hash) bool
AddCells func(common.Hash, map[string]*PeerCellDelivery, *types.CustodyBitmap) error
FetchPayloads func(string, []common.Hash, *types.CustodyBitmap) error
DropPeer func(string)
}
// BlobFetcher is responsible for managing type 3 transactions based on peer announcements.
//
// BlobFetcher manages three buffers:
// - Transactions not to be fetched are moved to "waitlist"
// if a payload(blob) seems to be possessed by D(threshold) other peers, request custody cells for that.
// Accept it when the cells are received. Otherwise, it is dropped.
// - Transactions queued to be fetched are moved to "announces"
// if a payload is received, it is added to the blob pool. Otherwise, the transaction is dropped.
// - Transactions to be fetched are moved to "fetching"
// if a payload/cell announcement is received during fetch, the peer is recorded as an alternate source.
type BlobFetcher struct {
notify chan *blobTxAnnounce
cleanup chan *payloadDelivery
drop chan *txDrop
quit chan struct{}
custody *types.CustodyBitmap
txSeq uint64 // To make transactions fetched in arrival order
full map[common.Hash]struct{}
partial map[common.Hash]struct{}
// Buffer 1: Set of blob txs whose blob data is waiting for availability confirmation (partial fetch)
waitlist map[common.Hash]map[string]struct{} // Peer set that announced blob availability
waittime map[common.Hash]mclock.AbsTime // Timestamp when added to waitlist
waitslots map[string]map[common.Hash]struct{} // Waiting announcements grouped by peer (DoS protection)
// waitSlots should also include announcements with partial cells
// Buffer 2: Transactions queued for fetching (full fetch + partial fetch)
// "announces" is shared with stage 3, for DoS protection
announces map[string]map[common.Hash]*cellWithSeq // Set of announced transactions, grouped by origin peer
// Buffer 2
// Stage 3: Transactions whose payloads/cells are currently being fetched (full fetch + partial fetch)
fetches map[common.Hash]*fetchStatus // Hash -> Bitmap, in-flight transaction cells
requests map[string][]*cellRequest // In-flight transaction retrievals
// todo simplify
alternates map[common.Hash]map[string]*types.CustodyBitmap // In-flight transaction alternate origins (in case the peer is dropped)
fn BlobFetcherFunctions // callbacks
step chan struct{} // Notification channel when the fetcher loop iterates
clock mclock.Clock // Monotonic clock or simulated clock for tests
realTime func() time.Time // Real system time or simulated time for tests
rand random // Randomizer
}
func NewBlobFetcher(fn BlobFetcherFunctions, custody *types.CustodyBitmap, rand random) *BlobFetcher {
return &BlobFetcher{
notify: make(chan *blobTxAnnounce),
cleanup: make(chan *payloadDelivery),
drop: make(chan *txDrop),
quit: make(chan struct{}),
full: make(map[common.Hash]struct{}),
partial: make(map[common.Hash]struct{}),
waitlist: make(map[common.Hash]map[string]struct{}),
waittime: make(map[common.Hash]mclock.AbsTime),
waitslots: make(map[string]map[common.Hash]struct{}),
announces: make(map[string]map[common.Hash]*cellWithSeq),
fetches: make(map[common.Hash]*fetchStatus),
requests: make(map[string][]*cellRequest),
alternates: make(map[common.Hash]map[string]*types.CustodyBitmap),
fn: fn,
custody: custody,
clock: mclock.System{},
realTime: time.Now,
rand: rand,
}
}
// Notify is called when a Type 3 transaction is observed on the network. (TransactionPacket / NewPooledTransactionHashesPacket)
func (f *BlobFetcher) Notify(peer string, txs []common.Hash, cells types.CustodyBitmap) error {
blobAnnounceInMeter.Mark(int64(len(txs)))
anns := make([]common.Hash, 0)
for _, tx := range txs {
if f.fn.HasPayload(tx) {
continue
}
anns = append(anns, tx)
}
blobAnnounce := &blobTxAnnounce{origin: peer, txs: anns, cells: cells}
select {
case f.notify <- blobAnnounce:
return nil
case <-f.quit:
return errTerminated
}
}
// Enqueue inserts a batch of received blob payloads into the blob pool.
// This is triggered by ethHandler upon receiving direct request responses.
func (f *BlobFetcher) Enqueue(peer string, hashes []common.Hash, cells [][]kzg4844.Cell, cellBitmap types.CustodyBitmap) error {
blobReplyInMeter.Mark(int64(len(hashes)))
select {
case f.cleanup <- &payloadDelivery{origin: peer, txs: hashes, cells: cells, cellBitmap: &cellBitmap}:
case <-f.quit:
return errTerminated
}
return nil
}
func (f *BlobFetcher) Drop(peer string) error {
select {
case f.drop <- &txDrop{peer: peer}:
return nil
case <-f.quit:
return errTerminated
}
}
func (f *BlobFetcher) UpdateCustody(cells types.CustodyBitmap) {
// todo use lock or process inside of loop
f.custody = &cells
}
func (f *BlobFetcher) Start() {
go f.loop()
}
func (f *BlobFetcher) Stop() {
close(f.quit)
}
func (f *BlobFetcher) loop() {
var (
waitTimer = new(mclock.Timer) // Timer for waitlist (availability)
waitTrigger = make(chan struct{}, 1)
timeoutTimer = new(mclock.Timer) // Timer for payload fetch request
timeoutTrigger = make(chan struct{}, 1)
)
for {
select {
case ann := <-f.notify:
// Drop part of the announcements if too many have accumulated from that peer
// This prevents a peer from dominating the queue with txs without responding to the request
used := len(f.waitslots[ann.origin]) + len(f.announces[ann.origin])
if used >= maxPayloadAnnounces {
blobAnnounceDOSMeter.Mark(int64(len(ann.txs)))
break
}
want := used + len(ann.txs)
if want >= maxPayloadAnnounces {
blobAnnounceDOSMeter.Mark(int64(want - maxPayloadAnnounces))
ann.txs = ann.txs[:maxPayloadAnnounces-used]
}
var (
idleWait = len(f.waittime) == 0
_, oldPeer = f.announces[ann.origin]
nextSeq = func() uint64 {
seq := f.txSeq
f.txSeq++
return seq
}
reschedule = make(map[string]struct{})
)
for _, hash := range ann.txs {
if oldPeer && f.announces[ann.origin][hash] != nil {
// Ignore already announced information
// We also have to prevent reannouncement by changing cells field.
// Considering cell custody transition is notified in advance of its finalization by consensus client,
// there is no reason to reannounce cells, and it has to be prevented.
continue
}
// Decide full or partial request
if _, ok := f.full[hash]; !ok {
if _, ok := f.partial[hash]; !ok {
// Not decided yet
var randomValue int
if f.rand == nil {
randomValue = rand.Intn(100)
} else {
randomValue = f.rand.Intn(100)
}
// For eager mode, always fetch immediately
if randomValue < fetchProbability || f.custody.OneCount() >= kzg4844.DataPerBlob {
f.full[hash] = struct{}{}
} else {
f.partial[hash] = struct{}{}
// Register for availability check
f.waitlist[hash] = make(map[string]struct{})
f.waittime[hash] = f.clock.Now()
}
}
}
if _, ok := f.full[hash]; ok {
// 1) Decided to send full request of the tx
if ann.cells != *types.CustodyBitmapAll {
continue
}
if f.announces[ann.origin] == nil {
f.announces[ann.origin] = make(map[common.Hash]*cellWithSeq)
}
f.announces[ann.origin][hash] = &cellWithSeq{
cells: types.CustodyBitmapData,
seq: nextSeq(),
}
reschedule[ann.origin] = struct{}{}
continue
}
if _, ok := f.partial[hash]; ok {
// 2) Decided to send partial request of the tx
if f.waitlist[hash] != nil {
if ann.cells != *types.CustodyBitmapAll {
// Availability check is only meaningful with full availability announcements
continue
}
// Transaction is at the stage of availability check
// Add the peer to the peer list with full availability (waitlist)
f.waitlist[hash][ann.origin] = struct{}{}
if waitslots := f.waitslots[ann.origin]; waitslots != nil {
waitslots[hash] = struct{}{}
} else {
f.waitslots[ann.origin] = map[common.Hash]struct{}{
hash: {},
}
}
if len(f.waitlist[hash]) >= availabilityThreshold {
// Passed availability check, move to fetching stage
blobFetcherWaitTime.Update(int64(time.Duration(f.clock.Now() - f.waittime[hash])))
for peer := range f.waitlist[hash] {
if f.announces[peer] == nil {
f.announces[peer] = make(map[common.Hash]*cellWithSeq)
}
f.announces[peer][hash] = &cellWithSeq{
cells: f.custody,
seq: nextSeq(),
}
delete(f.waitslots[peer], hash)
if len(f.waitslots[peer]) == 0 {
delete(f.waitslots, peer)
}
reschedule[peer] = struct{}{}
}
delete(f.waitlist, hash)
delete(f.waittime, hash)
}
continue
}
if ann.cells.Intersection(f.custody).OneCount() == 0 {
// If there's no custody overlapping in ann, it can be ignored
continue
}
// Add this peer as a possible fetch source
// todo: Did we remove fetch from partial
if f.announces[ann.origin] == nil {
f.announces[ann.origin] = make(map[common.Hash]*cellWithSeq)
}
f.announces[ann.origin][hash] = &cellWithSeq{
cells: ann.cells.Intersection(f.custody),
seq: nextSeq(),
}
reschedule[ann.origin] = struct{}{}
}
}
// If a new item was added to the waitlist, schedule its timeout
if idleWait && len(f.waittime) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
// If this is a new peer and that peer sent transaction with payload flag,
// schedule transaction fetches from it
if !oldPeer && len(f.announces[ann.origin]) > 0 {
f.scheduleFetches(timeoutTimer, timeoutTrigger, reschedule)
}
case <-waitTrigger:
// At least one transaction's waiting time ran out. Instead of dropping,
// convert timed-out partial fetches to full fetches so we don't lose
// the transaction. All peers in the waitlist announced full custody
// (that was the entry condition), so they can serve as full fetch sources.
reschedule := make(map[string]struct{})
for hash, instance := range f.waittime {
if time.Duration(f.clock.Now()-instance)+txGatherSlack > blobAvailabilityTimeout {
// partial -> full conversion
delete(f.partial, hash)
f.full[hash] = struct{}{}
blobAnnounceTimeoutMeter.Mark(1)
for peer := range f.waitlist[hash] {
if f.announces[peer] == nil {
f.announces[peer] = make(map[common.Hash]*cellWithSeq)
}
f.announces[peer][hash] = &cellWithSeq{
cells: types.CustodyBitmapData,
seq: f.txSeq,
}
f.txSeq++
delete(f.waitslots[peer], hash)
if len(f.waitslots[peer]) == 0 {
delete(f.waitslots, peer)
}
reschedule[peer] = struct{}{}
}
delete(f.waittime, hash)
delete(f.waitlist, hash)
}
}
if len(reschedule) > 0 {
f.scheduleFetches(timeoutTimer, timeoutTrigger, reschedule)
}
// If transactions are still waiting for availability, reschedule the wait timer
if len(f.waittime) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
case <-timeoutTrigger:
// Clean up any expired retrievals and avoid re-requesting them from the
// same peer (either overloaded or malicious, useless in both cases).
// Update blobpool according to availability result.
for peer, requests := range f.requests {
newRequests := make([]*cellRequest, 0)
for _, req := range requests {
if time.Duration(f.clock.Now()-req.time)+txGatherSlack > blobFetchTimeout {
blobRequestTimeoutMeter.Mark(int64(len(req.txs)))
for _, hash := range req.txs {
// Do not request the same tx from this peer
delete(f.announces[peer], hash)
delete(f.alternates[hash], peer)
// Allow other candidates to be requested these cells
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(req.cells)
// Drop cells if there is no alternate source to fetch cells from
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
delete(f.fetches, hash)
}
}
if len(f.announces[peer]) == 0 {
delete(f.announces, peer)
}
} else {
newRequests = append(newRequests, req)
}
}
f.requests[peer] = newRequests
if len(f.requests[peer]) == 0 {
delete(f.requests, peer)
}
}
// Schedule a new transaction retrieval
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
// Trigger timeout for new schedule
f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
case delivery := <-f.cleanup:
// Remove from announce
var requestId int
var request *cellRequest
for _, hash := range delivery.txs {
// Find the request
for i, req := range f.requests[delivery.origin] {
if slices.Contains(req.txs, hash) && *req.cells == *delivery.cellBitmap {
request = req
requestId = i
break
}
}
if request != nil {
break
}
}
if request == nil {
// peer sent cells not requested. ignore
break
}
for i, hash := range delivery.txs {
if !slices.Contains(request.txs, hash) {
// Unexpected hash, ignore
continue
}
indices := delivery.cellBitmap.Indices()
cellsPerBlob := len(indices)
if cellsPerBlob > 0 {
status := f.fetches[hash]
blobCount := len(delivery.cells[i]) / cellsPerBlob
if status.blobCount == 0 {
status.blobCount = blobCount
status.deliveries = make(map[string]*PeerCellDelivery)
}
status.deliveries[delivery.origin] = &PeerCellDelivery{
Cells: delivery.cells[i],
Indices: indices,
}
status.fetched = append(status.fetched, indices...)
}
// Update announces of this peer
delete(f.announces[delivery.origin], hash)
if len(f.announces[delivery.origin]) == 0 {
delete(f.announces, delivery.origin)
}
delete(f.alternates[hash], delivery.origin)
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
}
// Check whether the all required cells are fetched
completed := false
if _, ok := f.full[hash]; ok && len(f.fetches[hash].fetched) >= kzg4844.DataPerBlob {
completed = true
} else if _, ok := f.partial[hash]; ok {
fetched := make([]uint64, len(f.fetches[hash].fetched))
copy(fetched, f.fetches[hash].fetched)
slices.Sort(fetched)
custodyIndices := f.custody.Indices()
completed = slices.Equal(fetched, custodyIndices)
}
if completed {
blobFetcherFetchTime.Update(int64(time.Duration(f.clock.Now() - request.time)))
status := f.fetches[hash]
collectedCustody := types.NewCustodyBitmap(status.fetched)
f.fn.AddCells(hash, status.deliveries, &collectedCustody)
for peer, txset := range f.announces {
delete(txset, hash)
if len(txset) == 0 {
delete(f.announces, peer)
}
}
delete(f.alternates, hash)
delete(f.fetches, hash)
}
}
blobRequestDoneMeter.Mark(int64(len(delivery.txs)))
// Remove the request
f.requests[delivery.origin][requestId] = f.requests[delivery.origin][len(f.requests[delivery.origin])-1]
f.requests[delivery.origin] = f.requests[delivery.origin][:len(f.requests[delivery.origin])-1]
if len(f.requests[delivery.origin]) == 0 {
delete(f.requests, delivery.origin)
}
// Reschedule missing transactions in the request
// Anything not delivered should be re-scheduled (with or without
// this peer, depending on the response cutoff)
delivered := make(map[common.Hash]struct{})
for _, hash := range delivery.txs {
delivered[hash] = struct{}{}
}
cutoff := len(request.txs)
for i, hash := range request.txs {
if _, ok := delivered[hash]; ok {
cutoff = i
continue
}
}
// Reschedule missing hashes from alternates, not-fulfilled from alt+self
for i, hash := range request.txs {
if _, ok := delivered[hash]; !ok {
// Not delivered
if i < cutoff {
// Remove origin from candidate sources for partial responses
delete(f.alternates[hash], delivery.origin)
delete(f.announces[delivery.origin], hash)
if len(f.announces[delivery.origin]) == 0 {
delete(f.announces, delivery.origin)
}
}
// Mark cells deliverable by other peers
if f.fetches[hash] != nil {
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(delivery.cellBitmap)
}
}
}
// Something was delivered, try to reschedule requests
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too
case drop := <-f.drop:
// A peer was dropped, remove all traces of it
if _, ok := f.waitslots[drop.peer]; ok {
for hash := range f.waitslots[drop.peer] {
delete(f.waitlist[hash], drop.peer)
if len(f.waitlist[hash]) == 0 {
delete(f.waitlist, hash)
delete(f.waittime, hash)
}
}
delete(f.waitslots, drop.peer)
if len(f.waitlist) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
}
// Clean up general announcement tracking
if _, ok := f.announces[drop.peer]; ok {
for hash := range f.announces[drop.peer] {
delete(f.alternates[hash], drop.peer)
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
}
}
delete(f.announces, drop.peer)
}
delete(f.announces, drop.peer)
// Clean up any active requests
if request, ok := f.requests[drop.peer]; ok && len(request) != 0 {
for _, req := range request {
for _, hash := range req.txs {
// Undelivered hash, reschedule if there's an alternative origin available
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(req.cells)
delete(f.alternates[hash], drop.peer)
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
delete(f.fetches, hash)
}
}
}
delete(f.requests, drop.peer)
// If a request was cancelled, check if anything needs to be rescheduled
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
}
case <-f.quit:
return
}
// Update metrics gauges
blobFetcherWaitingPeers.Update(int64(len(f.waitslots)))
blobFetcherWaitingHashes.Update(int64(len(f.waitlist)))
blobFetcherQueueingPeers.Update(int64(len(f.announces) - len(f.requests)))
blobFetcherQueueingHashes.Update(int64(len(f.announces)))
blobFetcherFetchingPeers.Update(int64(len(f.requests)))
blobFetcherFetchingHashes.Update(int64(len(f.fetches)))
// Loop did something, ping the step notifier if needed (tests)
if f.step != nil {
f.step <- struct{}{}
}
}
}
func (f *BlobFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) {
if *timer != nil {
(*timer).Stop()
}
now := f.clock.Now()
earliest := now
for _, instance := range f.waittime {
if earliest > instance {
earliest = instance
if txArriveTimeout-time.Duration(now-earliest) < txGatherSlack {
break
}
}
}
*timer = f.clock.AfterFunc(txArriveTimeout-time.Duration(now-earliest), func() {
trigger <- struct{}{}
})
}
// Exactly same as the one in TxFetcher
func (f *BlobFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) {
if *timer != nil {
(*timer).Stop()
}
now := f.clock.Now()
earliest := now
for _, requests := range f.requests {
for _, req := range requests {
// If this request already timed out, skip it altogether
if req.txs == nil {
continue
}
if earliest > req.time {
earliest = req.time
if blobFetchTimeout-time.Duration(now-earliest) < txGatherSlack {
break
}
}
}
}
*timer = f.clock.AfterFunc(blobFetchTimeout-time.Duration(now-earliest), func() {
trigger <- struct{}{}
})
}
func (f *BlobFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, whitelist map[string]struct{}) {
// Gather the set of peers we want to retrieve from (default to all)
actives := whitelist
if actives == nil {
actives = make(map[string]struct{})
for peer := range f.announces {
actives[peer] = struct{}{}
}
}
if len(actives) == 0 {
return
}
// For each active peer, try to schedule some payload fetches
idle := len(f.requests) == 0
f.forEachPeer(actives, func(peer string) {
if len(f.announces[peer]) == 0 || len(f.requests[peer]) != 0 {
return // continue
}
var (
hashes = make([]common.Hash, 0, maxTxRetrievals)
custodies = make([]*types.CustodyBitmap, 0, maxTxRetrievals)
)
f.forEachAnnounce(f.announces[peer], func(hash common.Hash, cells *types.CustodyBitmap) bool {
var unfetched *types.CustodyBitmap
if f.fetches[hash] == nil {
// tx is not being fetched
unfetched = cells
} else {
unfetched = cells.Difference(f.fetches[hash].fetching)
}
// Mark fetching for unfetched cells
if unfetched.OneCount() > 0 {
if f.fetches[hash] == nil {
f.fetches[hash] = &fetchStatus{
fetching: unfetched,
fetched: make([]uint64, 0),
}
} else {
f.fetches[hash].fetching = f.fetches[hash].fetching.Union(unfetched)
}
// Accumulate the hash and stop if the limit was reached
hashes = append(hashes, hash)
custodies = append(custodies, unfetched)
}
// Mark alternatives
if f.alternates[hash] == nil {
f.alternates[hash] = map[string]*types.CustodyBitmap{
peer: cells,
}
} else {
f.alternates[hash][peer] = cells
}
return len(hashes) < maxPayloadRetrievals
})
// If any hashes were allocated, request them from the peer
if len(hashes) > 0 {
// Group hashes by custody bitmap
requestByCustody := make(map[types.CustodyBitmap]*cellRequest)
for i, hash := range hashes {
key := *custodies[i]
if _, ok := requestByCustody[key]; !ok {
requestByCustody[key] = &cellRequest{
txs: []common.Hash{},
cells: custodies[i],
time: f.clock.Now(),
}
}
requestByCustody[key].txs = append(requestByCustody[key].txs, hash)
}
// construct request
var request []*cellRequest
for _, cr := range requestByCustody {
request = append(request, cr)
}
f.requests[peer] = request
go func(peer string, request []*cellRequest) {
for _, req := range request {
blobRequestOutMeter.Mark(int64(len(req.txs)))
if err := f.fn.FetchPayloads(peer, req.txs, req.cells); err != nil {
blobRequestFailMeter.Mark(int64(len(req.txs)))
f.Drop(peer)
break
}
}
}(peer, request)
}
})
// If a new request was fired, schedule a timeout timer
if idle && len(f.requests) > 0 {
f.rescheduleTimeout(timer, timeout)
}
}
// forEachAnnounce loops over the given announcements in arrival order, invoking
// the do function for each until it returns false. We enforce an arrival
// ordering to minimize the chances of transaction nonce-gaps, which result in
// transactions being rejected by the txpool.
func (f *BlobFetcher) forEachAnnounce(announces map[common.Hash]*cellWithSeq, do func(hash common.Hash, cells *types.CustodyBitmap) bool) {
type announcement struct {
hash common.Hash
cells *types.CustodyBitmap
seq uint64
}
// Process announcements by their arrival order
list := make([]announcement, 0, len(announces))
for hash, entry := range announces {
list = append(list, announcement{hash: hash, cells: entry.cells, seq: entry.seq})
}
sort.Slice(list, func(i, j int) bool {
return list[i].seq < list[j].seq
})
for i := range list {
if !do(list[i].hash, list[i].cells) {
return
}
}
}
// forEachPeer does a range loop over a map of peers in production, but during
// testing it does a deterministic sorted random to allow reproducing issues.
func (f *BlobFetcher) forEachPeer(peers map[string]struct{}, do func(peer string)) {
// If we're running production(step == nil), use whatever Go's map gives us
if f.step == nil {
for peer := range peers {
do(peer)
}
return
}
// We're running the test suite, make iteration deterministic (sorted by peer id)
list := make([]string, 0, len(peers))
for peer := range peers {
list = append(list, peer)
}
sort.Strings(list)
for _, peer := range list {
do(peer)
}
}

1101
eth/fetcher/blob_fetcher_test.go Normal file
View file

File diff suppressed because it is too large Load diff

22
eth/fetcher/metrics.go
View file

@ -57,4 +57,26 @@ var (
// to become "unfrozen", either by eventually replying to the request
// or by being dropped, measuring from the moment the request was sent.
txFetcherSlowWait = metrics.NewRegisteredHistogram("eth/fetcher/transaction/slow/wait", nil, metrics.NewExpDecaySample(1028, 0.015))
blobAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/in", nil)
blobAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/dos", nil)
// This metric tracks partial→full conversions due to availability timeout
blobAnnounceTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/timeout", nil)
blobRequestOutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/out", nil)
blobRequestFailMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/fail", nil)
blobRequestDoneMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/done", nil)
blobRequestTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/timeout", nil)
blobReplyInMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/replies/in", nil)
blobFetcherWaitingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/waiting/peers", nil)
blobFetcherWaitingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/waiting/hashes", nil)
blobFetcherQueueingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/queueing/peers", nil)
blobFetcherQueueingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/queueing/hashes", nil)
blobFetcherFetchingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/fetching/peers", nil)
blobFetcherFetchingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/fetching/hashes", nil)
blobFetcherWaitTime = metrics.NewRegisteredHistogram("eth/fetcher/blob/wait/time", nil, metrics.NewExpDecaySample(1028, 0.015))
blobFetcherFetchTime = metrics.NewRegisteredHistogram("eth/fetcher/blob/fetch/time", nil, metrics.NewExpDecaySample(1028, 0.015))
)

34
eth/fetcher/tx_fetcher.go
View file

@ -180,10 +180,10 @@ type TxFetcher struct {
alternates map[common.Hash]map[string]struct{} // In-flight transaction alternate origins if retrieval fails
// Callbacks
validateMeta func(common.Hash, byte) error // Validate a tx metadata based on the local txpool
addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool
fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer
dropPeer func(string) // Drops a peer in case of announcement violation
validateMeta func(common.Hash, byte) error // Validate a tx metadata based on the local txpool
addTxs func(string, []*types.Transaction) []error // Insert a batch of transactions into local txpool
fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer
dropPeer func(string) // Drops a peer in case of announcement violation
step chan struct{} // Notification channel when the fetcher loop iterates
clock mclock.Clock // Monotonic clock or simulated clock for tests
@ -194,7 +194,7 @@ type TxFetcher struct {
// NewTxFetcher creates a transaction fetcher to retrieve transaction
// based on hash announcements.
// Chain can be nil to disable on-chain checks.
func NewTxFetcher(chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string)) *TxFetcher {
func NewTxFetcher(chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func(string, []*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string)) *TxFetcher {
return NewTxFetcherForTests(chain, validateMeta, addTxs, fetchTxs, dropPeer, mclock.System{}, time.Now, nil)
}
@ -202,7 +202,7 @@ func NewTxFetcher(chain *core.BlockChain, validateMeta func(common.Hash, byte) e
// a simulated version and the internal randomness with a deterministic one.
// Chain can be nil to disable on-chain checks.
func NewTxFetcherForTests(
chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string),
chain *core.BlockChain, validateMeta func(common.Hash, byte) error, addTxs func(string, []*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string),
clock mclock.Clock, realTime func() time.Time, rand *mrand.Rand) *TxFetcher {
return &TxFetcher{
notify: make(chan *txAnnounce),
@ -232,7 +232,7 @@ func NewTxFetcherForTests(
// Notify announces the fetcher of the potential availability of a new batch of
// transactions in the network.
func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []common.Hash) error {
func (f *TxFetcher) Notify(peer string, kinds []byte, sizes []uint32, hashes []common.Hash) ([]common.Hash, error) {
// Keep track of all the announced transactions
txAnnounceInMeter.Mark(int64(len(hashes)))
@ -245,13 +245,18 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
unknownHashes = make([]common.Hash, 0, len(hashes))
unknownMetas = make([]txMetadata, 0, len(hashes))
blobFetchHashes = make([]common.Hash, 0, len(hashes))
duplicate int64
onchain int64
underpriced int64
)
for i, hash := range hashes {
err := f.validateMeta(hash, types[i])
err := f.validateMeta(hash, kinds[i])
if errors.Is(err, txpool.ErrAlreadyKnown) {
if kinds[i] == types.BlobTxType {
blobFetchHashes = append(blobFetchHashes, hash)
}
duplicate++
continue
}
@ -271,11 +276,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
}
unknownHashes = append(unknownHashes, hash)
if kinds[i] == types.BlobTxType {
blobFetchHashes = append(blobFetchHashes, hash)
}
// Transaction metadata has been available since eth68, and all
// legacy eth protocols (prior to eth68) have been deprecated.
// Therefore, metadata is always expected in the announcement.
unknownMetas = append(unknownMetas, txMetadata{kind: types[i], size: sizes[i]})
unknownMetas = append(unknownMetas, txMetadata{kind: kinds[i], size: sizes[i]})
}
txAnnounceKnownMeter.Mark(duplicate)
txAnnounceUnderpricedMeter.Mark(underpriced)
@ -283,14 +291,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
// If anything's left to announce, push it into the internal loop
if len(unknownHashes) == 0 {
return nil
return blobFetchHashes, nil
}
announce := &txAnnounce{origin: peer, hashes: unknownHashes, metas: unknownMetas}
select {
case f.notify <- announce:
return nil
return blobFetchHashes, nil
case <-f.quit:
return errTerminated
return nil, errTerminated
}
}
@ -344,7 +352,7 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool)
)
batch := txs[i:end]
for j, err := range f.addTxs(batch) {
for j, err := range f.addTxs(peer, batch) {
// Track the transaction hash if the price is too low for us.
// Avoid re-request this transaction when we receive another
// announcement.

12
eth/fetcher/tx_fetcher_test.go
View file

@ -93,7 +93,7 @@ func newTestTxFetcher() *TxFetcher {
return NewTxFetcher(
nil,
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
func(_ string, txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
@ -1172,7 +1172,7 @@ func TestTransactionFetcherUnderpricedDedup(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
f := newTestTxFetcher()
f.addTxs = func(txs []*types.Transaction) []error {
f.addTxs = func(_ string, txs []*types.Transaction) []error {
errs := make([]error, len(txs))
for i := 0; i < len(errs); i++ {
if i%3 == 0 {
@ -1270,7 +1270,7 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) {
testTransactionFetcher(t, txFetcherTest{
init: func() *TxFetcher {
f := newTestTxFetcher()
f.addTxs = func(txs []*types.Transaction) []error {
f.addTxs = func(_ string, txs []*types.Transaction) []error {
errs := make([]error, len(txs))
for i := 0; i < len(errs); i++ {
errs[i] = txpool.ErrUnderpriced
@ -1787,7 +1787,7 @@ func TestTransactionProtocolViolation(t *testing.T) {
testTransactionFetcherParallel(t, txFetcherTest{
init: func() *TxFetcher {
f := newTestTxFetcher()
f.addTxs = func(txs []*types.Transaction) []error {
f.addTxs = func(_ string, txs []*types.Transaction) []error {
var errs []error
for range txs {
errs = append(errs, txpool.ErrKZGVerificationError)
@ -1888,7 +1888,7 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) {
// Process the original or expanded steps
switch step := step.(type) {
case doTxNotify:
if err := fetcher.Notify(step.peer, step.types, step.sizes, step.hashes); err != nil {
if _, err := fetcher.Notify(step.peer, step.types, step.sizes, step.hashes); err != nil {
t.Errorf("step %d: %v", i, err)
}
<-wait // Fetcher needs to process this, wait until it's done
@ -2194,7 +2194,7 @@ func TestTransactionForgotten(t *testing.T) {
fetcher := NewTxFetcherForTests(
nil,
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
func(_ string, txs []*types.Transaction) []error {
errs := make([]error, len(txs))
for i := 0; i < len(errs); i++ {
errs[i] = txpool.ErrUnderpriced

77
eth/handler.go
View file

@ -32,7 +32,9 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/fetcher"
@ -75,7 +77,7 @@ type txPool interface {
// GetRLP retrieves the RLP-encoded transaction from local txpool
// with given tx hash.
GetRLP(hash common.Hash) []byte
GetRLP(hash common.Hash, version uint) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
@ -97,6 +99,17 @@ type txPool interface {
FilterType(kind byte) bool
}
// blobPool defines the methods needed from a blob pool implementation to
// support cell-based blob data availability.
type blobPool interface {
Has(hash common.Hash) bool
GetBlobHashes(hash common.Hash) []common.Hash
GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error)
GetCustody(hash common.Hash) *types.CustodyBitmap
AddPooledTx(pooledTx *blobpool.BlobTxForPool) error
ValidateTxBasics(pooledTx *types.Transaction) error
}
// handlerConfig is the collection of initialization parameters to create a full
// node network handler.
type handlerConfig struct {
@ -104,10 +117,12 @@ type handlerConfig struct {
Database ethdb.Database // Database for direct sync insertions
Chain *core.BlockChain // Blockchain to serve data from
TxPool txPool // Transaction pool to propagate from
BlobPool blobPool // Blob pool for cell-based blob data availability
Network uint64 // Network identifier to advertise
Sync ethconfig.SyncMode // Whether to snap or full sync
BloomCache uint64 // Megabytes to alloc for snap sync bloom
RequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges
Custody types.CustodyBitmap
}
type handler struct {
@ -117,11 +132,14 @@ type handler struct {
database ethdb.Database
txpool txPool
blobpool blobPool
chain *core.BlockChain
maxPeers int
downloader *downloader.Downloader
txFetcher *fetcher.TxFetcher
blobFetcher *fetcher.BlobFetcher
blobBuffer *blobpool.BlobBuffer
peers *peerSet
txBroadcastKey [16]byte
@ -147,6 +165,7 @@ func newHandler(config *handlerConfig) (*handler, error) {
networkID: config.Network,
database: config.Database,
txpool: config.TxPool,
blobpool: config.BlobPool,
chain: config.Chain,
peers: newPeerSet(),
txBroadcastKey: newBroadcastChoiceKey(),
@ -169,11 +188,35 @@ func newHandler(config *handlerConfig) (*handler, error) {
}
return p.RequestTxs(hashes)
}
addTxs := func(txs []*types.Transaction) []error {
return h.txpool.Add(txs, false)
// Construct the blob buffer for assembling blob txs from separate tx and cell deliveries.
h.blobBuffer = blobpool.NewBlobBuffer(h.blobpool.ValidateTxBasics, h.blobpool.AddPooledTx, h.removePeer)
addTxs := func(peer string, txs []*types.Transaction) []error {
errs := make([]error, len(txs))
p := h.peers.peer(peer)
isETH72 := p != nil && p.Version() >= eth.ETH72
var poolTxs []*types.Transaction
var index []int
for i, tx := range txs {
if isETH72 && tx.Type() == types.BlobTxType {
errs[i] = h.blobBuffer.AddTx(tx, peer)
} else {
poolTxs = append(poolTxs, tx)
index = append(index, i)
}
}
if len(poolTxs) > 0 {
poolErrs := h.txpool.Add(poolTxs, false)
for j, idx := range index {
errs[idx] = poolErrs[j]
}
}
return errs
}
validateMeta := func(tx common.Hash, kind byte) error {
if h.txpool.Has(tx) {
if h.txpool.Has(tx) || h.blobBuffer.HasTx(tx) {
return txpool.ErrAlreadyKnown
}
if !h.txpool.FilterType(kind) {
@ -182,6 +225,29 @@ func newHandler(config *handlerConfig) (*handler, error) {
return nil
}
h.txFetcher = fetcher.NewTxFetcher(h.chain, validateMeta, addTxs, fetchTx, h.removePeer)
// Construct the blob fetcher for cell-based blob data availability
blobCallbacks := fetcher.BlobFetcherFunctions{
FetchPayloads: func(peer string, hashes []common.Hash, cells *types.CustodyBitmap) error {
p := h.peers.peer(peer)
if p == nil {
return errors.New("unknown peer")
}
return p.RequestPayload(hashes, cells)
},
HasPayload: func(hash common.Hash) bool {
return h.blobpool.Has(hash) || h.blobBuffer.HasCells(hash)
},
AddCells: func(hash common.Hash, deliveries map[string]*fetcher.PeerCellDelivery, custody *types.CustodyBitmap) error {
converted := make(map[string]*blobpool.PeerDelivery, len(deliveries))
for peer, d := range deliveries {
converted[peer] = &blobpool.PeerDelivery{Cells: d.Cells, Indices: d.Indices}
}
return h.blobBuffer.AddCells(hash, converted, custody)
},
DropPeer: h.removePeer,
}
h.blobFetcher = fetcher.NewBlobFetcher(blobCallbacks, &config.Custody, nil)
return h, nil
}
@ -396,6 +462,7 @@ func (h *handler) unregisterPeer(id string) {
}
h.downloader.UnregisterPeer(id)
h.txFetcher.Drop(id)
h.blobFetcher.Drop(id)
if err := h.peers.unregisterPeer(id); err != nil {
logger.Error("Ethereum peer removal failed", "err", err)
@ -418,6 +485,7 @@ func (h *handler) Start(maxPeers int) {
// start sync handlers
h.txFetcher.Start()
h.blobFetcher.Start()
// start peer handler tracker
h.wg.Add(1)
@ -428,6 +496,7 @@ func (h *handler) Stop() {
h.txsSub.Unsubscribe() // quits txBroadcastLoop
h.blockRange.stop()
h.txFetcher.Stop()
h.blobFetcher.Stop()
h.downloader.Terminate()
// Quit chainSync and txsync64.

19
eth/handler_eth.go
View file

@ -33,6 +33,7 @@ type ethHandler handler
func (h *ethHandler) Chain() *core.BlockChain { return h.chain }
func (h *ethHandler) TxPool() eth.TxPool { return h.txpool }
func (h *ethHandler) BlobPool() eth.BlobPool { return h.blobpool }
// RunPeer is invoked when a peer joins on the `eth` protocol.
func (h *ethHandler) RunPeer(peer *eth.Peer, hand eth.Handler) error {
@ -58,8 +59,19 @@ func (h *ethHandler) AcceptTxs() bool {
func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
// Consume any broadcasts and announces, forwarding the rest to the downloader
switch packet := packet.(type) {
case *eth.NewPooledTransactionHashesPacket:
return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
case *eth.NewPooledTransactionHashesPacket72:
hashes, err := h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
if err != nil {
return err
}
if len(hashes) != 0 {
return h.blobFetcher.Notify(peer.ID(), hashes, packet.Mask)
}
return nil
case *eth.NewPooledTransactionHashesPacket71:
_, err := h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
return err
case *eth.TransactionsPacket:
txs, err := packet.Items()
@ -81,6 +93,9 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
}
return h.txFetcher.Enqueue(peer.ID(), txs, true)
case *eth.CellsResponse:
return h.blobFetcher.Enqueue(peer.ID(), packet.Hashes, packet.Cells, packet.Mask)
default:
return fmt.Errorf("unexpected eth packet type: %T", packet)
}

34
eth/handler_eth_test.go
View file

@ -44,13 +44,14 @@ type testEthHandler struct {
func (h *testEthHandler) Chain() *core.BlockChain { panic("no backing chain") }
func (h *testEthHandler) TxPool() eth.TxPool { panic("no backing tx pool") }
func (h *testEthHandler) BlobPool() eth.BlobPool { return nil }
func (h *testEthHandler) AcceptTxs() bool { return true }
func (h *testEthHandler) RunPeer(*eth.Peer, eth.Handler) error { panic("not used in tests") }
func (h *testEthHandler) PeerInfo(enode.ID) interface{} { panic("not used in tests") }
func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
switch packet := packet.(type) {
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket71:
h.txAnnounces.Send(packet.Hashes)
return nil
@ -105,10 +106,12 @@ func testForkIDSplit(t *testing.T, protocol uint) {
_, blocksNoFork, _ = core.GenerateChainWithGenesis(gspecNoFork, engine, 2, nil)
_, blocksProFork, _ = core.GenerateChainWithGenesis(gspecProFork, engine, 2, nil)
txPool = newTestTxPool()
ethNoFork, _ = newHandler(&handlerConfig{
Database: dbNoFork,
Chain: chainNoFork,
TxPool: newTestTxPool(),
TxPool: txPool,
BlobPool: txPool,
Network: 1,
Sync: ethconfig.FullSync,
BloomCache: 1,
@ -116,7 +119,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
ethProFork, _ = newHandler(&handlerConfig{
Database: dbProFork,
Chain: chainProFork,
TxPool: newTestTxPool(),
TxPool: txPool,
BlobPool: txPool,
Network: 1,
Sync: ethconfig.FullSync,
BloomCache: 1,
@ -137,8 +141,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
peerNoFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil)
peerProFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil)
peerNoFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil, nil)
peerProFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@ -168,8 +172,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil, nil)
peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@ -199,8 +203,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
peerNoFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil)
peerNoFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@ -249,8 +253,8 @@ func testRecvTransactions(t *testing.T, protocol uint) {
defer p2pSrc.Close()
defer p2pSink.Close()
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, nil)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, nil)
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, handler.txpool, nil)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, handler.txpool, nil)
defer src.Close()
defer sink.Close()
@ -305,8 +309,8 @@ func testSendTransactions(t *testing.T, protocol uint) {
defer p2pSrc.Close()
defer p2pSink.Close()
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, nil)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, nil)
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, handler.blobpool, nil)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, handler.blobpool, nil)
defer src.Close()
defer sink.Close()
@ -380,8 +384,8 @@ func testTransactionPropagation(t *testing.T, protocol uint) {
defer sourcePipe.Close()
defer sinkPipe.Close()
sourcePeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{byte(i + 1)}, "", nil, sourcePipe), sourcePipe, source.txpool, nil)
sinkPeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, sink.txpool, nil)
sourcePeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{byte(i + 1)}, "", nil, sourcePipe), sourcePipe, source.txpool, source.txpool, nil)
sinkPeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, sink.txpool, sink.txpool, nil)
defer sourcePeer.Close()
defer sinkPeer.Close()

140
eth/handler_test.go
View file

@ -29,8 +29,10 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/txpool/blobpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/ethdb"
@ -54,7 +56,10 @@ var (
// Its goal is to get around setting up a valid statedb for the balance and nonce
// checks.
type testTxPool struct {
pool map[common.Hash]*types.Transaction // Hash map of collected transactions
txPool map[common.Hash]*types.Transaction // Hash map of collected transactions
cellPool map[common.Hash][]kzg4844.Cell
custody map[common.Hash]types.CustodyBitmap
txFeed event.Feed // Notification feed to allow waiting for inclusion
lock sync.RWMutex // Protects the transaction pool
@ -63,7 +68,9 @@ type testTxPool struct {
// newTestTxPool creates a mock transaction pool.
func newTestTxPool() *testTxPool {
return &testTxPool{
pool: make(map[common.Hash]*types.Transaction),
txPool: make(map[common.Hash]*types.Transaction),
cellPool: make(map[common.Hash][]kzg4844.Cell),
custody: make(map[common.Hash]types.CustodyBitmap),
}
}
@ -73,7 +80,16 @@ func (p *testTxPool) Has(hash common.Hash) bool {
p.lock.Lock()
defer p.lock.Unlock()
return p.pool[hash] != nil
return p.txPool[hash] != nil
}
// Has returns an indicator whether txpool has a transaction
// cached with the given hash.
func (p *testTxPool) HasPayload(hash common.Hash) bool {
p.lock.Lock()
defer p.lock.Unlock()
return p.cellPool[hash] != nil
}
// Get retrieves the transaction from local txpool with given
@ -81,16 +97,16 @@ func (p *testTxPool) Has(hash common.Hash) bool {
func (p *testTxPool) Get(hash common.Hash) *types.Transaction {
p.lock.Lock()
defer p.lock.Unlock()
return p.pool[hash]
return p.txPool[hash]
}
// Get retrieves the transaction from local txpool with given
// tx hash.
func (p *testTxPool) GetRLP(hash common.Hash) []byte {
func (p *testTxPool) GetRLP(hash common.Hash, _ uint) []byte {
p.lock.Lock()
defer p.lock.Unlock()
tx := p.pool[hash]
tx := p.txPool[hash]
if tx != nil {
blob, _ := rlp.EncodeToBytes(tx)
return blob
@ -104,7 +120,7 @@ func (p *testTxPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
p.lock.Lock()
defer p.lock.Unlock()
tx := p.pool[hash]
tx := p.txPool[hash]
if tx != nil {
return &txpool.TxMetadata{
Type: tx.Type(),
@ -121,7 +137,7 @@ func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error {
defer p.lock.Unlock()
for _, tx := range txs {
p.pool[tx.Hash()] = tx
p.txPool[tx.Hash()] = tx
}
p.txFeed.Send(core.NewTxsEvent{Txs: txs})
return make([]error, len(txs))
@ -134,7 +150,7 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]
var count int
batches := make(map[common.Address][]*types.Transaction)
for _, tx := range p.pool {
for _, tx := range p.txPool {
from, _ := types.Sender(types.HomesteadSigner{}, tx)
batches[from] = append(batches[from], tx)
}
@ -164,6 +180,87 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]
func (p *testTxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool) event.Subscription {
return p.txFeed.Subscribe(ch)
}
func (p *testTxPool) GetBlobHashes(hash common.Hash) []common.Hash {
p.lock.RLock()
defer p.lock.RUnlock()
tx, exists := p.txPool[hash]
if !exists {
return nil
}
return tx.BlobHashes()
}
func (p *testTxPool) GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error) {
p.lock.RLock()
defer p.lock.RUnlock()
requestedIndices := mask.Indices()
cells := make([][]*kzg4844.Cell, len(vhashes))
proofs := make([][]*kzg4844.Proof, len(vhashes))
for i, vhash := range vhashes {
// Find the tx containing this versioned hash
var foundTx *types.Transaction
var blobIdx int
for _, tx := range p.txPool {
for j, bh := range tx.BlobHashes() {
if bh == vhash {
foundTx = tx
blobIdx = j
break
}
}
if foundTx != nil {
break
}
}
if foundTx == nil {
continue
}
txCells, ok := p.cellPool[foundTx.Hash()]
if !ok {
continue
}
_ = blobIdx // cells in the mock are stored flat by cell index
blobCells := make([]*kzg4844.Cell, len(requestedIndices))
for j, idx := range requestedIndices {
if int(idx) < len(txCells) {
cell := txCells[idx]
blobCells[j] = &cell
}
}
cells[i] = blobCells
}
return cells, proofs, nil
}
func (p *testTxPool) GetCustody(hash common.Hash) *types.CustodyBitmap {
p.lock.RLock()
defer p.lock.RUnlock()
mask, ok := p.custody[hash]
if !ok {
return nil
}
return &mask
}
// AddCells adds cells for a specific transaction hash (for testing)
func (p *testTxPool) AddCells(hash common.Hash, cells []kzg4844.Cell, mask types.CustodyBitmap) {
p.lock.Lock()
defer p.lock.Unlock()
p.cellPool[hash] = cells
p.custody[hash] = mask
}
func (p *testTxPool) AddPooledTx(pooledTx *blobpool.BlobTxForPool) error {
p.lock.Lock()
defer p.lock.Unlock()
hash := pooledTx.Tx.Hash()
p.cellPool[hash] = pooledTx.Cells
p.txPool[hash] = pooledTx.Tx
return nil
}
// FilterType should check whether the pool supports the given type of transactions.
func (p *testTxPool) FilterType(kind byte) bool {
@ -174,14 +271,19 @@ func (p *testTxPool) FilterType(kind byte) bool {
return false
}
func (p *testTxPool) ValidateTxBasics(_ *types.Transaction) error {
return nil
}
// testHandler is a live implementation of the Ethereum protocol handler, just
// preinitialized with some sane testing defaults and the transaction pool mocked
// out.
type testHandler struct {
db ethdb.Database
chain *core.BlockChain
txpool *testTxPool
handler *handler
db ethdb.Database
chain *core.BlockChain
txpool *testTxPool
blobpool *testTxPool
handler *handler
}
// newTestHandler creates a new handler for testing purposes with no blocks.
@ -210,6 +312,7 @@ func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler
Database: db,
Chain: chain,
TxPool: txpool,
BlobPool: txpool,
Network: 1,
Sync: mode,
BloomCache: 1,
@ -217,10 +320,11 @@ func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler
handler.Start(1000)
return &testHandler{
db: db,
chain: chain,
txpool: txpool,
handler: handler,
db: db,
chain: chain,
txpool: txpool,
blobpool: txpool,
handler: handler,
}
}
@ -317,7 +421,7 @@ func createTestPeers(rand *rand.Rand, n int) []*ethPeer {
var id enode.ID
rand.Read(id[:])
p2pPeer := p2p.NewPeer(id, "test", nil)
ep := eth.NewPeer(eth.ETH69, p2pPeer, nil, nil, nil)
ep := eth.NewPeer(eth.ETH69, p2pPeer, nil, nil, nil, nil)
peers[i] = &ethPeer{Peer: ep}
}
return peers

36
eth/protocols/eth/broadcast.go
View file

@ -113,29 +113,55 @@ func (p *Peer) announceTransactions() {
pending []common.Hash
pendingTypes []byte
pendingSizes []uint32
mask types.CustodyBitmap
size common.StorageSize
processed = make(map[int]bool)
)
for count = 0; count < len(queue) && size < maxTxPacketSize; count++ {
if meta := p.txpool.GetMetadata(queue[count]); meta != nil {
custody := p.blobpool.GetCustody(queue[count])
if custody != nil {
// blob tx
if mask.OneCount() == 0 {
mask = *custody
} else {
if mask != *custody {
// group by mask
continue
}
}
}
pending = append(pending, queue[count])
pendingTypes = append(pendingTypes, meta.Type)
pendingSizes = append(pendingSizes, uint32(meta.Size))
if p.version >= ETH72 && meta.SizeWithoutBlob > 0 {
pendingSizes = append(pendingSizes, uint32(meta.SizeWithoutBlob))
} else {
pendingSizes = append(pendingSizes, uint32(meta.Size))
}
size += common.HashLength
processed[count] = true
}
}
// Shift and trim queue
queue = queue[:copy(queue, queue[count:])]
// Shift and trim queue using processed map
var remaining []common.Hash
for i, h := range queue {
if !processed[i] {
remaining = append(remaining, h)
}
}
queue = remaining
// If there's anything available to transfer, fire up an async writer
if len(pending) > 0 {
done = make(chan struct{})
go func() {
if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes); err != nil {
if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes, mask); err != nil {
fail <- err
return
}
close(done)
p.Log().Trace("Sent transaction announcements", "count", len(pending))
p.Log().Trace("Sent transaction announcements", "count", len(pending), "mask", mask, "tx", pending)
}()
}
}

38
eth/protocols/eth/handler.go
View file

@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
@ -68,6 +69,9 @@ type Backend interface {
// TxPool retrieves the transaction pool object to serve data.
TxPool() TxPool
// BlobPool retrieves the blob pool object to serve cell requests.
BlobPool() BlobPool
// AcceptTxs retrieves whether transaction processing is enabled on the node
// or if inbound transactions should simply be dropped.
AcceptTxs() bool
@ -87,6 +91,18 @@ type Backend interface {
Handle(peer *Peer, packet Packet) error
}
// BlobPool defines the methods needed by the protocol handler to serve cell requests.
type BlobPool interface {
// GetBlobHashes returns the blob versioned hashes for a given transaction hash.
GetBlobHashes(hash common.Hash) []common.Hash
// GetBlobCells retrieves cells and proofs for given versioned blob hashes filtered by the custody bitmap.
GetBlobCells(vhashes []common.Hash, mask types.CustodyBitmap) ([][]*kzg4844.Cell, [][]*kzg4844.Proof, error)
// GetCustody returns the custody bitmap for a given transaction hash.
GetCustody(hash common.Hash) *types.CustodyBitmap
// Has returns whether the blob pool contains a transaction with the given hash.
Has(hash common.Hash) bool
}
// TxPool defines the methods needed by the protocol handler to serve transactions.
type TxPool interface {
// Get retrieves the transaction from the local txpool with the given hash.
@ -94,7 +110,7 @@ type TxPool interface {
// GetRLP retrieves the RLP-encoded transaction from the local txpool with
// the given hash.
GetRLP(hash common.Hash) []byte
GetRLP(hash common.Hash, version uint) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
@ -110,7 +126,7 @@ func MakeProtocols(backend Backend, network uint64, disc enode.Iterator) []p2p.P
Version: version,
Length: protocolLengths[version],
Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
peer := NewPeer(version, p, rw, backend.TxPool(), backend.Chain().Config())
peer := NewPeer(version, p, rw, backend.TxPool(), backend.BlobPool(), backend.Chain().Config())
defer peer.Close()
return backend.RunPeer(peer, func(peer *Peer) error {
@ -197,6 +213,22 @@ var eth70 = map[uint64]msgHandler{
BlockRangeUpdateMsg: handleBlockRangeUpdate,
}
var eth72 = map[uint64]msgHandler{
TransactionsMsg: handleTransactions,
NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes71,
GetBlockHeadersMsg: handleGetBlockHeaders,
BlockHeadersMsg: handleBlockHeaders,
GetBlockBodiesMsg: handleGetBlockBodies,
BlockBodiesMsg: handleBlockBodies,
GetReceiptsMsg: handleGetReceipts70,
ReceiptsMsg: handleReceipts70,
GetPooledTransactionsMsg: handleGetPooledTransactions,
PooledTransactionsMsg: handlePooledTransactions,
BlockRangeUpdateMsg: handleBlockRangeUpdate,
GetCellsMsg: handleGetCells,
CellsMsg: handleCells,
}
// handleMessage is invoked whenever an inbound message is received from a remote
// peer. The remote connection is torn down upon returning any error.
func handleMessage(backend Backend, peer *Peer) error {
@ -216,6 +248,8 @@ func handleMessage(backend Backend, peer *Peer) error {
handlers = eth69
case ETH70:
handlers = eth70
case ETH72:
handlers = eth72
default:
return fmt.Errorf("unknown eth protocol version: %v", peer.version)
}

15
eth/protocols/eth/handler_test.go
View file

@ -62,9 +62,10 @@ func u64(val uint64) *uint64 { return &val }
// purpose is to allow testing the request/reply workflows and wire serialization
// in the `eth` protocol without actually doing any data processing.
type testBackend struct {
db ethdb.Database
chain *core.BlockChain
txpool *txpool.TxPool
db ethdb.Database
chain *core.BlockChain
txpool *txpool.TxPool
blobpool *blobpool.BlobPool
}
// newTestBackend creates an empty chain and wraps it into a mock backend.
@ -142,9 +143,10 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, cancun bool, generat
txpool, _ := txpool.New(txconfig.PriceLimit, chain, []txpool.SubPool{legacyPool, blobPool})
return &testBackend{
db: db,
chain: chain,
txpool: txpool,
db: db,
chain: chain,
txpool: txpool,
blobpool: blobPool,
}
}
@ -156,6 +158,7 @@ func (b *testBackend) close() {
func (b *testBackend) Chain() *core.BlockChain { return b.chain }
func (b *testBackend) TxPool() TxPool { return b.txpool }
func (b *testBackend) BlobPool() BlobPool { return b.blobpool }
func (b *testBackend) RunPeer(peer *Peer, handler Handler) error {
// Normally the backend would do peer maintenance and handshakes. All that

103
eth/protocols/eth/handlers.go
View file

@ -26,6 +26,7 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/tracker"
"github.com/ethereum/go-ethereum/rlp"
@ -568,7 +569,27 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer)
if !backend.AcceptTxs() {
return nil
}
ann := new(NewPooledTransactionHashesPacket)
ann := new(NewPooledTransactionHashesPacket71)
if err := msg.Decode(ann); err != nil {
return err
}
if len(ann.Hashes) != len(ann.Types) || len(ann.Hashes) != len(ann.Sizes) {
return fmt.Errorf("NewPooledTransactionHashes: invalid len of fields in %v %v %v", len(ann.Hashes), len(ann.Types), len(ann.Sizes))
}
// Schedule all the unknown hashes for retrieval
for _, hash := range ann.Hashes {
peer.MarkTransaction(hash)
}
return backend.Handle(peer, ann)
}
func handleNewPooledTransactionHashes71(backend Backend, msg Decoder, peer *Peer) error {
// New transaction announcement arrived, make sure we have
// a valid and fresh chain to handle them
if !backend.AcceptTxs() {
return nil
}
ann := new(NewPooledTransactionHashesPacket72)
if err := msg.Decode(ann); err != nil {
return err
}
@ -588,11 +609,11 @@ func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error
if err := msg.Decode(&query); err != nil {
return err
}
hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsRequest)
hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsRequest, peer.version)
return peer.ReplyPooledTransactionsRLP(query.RequestId, hashes, txs)
}
func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsRequest) ([]common.Hash, []rlp.RawValue) {
func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsRequest, version uint) ([]common.Hash, []rlp.RawValue) {
// Gather transactions until the fetch or network limits is reached
var (
bytes int
@ -604,7 +625,7 @@ func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsReq
break
}
// Retrieve the requested transaction, skipping if unknown to us
encoded := backend.TxPool().GetRLP(hash)
encoded := backend.TxPool().GetRLP(hash, version)
if len(encoded) == 0 {
continue
}
@ -666,3 +687,77 @@ func handleBlockRangeUpdate(backend Backend, msg Decoder, peer *Peer) error {
peer.lastRange.Store(&update)
return nil
}
func handleGetCells(backend Backend, msg Decoder, peer *Peer) error {
// Decode the cell retrieval message
var query GetCellsRequestPacket
if err := msg.Decode(&query); err != nil {
return err
}
hashes, cells, custody := answerGetCells(backend, query.GetCellsRequest)
return peer.ReplyCells(query.RequestId, hashes, cells, custody)
}
func answerGetCells(backend Backend, query GetCellsRequest) ([]common.Hash, [][]kzg4844.Cell, types.CustodyBitmap) {
var (
cellCounts int
hashes []common.Hash
cells [][]kzg4844.Cell
)
maxCells := softResponseLimit / 2048
for _, hash := range query.Hashes {
if cellCounts >= maxCells {
break
}
// Look up the blob versioned hashes for this transaction
vhashes := backend.BlobPool().GetBlobHashes(hash)
if len(vhashes) == 0 {
continue
}
blobCells, _, _ := backend.BlobPool().GetBlobCells(vhashes, query.Mask)
// Flatten per-blob cells into a single slice. If any blob has a nil
// entry (unavailable cell), skip the entire transaction.
var flat []kzg4844.Cell
skip := false
for _, bc := range blobCells {
if bc == nil {
skip = true
break
}
for _, c := range bc {
if c == nil {
skip = true
break
}
flat = append(flat, *c)
}
if skip {
break
}
}
if skip || len(flat) == 0 {
continue
}
hashes = append(hashes, hash)
cells = append(cells, flat)
cellCounts += len(flat)
}
return hashes, cells, query.Mask
}
func handleCells(backend Backend, msg Decoder, peer *Peer) error {
var cellsResponse CellsPacket
if err := msg.Decode(&cellsResponse); err != nil {
return err
}
tresp := tracker.Response{
ID: cellsResponse.RequestId,
MsgCode: CellsMsg,
Size: len(cellsResponse.CellsResponse.Hashes),
}
if err := peer.tracker.Fulfil(tresp); err != nil {
return fmt.Errorf("Cells: %w", err)
}
return backend.Handle(peer, &cellsResponse.CellsResponse)
}

2
eth/protocols/eth/handshake_test.go
View file

@ -77,7 +77,7 @@ func testHandshake(t *testing.T, protocol uint) {
defer app.Close()
defer net.Close()
peer := NewPeer(protocol, p2p.NewPeer(enode.ID{}, "peer", nil), net, nil, nil)
peer := NewPeer(protocol, p2p.NewPeer(enode.ID{}, "peer", nil), net, nil, nil, nil)
defer peer.Close()
// Send the junk test with one peer, check the handshake failure

51
eth/protocols/eth/peer.go
View file

@ -27,6 +27,7 @@ import (
mapset "github.com/deckarep/golang-set/v2"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/tracker"
"github.com/ethereum/go-ethereum/params"
@ -66,7 +67,8 @@ type Peer struct {
version uint // Protocol version negotiated
lastRange atomic.Pointer[BlockRangeUpdatePacket]
txpool TxPool // Transaction pool used by the broadcasters for liveness checks
txpool TxPool // Transaction pool used by the broadcasters for liveness checks
blobpool BlobPool
knownTxs *knownCache // Set of transaction hashes known to be known by this peer
txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests
txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests
@ -86,11 +88,11 @@ type Peer struct {
// NewPeer creates a wrapper for a network connection and negotiated protocol
// version.
func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool, chainConfig *params.ChainConfig) *Peer {
func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool, blobpool BlobPool, chainConfig *params.ChainConfig) *Peer {
cap := p2p.Cap{Name: ProtocolName, Version: version}
id := p.ID().String()
peer := &Peer{
id: p.ID().String(),
id: id,
Peer: p,
rw: rw,
version: version,
@ -102,6 +104,7 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool, cha
reqCancel: make(chan *cancel),
resDispatch: make(chan *response),
txpool: txpool,
blobpool: blobpool,
chainConfig: chainConfig,
receiptBuffer: make(map[uint64]*receiptRequest),
term: make(chan struct{}),
@ -186,10 +189,13 @@ func (p *Peer) AsyncSendTransactions(hashes []common.Hash) {
// This method is a helper used by the async transaction announcer. Don't call it
// directly as the queueing (memory) and transmission (bandwidth) costs should
// not be managed directly.
func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32) error {
func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32, cells types.CustodyBitmap) error {
// Mark all the transactions as known, but ensure we don't overflow our limits
p.knownTxs.Add(hashes...)
return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket{Types: types, Sizes: sizes, Hashes: hashes})
if p.version >= ETH72 {
return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket72{Types: types, Sizes: sizes, Hashes: hashes, Mask: cells})
}
return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket71{Types: types, Sizes: sizes, Hashes: hashes})
}
// AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually
@ -242,6 +248,41 @@ func (p *Peer) ReplyReceiptsRLP69(id uint64, receipts rlp.RawList[*ReceiptList])
})
}
// ReplyCells is the response to GetCells.
func (p *Peer) ReplyCells(id uint64, hashes []common.Hash, cells [][]kzg4844.Cell, mask types.CustodyBitmap) error {
return p2p.Send(p.rw, CellsMsg, &CellsPacket{
RequestId: id,
CellsResponse: CellsResponse{
Hashes: hashes,
Cells: cells,
Mask: mask,
},
})
}
// RequestPayload fetches a batch of cells from a remote node.
func (p *Peer) RequestPayload(hashes []common.Hash, cell *types.CustodyBitmap) error {
p.Log().Debug("Fetching batch of cells", "txcount", len(hashes), "cellcount", cell.OneCount())
id := rand.Uint64()
err := p.tracker.Track(tracker.Request{
ID: id,
ReqCode: GetCellsMsg,
RespCode: CellsMsg,
Size: len(hashes),
})
if err != nil {
return err
}
return p2p.Send(p.rw, GetCellsMsg, &GetCellsRequestPacket{
RequestId: id,
GetCellsRequest: GetCellsRequest{
Hashes: hashes,
Mask: *cell,
},
})
}
// ReplyReceiptsRLP70 is the response to GetReceipts.
func (p *Peer) ReplyReceiptsRLP70(id uint64, receipts rlp.RawList[*ReceiptList], lastBlockIncomplete bool) error {
return p2p.Send(p.rw, ReceiptsMsg, &ReceiptsPacket70{

2
eth/protocols/eth/peer_test.go
View file

@ -45,7 +45,7 @@ func newTestPeer(name string, version uint, backend Backend) (*testPeer, <-chan
var id enode.ID
rand.Read(id[:])
peer := NewPeer(version, p2p.NewPeer(id, name, nil), net, backend.TxPool(), nil)
peer := NewPeer(version, p2p.NewPeer(id, name, nil), net, backend.TxPool(), backend.BlobPool(), nil)
errc := make(chan error, 1)
go func() {
defer app.Close()

59
eth/protocols/eth/protocol.go
View file

@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/forkid"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/rlp"
)
@ -31,6 +32,7 @@ import (
const (
ETH69 = 69
ETH70 = 70
ETH72 = 72
)
// ProtocolName is the official short name of the `eth` protocol used during
@ -39,11 +41,11 @@ const ProtocolName = "eth"
// ProtocolVersions are the supported versions of the `eth` protocol (first
// is primary).
var ProtocolVersions = []uint{ETH70, ETH69}
var ProtocolVersions = []uint{ETH72, ETH70, ETH69}
// protocolLengths are the number of implemented message corresponding to
// different protocol versions.
var protocolLengths = map[uint]uint64{ETH69: 18, ETH70: 18}
var protocolLengths = map[uint]uint64{ETH69: 18, ETH70: 18, ETH72: 22}
// maxMessageSize is the maximum cap on the size of a protocol message.
const maxMessageSize = 10 * 1024 * 1024
@ -66,6 +68,8 @@ const (
GetReceiptsMsg = 0x0f
ReceiptsMsg = 0x10
BlockRangeUpdateMsg = 0x11
GetCellsMsg = 0x14
CellsMsg = 0x15
)
var (
@ -245,13 +249,22 @@ type ReceiptsPacket70 struct {
// ReceiptsRLPResponse is used for receipts, when we already have it encoded
type ReceiptsRLPResponse []rlp.RawValue
// NewPooledTransactionHashesPacket represents a transaction announcement packet on eth/68 and newer.
type NewPooledTransactionHashesPacket struct {
// NewPooledTransactionHashesPacket71 represents a transaction announcement packet on eth/69.
type NewPooledTransactionHashesPacket71 struct {
Types []byte
Sizes []uint32
Hashes []common.Hash
}
// NewPooledTransactionHashesPacket72 represents a transaction announcement packet on ETH/72
// with an additional custody bitmap field for cell-based blob data availability.
type NewPooledTransactionHashesPacket72 struct {
Types []byte
Sizes []uint32
Hashes []common.Hash
Mask types.CustodyBitmap
}
// GetPooledTransactionsRequest represents a transaction query.
type GetPooledTransactionsRequest []common.Hash
@ -288,6 +301,31 @@ type BlockRangeUpdatePacket struct {
LatestBlockHash common.Hash
}
// GetCellsRequest represents a request for cells of blob transactions.
type GetCellsRequest struct {
Hashes []common.Hash
Mask types.CustodyBitmap
}
// GetCellsRequestPacket represents a cell request with request ID wrapping.
type GetCellsRequestPacket struct {
RequestId uint64
GetCellsRequest
}
// CellsResponse represents a response containing cells for blob transactions.
type CellsResponse struct {
Hashes []common.Hash
Cells [][]kzg4844.Cell
Mask types.CustodyBitmap
}
// CellsPacket represents a cells response with request ID wrapping.
type CellsPacket struct {
RequestId uint64
CellsResponse
}
func (*StatusPacket) Name() string { return "Status" }
func (*StatusPacket) Kind() byte { return StatusMsg }
@ -306,8 +344,11 @@ func (*GetBlockBodiesRequest) Kind() byte { return GetBlockBodiesMsg }
func (*BlockBodiesResponse) Name() string { return "BlockBodies" }
func (*BlockBodiesResponse) Kind() byte { return BlockBodiesMsg }
func (*NewPooledTransactionHashesPacket) Name() string { return "NewPooledTransactionHashes" }
func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransactionHashesMsg }
func (*NewPooledTransactionHashesPacket71) Name() string { return "NewPooledTransactionHashes" }
func (*NewPooledTransactionHashesPacket71) Kind() byte { return NewPooledTransactionHashesMsg }
func (*NewPooledTransactionHashesPacket72) Name() string { return "NewPooledTransactionHashes" }
func (*NewPooledTransactionHashesPacket72) Kind() byte { return NewPooledTransactionHashesMsg }
func (*GetPooledTransactionsRequest) Name() string { return "GetPooledTransactions" }
func (*GetPooledTransactionsRequest) Kind() byte { return GetPooledTransactionsMsg }
@ -326,3 +367,9 @@ func (*ReceiptsRLPResponse) Kind() byte { return ReceiptsMsg }
func (*BlockRangeUpdatePacket) Name() string { return "BlockRangeUpdate" }
func (*BlockRangeUpdatePacket) Kind() byte { return BlockRangeUpdateMsg }
func (*GetCellsRequest) Name() string { return "GetCells" }
func (*GetCellsRequest) Kind() byte { return GetCellsMsg }
func (*CellsResponse) Name() string { return "Cells" }
func (*CellsResponse) Kind() byte { return CellsMsg }

4
eth/sync_test.go
View file

@ -50,8 +50,8 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) {
defer emptyPipeEth.Close()
defer fullPipeEth.Close()
emptyPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{1}, "", caps), emptyPipeEth, empty.txpool, nil)
fullPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{2}, "", caps), fullPipeEth, full.txpool, nil)
emptyPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{1}, "", caps), emptyPipeEth, empty.txpool, empty.blobpool, nil)
fullPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{2}, "", caps), fullPipeEth, full.txpool, full.blobpool, nil)
defer emptyPeerEth.Close()
defer fullPeerEth.Close()

4
tests/fuzzers/txfetcher/txfetcher_fuzzer.go
View file

@ -80,7 +80,7 @@ func fuzz(input []byte) int {
f := fetcher.NewTxFetcherForTests(
nil,
func(common.Hash, byte) error { return nil },
func(txs []*types.Transaction) []error {
func(_ string, txs []*types.Transaction) []error {
return make([]error, len(txs))
},
func(string, []common.Hash) error { return nil },
@ -139,7 +139,7 @@ func fuzz(input []byte) int {
if verbose {
fmt.Println("Notify", peer, announceIdxs)
}
if err := f.Notify(peer, types, sizes, announces); err != nil {
if _, err := f.Notify(peer, types, sizes, announces); err != nil {
panic(err)
}