forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-05-24 08:49:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

implement sparse blobpool

Browse source
This commit is contained in:
healthykim 2026-03-19 02:52:39 +09:00
parent ab357151da
commit e84b6eb568
41 changed files with 3638 additions and 361 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/devp2p/internal/ethtest/conn.go
View file

@ -167,7 +167,7 @@ func (c *Conn) ReadEth() (any, error) {
case eth.TransactionsMsg:
msg = new(eth.TransactionsPacket)
case eth.NewPooledTransactionHashesMsg:
msg = new(eth.NewPooledTransactionHashesPacket)
msg = new(eth.NewPooledTransactionHashesPacket70)
case eth.GetPooledTransactionsMsg:
msg = new(eth.GetPooledTransactionsPacket)
case eth.PooledTransactionsMsg:

14
cmd/devp2p/internal/ethtest/suite.go
View file

@ -865,7 +865,7 @@ the transactions using a GetPooledTransactions request.`)
}
// Send announcement.
ann := eth.NewPooledTransactionHashesPacket{Types: txTypes, Sizes: sizes, Hashes: hashes}
ann := eth.NewPooledTransactionHashesPacket70{Types: txTypes, Sizes: sizes, Hashes: hashes}
err = conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann)
if err != nil {
t.Fatalf("failed to write to connection: %v", err)
@ -883,7 +883,7 @@ the transactions using a GetPooledTransactions request.`)
t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsRequest))
}
return
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket70:
continue
case *eth.TransactionsPacket:
continue
@ -949,12 +949,12 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
t2 = s.makeBlobTxs(2, 3, 0x2)
)
for _, test := range []struct {
ann eth.NewPooledTransactionHashesPacket
ann eth.NewPooledTransactionHashesPacket70
resp eth.PooledTransactionsResponse
}{
// Invalid tx size.
{
ann: eth.NewPooledTransactionHashesPacket{
ann: eth.NewPooledTransactionHashesPacket70{
Types: []byte{types.BlobTxType, types.BlobTxType},
Sizes: []uint32{uint32(t1[0].Size()), uint32(t1[1].Size() + 10)},
Hashes: []common.Hash{t1[0].Hash(), t1[1].Hash()},
@ -963,7 +963,7 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
},
// Wrong tx type.
{
ann: eth.NewPooledTransactionHashesPacket{
ann: eth.NewPooledTransactionHashesPacket70{
Types: []byte{types.DynamicFeeTxType, types.BlobTxType},
Sizes: []uint32{uint32(t2[0].Size()), uint32(t2[1].Size())},
Hashes: []common.Hash{t2[0].Hash(), t2[1].Hash()},
@ -1092,7 +1092,7 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types
return
}
ann := eth.NewPooledTransactionHashesPacket{
ann := eth.NewPooledTransactionHashesPacket70{
Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(badTx.Size())},
Hashes: []common.Hash{badTx.Hash()},
@ -1143,7 +1143,7 @@ func (s *Suite) testBadBlobTx(t *utesting.T, tx *types.Transaction, badTx *types
return
}
ann := eth.NewPooledTransactionHashesPacket{
ann := eth.NewPooledTransactionHashesPacket70{
Types: []byte{types.BlobTxType},
Sizes: []uint32{uint32(tx.Size())},
Hashes: []common.Hash{tx.Hash()},

4
cmd/devp2p/internal/ethtest/transaction.go
View file

@ -74,7 +74,7 @@ func (s *Suite) sendTxs(t *utesting.T, txs []*types.Transaction) error {
for _, tx := range txs {
got[tx.Hash()] = true
}
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket70:
for _, hash := range msg.Hashes {
got[hash] = true
}
@ -160,7 +160,7 @@ func (s *Suite) sendInvalidTxs(t *utesting.T, txs []*types.Transaction) error {
return fmt.Errorf("received bad tx: %s", tx.Hash())
}
}
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket70:
for _, hash := range msg.Hashes {
if _, ok := invalids[hash]; ok {
return fmt.Errorf("received bad tx: %s", hash)

728
core/txpool/blobpool/blobpool.go
View file

File diff suppressed because it is too large Load diff

268
core/txpool/blobpool/blobpool_test.go
View file

@ -486,7 +486,7 @@ func verifyBlobRetrievals(t *testing.T, pool *BlobPool) {
// - 8. Fully duplicate transactions (matching hash) must be dropped
// - 9. Duplicate nonces from the same account must be dropped
func TestOpenDrops(t *testing.T) {
//log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true)))
// log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelTrace, true)))
// Create a temporary folder for the persistent backend
storage := t.TempDir()
@ -513,75 +513,76 @@ func TestOpenDrops(t *testing.T) {
S: new(uint256.Int),
})
blob, _ := rlp.EncodeToBytes(tx)
badsig, _ := store.Put(blob)
badsig := tx.Hash()
store.Put(blob)
// Insert a sequence of transactions with a nonce gap in between to verify
// that anything gapped will get evicted (case 3).
var (
gapper, _ = crypto.GenerateKey()
valids = make(map[uint64]struct{})
gapped = make(map[uint64]struct{})
valids = make(map[common.Hash]struct{})
gapped = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 3, 4, 6, 7} { // first gap at #2, another at #5
tx := makeTx(nonce, 1, 1, 1, gapper)
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce < 2 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
gapped[id] = struct{}{}
gapped[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions with a gapped starting nonce to verify
// that the entire set will get dropped (case 3).
var (
dangler, _ = crypto.GenerateKey()
dangling = make(map[uint64]struct{})
dangling = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{1, 2, 3} { // first gap at #0, all set dangling
tx := makeTx(nonce, 1, 1, 1, dangler)
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob)
dangling[id] = struct{}{}
store.Put(blob)
dangling[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with already passed nonces to veirfy
// that the entire set will get dropped (case 4).
var (
filler, _ = crypto.GenerateKey()
filled = make(map[uint64]struct{})
filled = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // account nonce at 3, all set filled
tx := makeTx(nonce, 1, 1, 1, filler)
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob)
filled[id] = struct{}{}
store.Put(blob)
filled[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with partially passed nonces to verify
// that the included part of the set will get dropped (case 4).
var (
overlapper, _ = crypto.GenerateKey()
overlapped = make(map[uint64]struct{})
overlapped = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2, 3} { // account nonce at 2, half filled
tx := makeTx(nonce, 1, 1, 1, overlapper)
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce >= 2 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
overlapped[id] = struct{}{}
overlapped[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions with an underpriced first to verify that
// the entire set will get dropped (case 5).
var (
underpayer, _ = crypto.GenerateKey()
underpaid = make(map[uint64]struct{})
underpaid = make(map[common.Hash]struct{})
)
for i := 0; i < 5; i++ { // make #0 underpriced
var tx *types.Transaction
@ -592,15 +593,15 @@ func TestOpenDrops(t *testing.T) {
}
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob)
underpaid[id] = struct{}{}
store.Put(blob)
underpaid[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions with an underpriced in between to verify
// that it and anything newly gapped will get evicted (case 5).
var (
outpricer, _ = crypto.GenerateKey()
outpriced = make(map[uint64]struct{})
outpriced = make(map[common.Hash]struct{})
)
for i := 0; i < 5; i++ { // make #2 underpriced
var tx *types.Transaction
@ -611,18 +612,18 @@ func TestOpenDrops(t *testing.T) {
}
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob)
store.Put(blob)
if i < 2 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
outpriced[id] = struct{}{}
outpriced[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions fully overdrafted to verify that the
// entire set will get invalidated (case 6).
var (
exceeder, _ = crypto.GenerateKey()
exceeded = make(map[uint64]struct{})
exceeded = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // nonce 0 overdrafts the account
var tx *types.Transaction
@ -633,14 +634,14 @@ func TestOpenDrops(t *testing.T) {
}
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob)
exceeded[id] = struct{}{}
store.Put(blob)
exceeded[tx.Hash()] = struct{}{}
}
// Insert a sequence of transactions partially overdrafted to verify that part
// of the set will get invalidated (case 6).
var (
overdrafter, _ = crypto.GenerateKey()
overdrafted = make(map[uint64]struct{})
overdrafted = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} { // nonce 1 overdrafts the account
var tx *types.Transaction
@ -651,44 +652,46 @@ func TestOpenDrops(t *testing.T) {
}
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce < 1 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
overdrafted[id] = struct{}{}
overdrafted[tx.Hash()] = struct{}{}
}
}
// Insert a sequence of transactions overflowing the account cap to verify
// that part of the set will get invalidated (case 7).
var (
overcapper, _ = crypto.GenerateKey()
overcapped = make(map[uint64]struct{})
overcapped = make(map[common.Hash]struct{})
)
for nonce := uint64(0); nonce < maxTxsPerAccount+3; nonce++ {
blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, 1, 1, overcapper))
tx := makeTx(nonce, 1, 1, 1, overcapper)
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob)
store.Put(blob)
if nonce < maxTxsPerAccount {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
overcapped[id] = struct{}{}
overcapped[tx.Hash()] = struct{}{}
}
}
// Insert a batch of duplicated transactions to verify that only one of each
// version will remain (case 8).
var (
duplicater, _ = crypto.GenerateKey()
duplicated = make(map[uint64]struct{})
duplicated = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} {
blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, 1, 1, duplicater))
tx := makeTx(nonce, 1, 1, 1, duplicater)
blob, _ := rlp.EncodeToBytes(tx)
for i := 0; i < int(nonce)+1; i++ {
id, _ := store.Put(blob)
store.Put(blob)
if i == 0 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
duplicated[id] = struct{}{}
duplicated[tx.Hash()] = struct{}{}
}
}
}
@ -696,17 +699,18 @@ func TestOpenDrops(t *testing.T) {
// remain (case 9).
var (
repeater, _ = crypto.GenerateKey()
repeated = make(map[uint64]struct{})
repeated = make(map[common.Hash]struct{})
)
for _, nonce := range []uint64{0, 1, 2} {
for i := 0; i < int(nonce)+1; i++ {
blob, _ := rlp.EncodeToBytes(makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater))
tx := makeTx(nonce, 1, uint64(i)+1 /* unique hashes */, 1, repeater)
blob, _ := rlp.EncodeToBytes(tx)
id, _ := store.Put(blob)
store.Put(blob)
if i == 0 {
valids[id] = struct{}{}
valids[tx.Hash()] = struct{}{}
} else {
repeated[id] = struct{}{}
repeated[tx.Hash()] = struct{}{}
}
}
}
@ -743,39 +747,41 @@ func TestOpenDrops(t *testing.T) {
// Verify that the malformed (case 1), badly signed (case 2) and gapped (case
// 3) txs have been deleted from the pool
alive := make(map[uint64]struct{})
alive := make(map[common.Hash]struct{})
for _, txs := range pool.index {
for _, tx := range txs {
switch tx.id {
case malformed:
t.Errorf("malformed RLP transaction remained in storage")
case badsig:
t.Errorf("invalidly signed transaction remained in storage")
default:
if _, ok := dangling[tx.id]; ok {
if badsig == tx.hash {
t.Errorf("invalidly signed transaction remained in storage")
}
if _, ok := dangling[tx.hash]; ok {
t.Errorf("dangling transaction remained in storage: %d", tx.id)
} else if _, ok := filled[tx.id]; ok {
} else if _, ok := filled[tx.hash]; ok {
t.Errorf("filled transaction remained in storage: %d", tx.id)
} else if _, ok := overlapped[tx.id]; ok {
} else if _, ok := overlapped[tx.hash]; ok {
t.Errorf("overlapped transaction remained in storage: %d", tx.id)
} else if _, ok := gapped[tx.id]; ok {
} else if _, ok := gapped[tx.hash]; ok {
t.Errorf("gapped transaction remained in storage: %d", tx.id)
} else if _, ok := underpaid[tx.id]; ok {
} else if _, ok := underpaid[tx.hash]; ok {
t.Errorf("underpaid transaction remained in storage: %d", tx.id)
} else if _, ok := outpriced[tx.id]; ok {
} else if _, ok := outpriced[tx.hash]; ok {
t.Errorf("outpriced transaction remained in storage: %d", tx.id)
} else if _, ok := exceeded[tx.id]; ok {
} else if _, ok := exceeded[tx.hash]; ok {
t.Errorf("fully overdrafted transaction remained in storage: %d", tx.id)
} else if _, ok := overdrafted[tx.id]; ok {
} else if _, ok := overdrafted[tx.hash]; ok {
t.Errorf("partially overdrafted transaction remained in storage: %d", tx.id)
} else if _, ok := overcapped[tx.id]; ok {
} else if _, ok := overcapped[tx.hash]; ok {
t.Errorf("overcapped transaction remained in storage: %d", tx.id)
} else if _, ok := duplicated[tx.id]; ok {
t.Errorf("duplicated transaction remained in storage: %d", tx.id)
} else if _, ok := repeated[tx.id]; ok {
} else if _, ok := repeated[tx.hash]; ok {
t.Errorf("repeated nonce transaction remained in storage: %d", tx.id)
} else {
alive[tx.id] = struct{}{}
if _, ok := alive[tx.hash]; ok {
t.Errorf("duplicated transaction remained in storage: %d", tx.id)
}
alive[tx.hash] = struct{}{}
}
}
}
@ -784,14 +790,14 @@ func TestOpenDrops(t *testing.T) {
if len(alive) != len(valids) {
t.Errorf("valid transaction count mismatch: have %d, want %d", len(alive), len(valids))
}
for id := range alive {
if _, ok := valids[id]; !ok {
t.Errorf("extra transaction %d", id)
for hash := range alive {
if _, ok := valids[hash]; !ok {
t.Errorf("extra transaction %s", hash)
}
}
for id := range valids {
if _, ok := alive[id]; !ok {
t.Errorf("missing transaction %d", id)
for hash := range valids {
if _, ok := alive[hash]; !ok {
t.Errorf("missing transaction %s", hash)
}
}
// Verify all the calculated pool internals. Interestingly, this is **not**
@ -1010,7 +1016,10 @@ func TestOpenCap(t *testing.T) {
keep = []common.Address{addr1, addr3}
drop = []common.Address{addr2}
size = 2 * (txAvgSize + blobSize + uint64(txBlobOverhead))
// After migration to pooledBlobTx, cells (128 x 2048 = 2*blobSize) replace blobs.
// The actual billy slot size for pooledBlobTx is 2*(blobSize+txBlobOverhead)+txAvgSize.
pooledSlotSize uint64 = 2*(blobSize+uint64(txBlobOverhead)) + txAvgSize
size = 2 * pooledSlotSize
)
store.Put(blob1)
store.Put(blob2)
@ -1019,7 +1028,7 @@ func TestOpenCap(t *testing.T) {
// Verify pool capping twice: first by reducing the data cap, then restarting
// with a high cap to ensure everything was persisted previously
for _, datacap := range []uint64{2 * (txAvgSize + blobSize + uint64(txBlobOverhead)), 1000 * (txAvgSize + blobSize + uint64(txBlobOverhead))} {
for _, datacap := range []uint64{size, 1000 * pooledSlotSize} {
// Create a blob pool out of the pre-seeded data, but cap it to 2 blob transaction
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
@ -1325,7 +1334,7 @@ func TestBlobCountLimit(t *testing.T) {
// Check that first succeeds second fails.
if errs[0] != nil {
t.Fatalf("expected tx with 7 blobs to succeed, got %v", errs[0])
t.Fatalf("expected tx with 7 blobs to succeed, got: %v", errs[0])
}
if !errors.Is(errs[1], txpool.ErrTxBlobLimitExceeded) {
t.Fatalf("expected tx with 8 blobs to fail, got: %v", errs[1])
@ -2112,7 +2121,8 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) {
b.Fatal(err)
}
statedb.AddBalance(addr, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
pool.add(tx)
pooledTx, _ := newPooledBlobTx(tx)
pool.add(pooledTx)
}
statedb.Commit(0, true, false)
defer pool.Close()
@ -2133,3 +2143,117 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) {
}
}
}
func TestGetCells(t *testing.T) {
storage := t.TempDir()
os.MkdirAll(filepath.Join(storage, pendingTransactionStore), 0700)
store, _ := billy.Open(billy.Options{Path: filepath.Join(storage, pendingTransactionStore)}, newSlotter(params.BlobTxMaxBlobs), nil)
var (
key1, _ = crypto.GenerateKey()
addr1 = crypto.PubkeyToAddress(key1.PublicKey)
tx1 = makeMultiBlobTx(0, 1, 1000, 100, 3, 0, key1, types.BlobSidecarVersion1) // blobs [0, 3)
pooledTx1, _ = newPooledBlobTx(tx1)
blob1, _ = rlp.EncodeToBytes(pooledTx1)
)
store.Put(blob1)
store.Close()
statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting())
statedb.AddBalance(addr1, uint256.NewInt(1_000_000_000), tracing.BalanceChangeUnspecified)
statedb.Commit(0, true, false)
chain := &testBlockChain{
config: params.MainnetChainConfig,
basefee: uint256.NewInt(params.InitialBaseFee),
blobfee: uint256.NewInt(params.BlobTxMinBlobGasprice),
statedb: statedb,
}
pool := New(Config{Datadir: storage}, chain, nil)
if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil {
t.Fatalf("failed to create blob pool: %v", err)
}
defer pool.Close()
tests := []struct {
name string
hash common.Hash
mask types.CustodyBitmap
expectedLen int
shouldFail bool
}{
{
name: "Get cells with single index",
hash: tx1.Hash(),
mask: types.NewCustodyBitmap([]uint64{5}),
expectedLen: 3,
shouldFail: false,
},
{
name: "Get cells with all indices",
hash: tx1.Hash(),
mask: *types.CustodyBitmapAll,
expectedLen: 384,
shouldFail: false,
},
{
name: "Get cells with no indices",
hash: tx1.Hash(),
mask: types.CustodyBitmap{},
expectedLen: 0,
shouldFail: false,
},
{
name: "Get cells for non-existent transaction",
hash: common.Hash{0x01, 0x02, 0x03},
mask: types.NewCustodyBitmap([]uint64{0, 1}),
expectedLen: 0,
shouldFail: true,
},
{
name: "Get cells with random indices",
hash: tx1.Hash(),
mask: types.NewRandomCustodyBitmap(8),
expectedLen: 24,
shouldFail: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cells, err := pool.GetCells(tt.hash, tt.mask)
if err != nil && !tt.shouldFail {
t.Errorf("expected to success, got %v", err)
}
if err == nil && tt.shouldFail {
t.Errorf("expected to fail, got %v", err)
}
if len(cells) != tt.expectedLen {
t.Errorf("expected %d cells, got %d", tt.expectedLen, len(cells))
}
if tt.expectedLen > 0 && tt.expectedLen%3 == 0 {
blobCount := 3
cellsPerBlob := tt.expectedLen / blobCount
for blobIdx := 0; blobIdx < blobCount; blobIdx++ {
startIdx := blobIdx * cellsPerBlob
endIdx := startIdx + cellsPerBlob
if endIdx > len(cells) {
t.Errorf("blob %d: expected cells up to index %d, but only have %d cells",
blobIdx, endIdx-1, len(cells))
}
}
}
})
}
}

16
core/txpool/blobpool/limbo.go
View file

@ -33,7 +33,7 @@ import (
type limboBlob struct {
TxHash common.Hash // Owner transaction's hash to support resurrecting reorged txs
Block uint64 // Block in which the blob transaction was included
Tx *types.Transaction
Tx *pooledBlobTx
}
// limbo is a light, indexed database to temporarily store recently included
@ -146,15 +146,15 @@ func (l *limbo) finalize(final *types.Header) {
// push stores a new blob transaction into the limbo, waiting until finality for
// it to be automatically evicted.
func (l *limbo) push(tx *types.Transaction, block uint64) error {
func (l *limbo) push(tx *pooledBlobTx, block uint64) error {
// If the blobs are already tracked by the limbo, consider it a programming
// error. There's not much to do against it, but be loud.
if _, ok := l.index[tx.Hash()]; ok {
log.Error("Limbo cannot push already tracked blobs", "tx", tx.Hash())
if _, ok := l.index[tx.Transaction.Hash()]; ok {
log.Error("Limbo cannot push already tracked blobs", "tx", tx.Transaction.Hash())
return errors.New("already tracked blob transaction")
}
if err := l.setAndIndex(tx, block); err != nil {
log.Error("Failed to set and index limboed blobs", "tx", tx.Hash(), "err", err)
log.Error("Failed to set and index limboed blobs", "tx", tx.Transaction.Hash(), "err", err)
return err
}
return nil
@ -163,7 +163,7 @@ func (l *limbo) push(tx *types.Transaction, block uint64) error {
// pull retrieves a previously pushed set of blobs back from the limbo, removing
// it at the same time. This method should be used when a previously included blob
// transaction gets reorged out.
func (l *limbo) pull(tx common.Hash) (*types.Transaction, error) {
func (l *limbo) pull(tx common.Hash) (*pooledBlobTx, error) {
// If the blobs are not tracked by the limbo, there's not much to do. This
// can happen for example if a blob transaction is mined without pushing it
// into the network first.
@ -240,8 +240,8 @@ func (l *limbo) getAndDrop(id uint64) (*limboBlob, error) {
// setAndIndex assembles a limbo blob database entry and stores it, also updating
// the in-memory indices.
func (l *limbo) setAndIndex(tx *types.Transaction, block uint64) error {
txhash := tx.Hash()
func (l *limbo) setAndIndex(tx *pooledBlobTx, block uint64) error {
txhash := tx.Transaction.Hash()
item := &limboBlob{
TxHash: txhash,
Block: block,

11
core/txpool/blobpool/lookup.go
View file

@ -18,11 +18,13 @@ package blobpool
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
type txMetadata struct {
id uint64 // the billy id of transction
size uint64 // the RLP encoded size of transaction (blobs are included)
id uint64 // the billy id of transction
size uint64 // the RLP encoded size of transaction (blobs are included)
custody types.CustodyBitmap
}
// lookup maps blob versioned hashes to transaction hashes that include them,
@ -91,8 +93,9 @@ func (l *lookup) track(tx *blobTxMeta) {
}
// Map the transaction hash to the datastore id and RLP-encoded transaction size
l.txIndex[tx.hash] = &txMetadata{
id: tx.id,
size: tx.size,
id: tx.id,
size: tx.size,
custody: *tx.custody,
}
}

2
core/txpool/legacypool/legacypool.go
View file

@ -1011,7 +1011,7 @@ func (pool *LegacyPool) get(hash common.Hash) *types.Transaction {
}
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
func (pool *LegacyPool) GetRLP(hash common.Hash) []byte {
func (pool *LegacyPool) GetRLP(hash common.Hash, _ bool) []byte {
tx := pool.all.Get(hash)
if tx == nil {
return nil

3
core/txpool/subpool.go
View file

@ -132,7 +132,8 @@ type SubPool interface {
Get(hash common.Hash) *types.Transaction
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
GetRLP(hash common.Hash) []byte
// If includeBlob is false, blob data is stripped from blob transactions (ETH/71).
GetRLP(hash common.Hash, includeBlob bool) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.

4
core/txpool/txpool.go
View file

@ -284,9 +284,9 @@ func (p *TxPool) Get(hash common.Hash) *types.Transaction {
}
// GetRLP returns a RLP-encoded transaction if it is contained in the pool.
func (p *TxPool) GetRLP(hash common.Hash) []byte {
func (p *TxPool) GetRLP(hash common.Hash, includeBlob bool) []byte {
for _, subpool := range p.subpools {
encoded := subpool.GetRLP(hash)
encoded := subpool.GetRLP(hash, includeBlob)
if len(encoded) != 0 {
return encoded
}

85
core/txpool/validation.go
View file

@ -64,9 +64,6 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
if opts.Accept&(1<<tx.Type()) == 0 {
return fmt.Errorf("%w: tx type %v not supported by this pool", core.ErrTxTypeNotSupported, tx.Type())
}
if blobCount := len(tx.BlobHashes()); blobCount > opts.MaxBlobCount {
return fmt.Errorf("%w: blob count %v, limit %v", ErrTxBlobLimitExceeded, blobCount, opts.MaxBlobCount)
}
// Before performing any expensive validations, sanity check that the tx is
// smaller than the maximum limit the pool can meaningfully handle
if tx.Size() > opts.MaxSize {
@ -146,9 +143,6 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
if tx.GasTipCapIntCmp(opts.MinTip) < 0 {
return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.GasTipCap(), opts.MinTip)
}
if tx.Type() == types.BlobTxType {
return validateBlobTx(tx, head, opts)
}
if tx.Type() == types.SetCodeTxType {
if len(tx.SetCodeAuthorizations()) == 0 {
return errors.New("set code tx must have at least one authorization tuple")
@ -157,14 +151,33 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
return nil
}
// validateBlobTx implements the blob-transaction specific validations.
func validateBlobTx(tx *types.Transaction, head *types.Header, opts *ValidationOptions) error {
sidecar := tx.BlobTxSidecar()
if sidecar == nil {
return errors.New("missing sidecar in blob transaction")
func ValidateBlobSidecar(tx *types.Transaction, sidecar *types.BlobTxCellSidecar, head *types.Header, opts *ValidationOptions) error {
if sidecar.Custody.OneCount() == 0 {
return errors.New("blobless blob transaction")
}
// Ensure the sidecar is constructed with the correct version, consistent
// with the current fork.
// Ensure the blob fee cap satisfies the minimum blob gas price
if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
}
// Verify whether the blob count is consistent with other parts of the sidecar and the transaction
blobCount := len(sidecar.Cells) / sidecar.Custody.OneCount()
hashes := tx.BlobHashes()
if blobCount == 0 {
return errors.New("blobless blob transaction")
}
if blobCount != len(sidecar.Commitments) || blobCount != len(hashes) {
return fmt.Errorf("invalid number of %d blobs compared to %d commitments and %d blob hashes", blobCount, len(sidecar.Commitments), len(tx.BlobHashes()))
}
// Check whether the blob count does not exceed the max blob count
if blobCount > opts.MaxBlobCount {
return fmt.Errorf("%w: blob count %v, limit %v", ErrTxBlobLimitExceeded, blobCount, opts.MaxBlobCount)
}
if err := sidecar.ValidateBlobCommitmentHashes(hashes); err != nil {
return err
}
// Ensure the sidecar version is correct for the current fork (master: bd77b77ed)
version := types.BlobSidecarVersion0
if opts.Config.IsOsaka(head.Number, head.Time) {
version = types.BlobSidecarVersion1
@ -172,50 +185,42 @@ func validateBlobTx(tx *types.Transaction, head *types.Header, opts *ValidationO
if sidecar.Version != version {
return fmt.Errorf("unexpected sidecar version, want: %d, got: %d", version, sidecar.Version)
}
// Ensure the blob fee cap satisfies the minimum blob gas price
if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
}
// Ensure the number of items in the blob transaction and various side
// data match up before doing any expensive validations
hashes := tx.BlobHashes()
if len(hashes) == 0 {
return errors.New("blobless blob transaction")
}
if len(hashes) > params.BlobTxMaxBlobs {
return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.BlobTxMaxBlobs)
}
if len(sidecar.Blobs) != len(hashes) {
return fmt.Errorf("invalid number of %d blobs compared to %d blob hashes", len(sidecar.Blobs), len(hashes))
}
if err := sidecar.ValidateBlobCommitmentHashes(hashes); err != nil {
return err
}
// Fork-specific sidecar checks, including proof verification.
if sidecar.Version == types.BlobSidecarVersion1 {
return validateBlobSidecarOsaka(sidecar, hashes)
} else {
return validateBlobSidecarLegacy(sidecar, hashes)
}
return validateBlobSidecarLegacy(sidecar, hashes)
}
func validateBlobSidecarLegacy(sidecar *types.BlobTxSidecar, hashes []common.Hash) error {
func validateBlobSidecarLegacy(sidecar *types.BlobTxCellSidecar, hashes []common.Hash) error {
if len(sidecar.Proofs) != len(hashes) {
return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes))
}
for i := range sidecar.Blobs {
if err := kzg4844.VerifyBlobProof(&sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil {
return fmt.Errorf("%w: invalid blob proof: %v", ErrKZGVerificationError, err)
blobs, err := kzg4844.RecoverBlobs(sidecar.Cells, sidecar.Custody.Indices())
if err != nil {
return fmt.Errorf("%w: %v", ErrKZGVerificationError, err)
}
for i := range blobs {
if err := kzg4844.VerifyBlobProof(&blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil {
return fmt.Errorf("%w: invalid blob %d: %v", ErrKZGVerificationError, i, err)
}
}
return nil
}
func validateBlobSidecarOsaka(sidecar *types.BlobTxSidecar, hashes []common.Hash) error {
func validateBlobSidecarOsaka(sidecar *types.BlobTxCellSidecar, hashes []common.Hash) error {
if len(sidecar.Proofs) != len(hashes)*kzg4844.CellProofsPerBlob {
return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes)*kzg4844.CellProofsPerBlob)
}
if err := kzg4844.VerifyCellProofs(sidecar.Blobs, sidecar.Commitments, sidecar.Proofs); err != nil {
indices := sidecar.Custody.Indices()
cellProofs := make([]kzg4844.Proof, 0)
for blobIdx := range len(sidecar.Commitments) {
for _, proofIdx := range indices {
idx := blobIdx*kzg4844.CellProofsPerBlob + int(proofIdx)
cellProofs = append(cellProofs, sidecar.Proofs[idx])
}
}
if err := kzg4844.VerifyCells(sidecar.Cells, sidecar.Commitments, cellProofs, sidecar.Custody.Indices()); err != nil {
return fmt.Errorf("%w: %v", ErrKZGVerificationError, err)
}
return nil

132
core/types/custody_bitmap.go Normal file
View file

@ -0,0 +1,132 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package types
import (
"math/bits"
"math/rand"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
// `CustodyBitmap` is a bitmap to represent which custody index to store (little endian)
type CustodyBitmap [16]byte
var (
CustodyBitmapAll = func() *CustodyBitmap {
var result CustodyBitmap
for i := range result {
result[i] = 0xFF
}
return &result
}()
CustodyBitmapData = func() *CustodyBitmap {
var result CustodyBitmap
for i := 0; i < kzg4844.DataPerBlob/8; i++ {
result[i] = 0xFF
}
return &result
}()
)
func NewCustodyBitmap(indices []uint64) CustodyBitmap {
var result CustodyBitmap
for _, i := range indices {
if i >= uint64(kzg4844.CellsPerBlob) {
panic("CustodyBitmap: bit index out of range")
}
result[i/8] |= 1 << (i % 8)
}
return result
}
// NewRandomCustodyBitmap creates a CustodyBitmap with n randomly selected indices.
// This should be used only for tests.
func NewRandomCustodyBitmap(n int) CustodyBitmap {
if n <= 0 || n > kzg4844.CellsPerBlob {
panic("CustodyBitmap: invalid number of indices")
}
indices := make([]uint64, 0, n)
used := make(map[uint64]bool)
for len(indices) < n {
idx := uint64(rand.Intn(kzg4844.CellsPerBlob))
if !used[idx] {
used[idx] = true
indices = append(indices, idx)
}
}
return NewCustodyBitmap(indices)
}
// IsSet returns whether bit i is set.
func (b CustodyBitmap) IsSet(i uint64) bool {
if i >= uint64(kzg4844.CellsPerBlob) {
return false
}
return (b[i/8]>>(i%8))&1 == 1
}
// OneCount returns the number of bits set to 1.
func (b CustodyBitmap) OneCount() int {
total := 0
for _, v := range b {
total += bits.OnesCount8(v)
}
return total
}
// Indices returns the bit positions set to 1, in ascending order.
func (b CustodyBitmap) Indices() []uint64 {
out := make([]uint64, 0, b.OneCount())
for byteIdx, val := range b {
v := val
for v != 0 {
tz := bits.TrailingZeros8(v)
out = append(out, uint64(byteIdx*8+tz))
v &^= 1 << tz
}
}
return out
}
// Difference returns b AND NOT set (bits in b but not in set).
func (b CustodyBitmap) Difference(set *CustodyBitmap) *CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] &^ set[i]
}
return &out
}
// Intersection returns b AND set.
func (b CustodyBitmap) Intersection(set *CustodyBitmap) *CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] & set[i]
}
return &out
}
// Union returns b OR set.
func (b CustodyBitmap) Union(set *CustodyBitmap) *CustodyBitmap {
var out CustodyBitmap
for i := range b {
out[i] = b[i] | set[i]
}
return &out
}

12
core/types/transaction.go
View file

@ -510,6 +510,18 @@ func (tx *Transaction) WithoutBlobTxSidecar() *Transaction {
return cpy
}
// WithoutBlob returns a copy of tx with the blob data removed from the sidecar,
// keeping commitments, proofs and other metadata intact.
func (tx *Transaction) WithoutBlob() *Transaction {
blobtx, ok := tx.inner.(*BlobTx)
if !ok || blobtx.Sidecar == nil {
return tx
}
sidecarWithoutBlob := blobtx.Sidecar.Copy()
sidecarWithoutBlob.Blobs = nil
return tx.WithBlobTxSidecar(sidecarWithoutBlob)
}
// WithBlobTxSidecar returns a copy of tx with the blob sidecar added.
func (tx *Transaction) WithBlobTxSidecar(sideCar *BlobTxSidecar) *Transaction {
blobtx, ok := tx.inner.(*BlobTx)

31
core/types/tx_blob.go
View file

@ -176,6 +176,37 @@ func (sc *BlobTxSidecar) Copy() *BlobTxSidecar {
}
}
func (sc *BlobTxSidecar) ToBlobTxCellSidecar() (*BlobTxCellSidecar, error) {
cells, err := kzg4844.ComputeCells(sc.Blobs)
if err != nil {
return nil, err
}
return &BlobTxCellSidecar{
Version: sc.Version,
Cells: cells,
Commitments: sc.Commitments,
Proofs: sc.Proofs,
Custody: *CustodyBitmapAll,
}, nil
}
type BlobTxCellSidecar struct {
Version byte
Cells []kzg4844.Cell
Commitments []kzg4844.Commitment
Proofs []kzg4844.Proof
Custody CustodyBitmap
}
// ValidateBlobCommitmentHashes checks whether the given hashes correspond to the
// commitments in the sidecar
func (c *BlobTxCellSidecar) ValidateBlobCommitmentHashes(hashes []common.Hash) error {
sc := BlobTxSidecar{
Commitments: c.Commitments,
}
return sc.ValidateBlobCommitmentHashes(hashes)
}
// blobTxWithBlobs represents blob tx with its corresponding sidecar.
// This is an interface because sidecars are versioned.
type blobTxWithBlobs interface {

44
crypto/kzg4844/kzg4844.go
View file

@ -34,9 +34,27 @@ var (
blobT = reflect.TypeFor[Blob]()
commitmentT = reflect.TypeFor[Commitment]()
proofT = reflect.TypeFor[Proof]()
cellT = reflect.TypeFor[Cell]()
)
const CellProofsPerBlob = 128
const (
CellProofsPerBlob = 128
CellsPerBlob = 128
DataPerBlob = 64
)
// Cell represents a single cell in a blob.
type Cell [2048]byte
// UnmarshalJSON parses a cell in hex syntax.
func (c *Cell) UnmarshalJSON(input []byte) error {
return hexutil.UnmarshalFixedJSON(cellT, input, c[:])
}
// MarshalText returns the hex representation of c.
func (c *Cell) MarshalText() ([]byte, error) {
return hexutil.Bytes(c[:]).MarshalText()
}
// Blob represents a 4844 data blob.
type Blob [131072]byte
@ -189,3 +207,27 @@ func CalcBlobHashV1(hasher hash.Hash, commit *Commitment) (vh [32]byte) {
func IsValidVersionedHash(h []byte) bool {
return len(h) == 32 && h[0] == 0x01
}
// VerifyCells verifies a batch of proofs corresponding to the cells and commitments.
func VerifyCells(cells []Cell, commitments []Commitment, proofs []Proof, cellIndices []uint64) error {
if useCKZG.Load() {
return ckzgVerifyCells(cells, commitments, proofs, cellIndices)
}
return gokzgVerifyCells(cells, commitments, proofs, cellIndices)
}
// ComputeCells computes the cells from the given blobs.
func ComputeCells(blobs []Blob) ([]Cell, error) {
if useCKZG.Load() {
return ckzgComputeCells(blobs)
}
return gokzgComputeCells(blobs)
}
// RecoverBlobs recovers blobs from the given cells and cell indices.
func RecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
if useCKZG.Load() {
return ckzgRecoverBlobs(cells, cellIndices)
}
return gokzgRecoverBlobs(cells, cellIndices)
}

87
crypto/kzg4844/kzg4844_ckzg_cgo.go
View file

@ -190,3 +190,90 @@ func ckzgVerifyCellProofBatch(blobs []Blob, commitments []Commitment, cellProofs
}
return nil
}
func ckzgVerifyCells(cells []Cell, commitments []Commitment, cellProofs []Proof, cellIndices []uint64) error {
ckzgIniter.Do(ckzgInit)
var (
proofs = make([]ckzg4844.Bytes48, len(cellProofs))
commits = make([]ckzg4844.Bytes48, 0, len(cellProofs))
indices = make([]uint64, 0, len(cellProofs))
kzgcells = make([]ckzg4844.Cell, 0, len(cellProofs))
)
for i := range cellProofs {
proofs[i] = (ckzg4844.Bytes48)(cellProofs[i])
kzgcells = append(kzgcells, (ckzg4844.Cell)(cells[i]))
}
if len(cellProofs)%len(commitments) != 0 {
return errors.New("wrong cell proofs and commitments length")
}
cellCounts := len(cellProofs) / len(commitments)
for _, commitment := range commitments {
for j := 0; j < cellCounts; j++ {
commits = append(commits, (ckzg4844.Bytes48)(commitment))
}
}
blobCounts := len(cellProofs) / len(cellIndices)
for j := 0; j < blobCounts; j++ {
indices = append(indices, cellIndices...)
}
valid, err := ckzg4844.VerifyCellKZGProofBatch(commits, indices, kzgcells, proofs)
if err != nil {
return err
}
if !valid {
return errors.New("invalid proof")
}
return nil
}
func ckzgComputeCells(blobs []Blob) ([]Cell, error) {
ckzgIniter.Do(ckzgInit)
cells := make([]Cell, 0, ckzg4844.CellsPerExtBlob*len(blobs))
for i := range blobs {
cellsI, err := ckzg4844.ComputeCells((*ckzg4844.Blob)(&blobs[i]))
if err != nil {
return []Cell{}, err
}
for _, c := range cellsI {
cells = append(cells, Cell(c))
}
}
return cells, nil
}
func ckzgRecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
ckzgIniter.Do(ckzgInit)
if len(cellIndices) == 0 || len(cells)%len(cellIndices) != 0 {
return []Blob{}, errors.New("cells with wrong length")
}
blobCount := len(cells) / len(cellIndices)
blobs := make([]Blob, 0, blobCount)
offset := 0
for range blobCount {
kzgcells := make([]ckzg4844.Cell, 0, len(cellIndices))
for _, cell := range cells[offset : offset+len(cellIndices)] {
kzgcells = append(kzgcells, ckzg4844.Cell(cell))
}
extCells, err := ckzg4844.RecoverCells(cellIndices, kzgcells)
if err != nil {
return []Blob{}, err
}
var blob Blob
for i, cell := range extCells[:64] {
copy(blob[i*len(cell):], cell[:])
}
blobs = append(blobs, blob)
offset = offset + len(cellIndices)
}
return blobs, nil
}

12
crypto/kzg4844/kzg4844_ckzg_nocgo.go
View file

@ -73,3 +73,15 @@ func ckzgVerifyCellProofBatch(blobs []Blob, commitments []Commitment, proof []Pr
func ckzgComputeCellProofs(blob *Blob) ([]Proof, error) {
panic("unsupported platform")
}
func ckzgVerifyCells(cells []Cell, commitments []Commitment, cellProofs []Proof, cellIndices []uint64) error {
panic("unsupported platform")
}
func ckzgComputeCells(blobs []Blob) ([]Cell, error) {
panic("unsupported platform")
}
func ckzgRecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
panic("unsupported platform")
}

90
crypto/kzg4844/kzg4844_gokzg.go
View file

@ -18,6 +18,7 @@ package kzg4844
import (
"encoding/json"
"errors"
"sync"
gokzg4844 "github.com/crate-crypto/go-eth-kzg"
@ -148,3 +149,92 @@ func gokzgVerifyCellProofBatch(blobs []Blob, commitments []Commitment, cellProof
}
return context.VerifyCellKZGProofBatch(commits, cellIndices, cells[:], proofs)
}
// gokzgVerifyCells verifies that the cell data corresponds to the provided commitment.
func gokzgVerifyCells(cells []Cell, commitments []Commitment, cellProofs []Proof, cellIndices []uint64) error {
gokzgIniter.Do(gokzgInit)
var (
proofs = make([]gokzg4844.KZGProof, len(cellProofs))
commits = make([]gokzg4844.KZGCommitment, 0, len(cellProofs))
indices = make([]uint64, 0, len(cellProofs))
kzgcells = make([]*gokzg4844.Cell, 0, len(cellProofs))
)
// Copy over the cell proofs and cells
for i := range cellProofs {
proofs[i] = gokzg4844.KZGProof(cellProofs[i])
gc := gokzg4844.Cell(cells[i])
kzgcells = append(kzgcells, &gc)
}
if len(cellProofs)%len(commitments) != 0 {
return errors.New("wrong cell proofs and commitments length")
}
cellCounts := len(cellProofs) / len(commitments)
// Blow up the commitments to be the same length as the proofs
for _, commitment := range commitments {
for j := 0; j < cellCounts; j++ {
commits = append(commits, gokzg4844.KZGCommitment(commitment))
}
}
blobCounts := len(cellProofs) / len(cellIndices)
for j := 0; j < blobCounts; j++ {
indices = append(indices, cellIndices...)
}
return context.VerifyCellKZGProofBatch(commits, indices, kzgcells, proofs)
}
// gokzgComputeCells computes cells from blobs.
func gokzgComputeCells(blobs []Blob) ([]Cell, error) {
gokzgIniter.Do(gokzgInit)
cells := make([]Cell, 0, gokzg4844.CellsPerExtBlob*len(blobs))
for i := range blobs {
cellsI, err := context.ComputeCells((*gokzg4844.Blob)(&blobs[i]), 2)
if err != nil {
return []Cell{}, err
}
for _, c := range cellsI {
if c != nil {
cells = append(cells, Cell(*c))
}
}
}
return cells, nil
}
// gokzgRecoverBlobs recovers blobs from cells and cell indices.
func gokzgRecoverBlobs(cells []Cell, cellIndices []uint64) ([]Blob, error) {
gokzgIniter.Do(gokzgInit)
if len(cellIndices) == 0 || len(cells)%len(cellIndices) != 0 {
return []Blob{}, errors.New("cells with wrong length")
}
blobCount := len(cells) / len(cellIndices)
blobs := make([]Blob, 0, blobCount)
offset := 0
for range blobCount {
kzgcells := make([]*gokzg4844.Cell, 0, len(cellIndices))
for _, cell := range cells[offset : offset+len(cellIndices)] {
gc := gokzg4844.Cell(cell)
kzgcells = append(kzgcells, &gc)
}
extCells, err := context.RecoverCells(cellIndices, kzgcells, 2)
if err != nil {
return []Blob{}, err
}
var blob Blob
for i, cell := range extCells[:64] {
copy(blob[i*len(cell):], cell[:])
}
blobs = append(blobs, blob)
offset = offset + len(cellIndices)
}
return blobs, nil
}

116
crypto/kzg4844/kzg4844_test.go
View file

@ -253,3 +253,119 @@ func benchmarkComputeCellProofs(b *testing.B, ckzg bool) {
}
}
}
func TestCKZGVerifyPartialCells(t *testing.T) { testVerifyPartialCells(t, true) }
func TestGoKZGVerifyPartialCells(t *testing.T) { testVerifyPartialCells(t, false) }
func testVerifyPartialCells(t *testing.T, ckzg bool) {
if ckzg && !ckzgAvailable {
t.Skip("CKZG unavailable in this test build")
}
defer func(old bool) { useCKZG.Store(old) }(useCKZG.Load())
useCKZG.Store(ckzg)
const blobCount = 3
var blobs []*Blob
var commitments []Commitment
for range blobCount {
blob := randBlob()
commitment, err := BlobToCommitment(blob)
if err != nil {
t.Fatalf("failed to commit blob: %v", err)
}
blobs = append(blobs, blob)
commitments = append(commitments, commitment)
}
var (
partialCells []Cell
partialProofs []Proof
commits []Commitment
indices []uint64
)
for bi, blob := range blobs {
proofs, err := ComputeCellProofs(blob)
if err != nil {
t.Fatalf("failed to compute cell proofs: %v", err)
}
cells, err := ComputeCells([]Blob{*blob})
if err != nil {
t.Fatalf("failed to compute cells: %v", err)
}
commits = append(commits, commitments[bi])
// sample 0, 31, 63, 95 cells
step := len(cells) / 4
indices = []uint64{0, uint64(step - 1), uint64(2*step - 1), uint64(3*step - 1)}
for _, idx := range indices {
partialCells = append(partialCells, cells[idx])
partialProofs = append(partialProofs, proofs[idx])
}
}
if err := VerifyCells(partialCells, commits, partialProofs, indices); err != nil {
t.Fatalf("failed to verify partial cell proofs: %v", err)
}
}
func TestCKZGRecoverBlob(t *testing.T) { testRecoverBlob(t, true) }
func TestGoKZGRecoverBlob(t *testing.T) { testRecoverBlob(t, false) }
func testRecoverBlob(t *testing.T, ckzg bool) {
if ckzg && !ckzgAvailable {
t.Skip("CKZG unavailable in this test build")
}
defer func(old bool) { useCKZG.Store(old) }(useCKZG.Load())
useCKZG.Store(ckzg)
blobs := []Blob{}
blobs = append(blobs, *randBlob())
blobs = append(blobs, *randBlob())
blobs = append(blobs, *randBlob())
cells, err := ComputeCells(blobs)
if err != nil {
t.Fatalf("failed to compute cells: %v", err)
}
proofs := make([]Proof, 0)
commitments := make([]Commitment, len(blobs))
for i, blob := range blobs {
proof, err := ComputeCellProofs(&blob)
if err != nil {
t.Fatalf("failed to compute proof: %v", err)
}
proofs = append(proofs, proof...)
commitment, err := BlobToCommitment(&blob)
if err != nil {
t.Fatalf("failed to compute commitment: %v", err)
}
commitments[i] = commitment
}
var (
partialCells []Cell
indices []uint64
)
for ci := 64; ci < 128; ci++ {
indices = append(indices, uint64(ci))
}
for i := 0; i < len(cells); i += 128 {
start := i + 64
end := i + 128
partialCells = append(partialCells, cells[start:end]...)
}
recoverBlobs, err := RecoverBlobs(partialCells, indices)
if err != nil {
t.Fatalf("failed to recover blob: %v", err)
}
if err := VerifyCellProofs(recoverBlobs, commitments, proofs); err != nil {
t.Fatalf("failed to verify recovered blob: %v", err)
}
}

4
eth/backend.go
View file

@ -43,6 +43,7 @@ import (
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/fetcher"
"github.com/ethereum/go-ethereum/eth/gasprice"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/eth/protocols/snap"
@ -335,11 +336,13 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
Database: chainDb,
Chain: eth.blockchain,
TxPool: eth.txPool,
BlobPool: eth.blobTxPool,
Network: networkID,
Sync: config.SyncMode,
BloomCache: uint64(cacheLimit),
EventMux: eth.eventMux,
RequiredBlocks: config.RequiredBlocks,
Custody: *types.CustodyBitmapAll,
}); err != nil {
return nil, err
}
@ -423,6 +426,7 @@ func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager
func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain }
func (s *Ethereum) TxPool() *txpool.TxPool { return s.txPool }
func (s *Ethereum) BlobTxPool() *blobpool.BlobPool { return s.blobTxPool }
func (s *Ethereum) BlobFetcher() *fetcher.BlobFetcher { return s.handler.blobFetcher }
func (s *Ethereum) Engine() consensus.Engine { return s.engine }
func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb }
func (s *Ethereum) IsListening() bool { return true } // Always listening

5
eth/catalyst/api.go
View file

@ -1163,6 +1163,11 @@ func (api *ConsensusAPI) getBodiesByRange(start, count hexutil.Uint64) ([]*engin
return bodies, nil
}
func (api *ConsensusAPI) BlobCustodyUpdatedV1(custodyColumns []uint64) {
bitmap := types.NewCustodyBitmap(custodyColumns)
api.eth.BlobFetcher().UpdateCustody(bitmap)
}
func getBody(block *types.Block) *engine.ExecutionPayloadBody {
if block == nil {
return nil

16
eth/catalyst/api_test.go
View file

@ -1911,7 +1911,12 @@ func newGetBlobEnv(t testing.TB, version byte) (*node.Node, *ConsensusAPI) {
tx1 := makeMultiBlobTx(&config, 0, 2, 0, key1, version) // blob[0, 2)
tx2 := makeMultiBlobTx(&config, 0, 2, 2, key2, version) // blob[2, 4)
tx3 := makeMultiBlobTx(&config, 0, 2, 4, key3, version) // blob[4, 6)
ethServ.TxPool().Add([]*types.Transaction{tx1, tx2, tx3}, true)
errs := ethServ.TxPool().Add([]*types.Transaction{tx1, tx2, tx3}, true)
for i, err := range errs {
if err != nil {
t.Logf("Add tx %d failed: %v", i, err)
}
}
api := newConsensusAPIWithoutHeartbeat(ethServ)
return n, api
@ -2108,6 +2113,15 @@ func runGetBlobs(t testing.TB, getBlobs getBlobsFn, start, limit int, fillRandom
}
}
if !reflect.DeepEqual(result, expect) {
t.Logf("result len=%d, expect len=%d", len(result), len(expect))
if len(result) > 0 && result[0] != nil && len(expect) > 0 && expect[0] != nil {
t.Logf("result[0].Blob len=%d, expect[0].Blob len=%d", len(result[0].Blob), len(expect[0].Blob))
t.Logf("result[0].CellProofs len=%d, expect[0].CellProofs len=%d", len(result[0].CellProofs), len(expect[0].CellProofs))
t.Logf("result[0].Blob == expect[0].Blob: %v", reflect.DeepEqual(result[0].Blob, expect[0].Blob))
t.Logf("result[0].CellProofs == expect[0].CellProofs: %v", reflect.DeepEqual(result[0].CellProofs, expect[0].CellProofs))
} else {
t.Logf("result[0]=%v, expect[0]=%v", result, expect)
}
t.Fatalf("Unexpected result for case %s", name)
}
}

784
eth/fetcher/blob_fetcher.go Normal file
View file

@ -0,0 +1,784 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package fetcher
import (
"math/rand"
"slices"
"sort"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
// todo remove partial / full
type random interface {
Intn(n int) int
}
// BlobFetcher fetches blobs of new type-3 transactions with probability p,
// and for the remaining (1-p) transactions, it performs availability checks.
// For availability checks, it fetches cells from each blob in the transaction
// according to the custody cell indices provided by the consensus client
// connected to this execution client.
var blobFetchTimeout = 5 * time.Second
// todo tuning
const (
availabilityThreshold = 2
maxPayloadRetrievals = 128
maxPayloadAnnounces = 4096
MAX_CELLS_PER_PARTIAL_REQUEST = 8
blobAvailabilityTimeout = 500 * time.Millisecond
)
type blobTxAnnounce struct {
origin string // Identifier of the peer that sent the announcement
txs []common.Hash // Hashes of transactions announced
cells types.CustodyBitmap // Custody information of transactions being announced
}
type cellRequest struct {
txs []common.Hash // Transactions that have been requested for their cells
cells *types.CustodyBitmap // Requested cell indices
time mclock.AbsTime // Timestamp when the request was made
}
type payloadDelivery struct {
origin string // Peer from which the payloads were delivered
txs []common.Hash // Hashes of transactions that were delivered
cells [][]kzg4844.Cell
cellBitmap *types.CustodyBitmap
}
type cellWithSeq struct {
seq uint64
cells *types.CustodyBitmap
}
type fetchStatus struct {
fetching *types.CustodyBitmap // To avoid fetching cells which had already been fetched / currently being fetched
fetched []uint64 // To sort cells
cells []kzg4844.Cell
}
// BlobFetcher is responsible for managing type 3 transactions based on peer announcements.
//
// BlobFetcher manages three buffers:
// - Transactions not to be fetched are moved to "waitlist"
// if a payload(blob) seems to be possessed by D(threshold) other peers, request custody cells for that.
// Accept it when the cells are received. Otherwise, it is dropped.
// - Transactions queued to be fetched are moved to "announces"
// if a payload is received, it is added to the blob pool. Otherwise, the transaction is dropped.
// - Transactions to be fetched are moved to "fetching"
// if a payload/cell announcement is received during fetch, the peer is recorded as an alternate source.
type BlobFetcher struct {
notify chan *blobTxAnnounce
cleanup chan *payloadDelivery
drop chan *txDrop
quit chan struct{}
custody *types.CustodyBitmap
txSeq uint64 // To make transactions fetched in arrival order
full map[common.Hash]struct{}
partial map[common.Hash]struct{}
// Buffer 1: Set of blob txs whose blob data is waiting for availability confirmation (not pull decision)
waitlist map[common.Hash]map[string]struct{} // Peer set that announced blob availability
waittime map[common.Hash]mclock.AbsTime // Timestamp when added to waitlist
waitslots map[string]map[common.Hash]struct{} // Waiting announcements grouped by peer (DoS protection)
// waitSlots should also include announcements with partial cells
// Buffer 2: Transactions queued for fetching (pull decision + not pull decision)
// "announces" is shared with stage 3, for DoS protection
announces map[string]map[common.Hash]*cellWithSeq // Set of announced transactions, grouped by origin peer
// Buffer 2
// Stage 3: Transactions whose payloads/cells are currently being fetched (pull decision + not pull decision)
fetches map[common.Hash]*fetchStatus // Hash -> Bitmap, in-flight transaction cells
requests map[string][]*cellRequest // In-flight transaction retrievals
// todo simplify
alternates map[common.Hash]map[string]*types.CustodyBitmap // In-flight transaction alternate origins (in case the peer is dropped)
// Callbacks
validateCells func([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error
addPayload func([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error
fetchPayloads func(string, []common.Hash, *types.CustodyBitmap) error
dropPeer func(string)
step chan struct{} // Notification channel when the fetcher loop iterates
clock mclock.Clock // Monotonic clock or simulated clock for tests
realTime func() time.Time // Real system time or simulated time for tests
rand random // Randomizer
}
func NewBlobFetcher(
validateCells func([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error,
addPayload func([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error,
fetchPayloads func(string, []common.Hash, *types.CustodyBitmap) error, dropPeer func(string),
custody *types.CustodyBitmap, rand random) *BlobFetcher {
return &BlobFetcher{
notify: make(chan *blobTxAnnounce),
cleanup: make(chan *payloadDelivery),
drop: make(chan *txDrop),
quit: make(chan struct{}),
full: make(map[common.Hash]struct{}),
partial: make(map[common.Hash]struct{}),
waitlist: make(map[common.Hash]map[string]struct{}),
waittime: make(map[common.Hash]mclock.AbsTime),
waitslots: make(map[string]map[common.Hash]struct{}),
announces: make(map[string]map[common.Hash]*cellWithSeq),
fetches: make(map[common.Hash]*fetchStatus),
requests: make(map[string][]*cellRequest),
alternates: make(map[common.Hash]map[string]*types.CustodyBitmap),
validateCells: validateCells,
addPayload: addPayload,
fetchPayloads: fetchPayloads,
dropPeer: dropPeer,
custody: custody,
clock: mclock.System{},
realTime: time.Now,
rand: rand,
}
}
// Notify is called when a Type 3 transaction is observed on the network. (TransactionPacket / NewPooledTransactionHashesPacket)
func (f *BlobFetcher) Notify(peer string, txs []common.Hash, cells types.CustodyBitmap) error {
blobAnnounce := &blobTxAnnounce{origin: peer, txs: txs, cells: cells}
select {
case f.notify <- blobAnnounce:
return nil
case <-f.quit:
return errTerminated
}
}
// Enqueue inserts a batch of received blob payloads into the blob pool.
// This is triggered by ethHandler upon receiving direct request responses.
func (f *BlobFetcher) Enqueue(peer string, hashes []common.Hash, cells [][]kzg4844.Cell, cellBitmap types.CustodyBitmap) error {
blobReplyInMeter.Mark(int64(len(hashes)))
select {
case f.cleanup <- &payloadDelivery{origin: peer, txs: hashes, cells: cells, cellBitmap: &cellBitmap}:
case <-f.quit:
return errTerminated
}
return nil
}
func (f *BlobFetcher) Drop(peer string) error {
select {
case f.drop <- &txDrop{peer: peer}:
return nil
case <-f.quit:
return errTerminated
}
}
func (f *BlobFetcher) UpdateCustody(cells types.CustodyBitmap) {
f.custody = &cells
}
func (f *BlobFetcher) Start() {
go f.loop()
}
func (f *BlobFetcher) Stop() {
close(f.quit)
}
func (f *BlobFetcher) loop() {
var (
waitTimer = new(mclock.Timer) // Timer for waitlist (availability)
waitTrigger = make(chan struct{}, 1)
timeoutTimer = new(mclock.Timer) // Timer for payload fetch request
timeoutTrigger = make(chan struct{}, 1)
)
for {
select {
case ann := <-f.notify:
// Drop part of the announcements if too many have accumulated from that peer
// This prevents a peer from dominating the queue with txs without responding to the request
// todo maxPayloadAnnounces -> according to the number of blobs
used := len(f.waitslots[ann.origin]) + len(f.announces[ann.origin])
if used >= maxPayloadAnnounces {
// Already full
break
}
want := used + len(ann.txs)
if want >= maxPayloadAnnounces {
// drop part of announcements
ann.txs = ann.txs[:maxPayloadAnnounces-used]
}
var (
idleWait = len(f.waittime) == 0
_, oldPeer = f.announces[ann.origin]
nextSeq = func() uint64 {
seq := f.txSeq
f.txSeq++
return seq
}
reschedule = make(map[string]struct{})
)
for _, hash := range ann.txs {
if oldPeer && f.announces[ann.origin][hash] != nil {
// Ignore already announced information
// We also have to prevent reannouncement by changing cells field.
// Considering cell custody transition is notified in advance of its finalization by consensus client,
// there is no reason to reannounce cells, and it has to be prevented.
continue
}
// Decide full or partial request
if _, ok := f.full[hash]; !ok {
if _, ok := f.partial[hash]; !ok {
// Not decided yet
var randomValue int
if f.rand == nil {
randomValue = rand.Intn(100)
} else {
randomValue = f.rand.Intn(100)
}
if randomValue < 15 {
f.full[hash] = struct{}{}
} else {
f.partial[hash] = struct{}{}
// Register for availability check
f.waitlist[hash] = make(map[string]struct{})
f.waittime[hash] = f.clock.Now()
}
}
}
if _, ok := f.full[hash]; ok {
// 1) Decided to send full request of the tx
if ann.cells != *types.CustodyBitmapAll {
continue
}
if f.announces[ann.origin] == nil {
f.announces[ann.origin] = make(map[common.Hash]*cellWithSeq)
}
f.announces[ann.origin][hash] = &cellWithSeq{
cells: types.CustodyBitmapData,
seq: nextSeq(),
}
reschedule[ann.origin] = struct{}{}
continue
}
if _, ok := f.partial[hash]; ok {
// 2) Decided to send partial request of the tx
if f.waitlist[hash] != nil {
if ann.cells != *types.CustodyBitmapAll {
continue
}
// Transaction is at the stage of availability check
// Add the peer to the peer list with full availability (waitlist)
f.waitlist[hash][ann.origin] = struct{}{}
if waitslots := f.waitslots[ann.origin]; waitslots != nil {
waitslots[hash] = struct{}{}
} else {
f.waitslots[ann.origin] = map[common.Hash]struct{}{
hash: {},
}
}
if len(f.waitlist[hash]) >= availabilityThreshold {
for peer := range f.waitlist[hash] {
if f.announces[peer] == nil {
f.announces[peer] = make(map[common.Hash]*cellWithSeq)
}
f.announces[peer][hash] = &cellWithSeq{
cells: f.custody,
seq: nextSeq(),
}
delete(f.waitslots[peer], hash)
if len(f.waitslots[peer]) == 0 {
delete(f.waitslots, peer)
}
reschedule[peer] = struct{}{}
}
delete(f.waitlist, hash)
}
continue
}
if ann.cells.Intersection(f.custody).OneCount() == 0 {
// No custody overlapping
continue
}
if f.announces[ann.origin] == nil {
f.announces[ann.origin] = make(map[common.Hash]*cellWithSeq)
}
f.announces[ann.origin][hash] = &cellWithSeq{
cells: ann.cells.Intersection(f.custody),
seq: nextSeq(),
}
reschedule[ann.origin] = struct{}{}
}
}
// If a new item was added to the waitlist, schedule its timeout
if idleWait && len(f.waittime) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
// If this is a new peer and that peer sent transaction with payload flag,
// schedule transaction fetches from it
//todo
if !oldPeer && len(f.announces[ann.origin]) > 0 {
f.scheduleFetches(timeoutTimer, timeoutTrigger, reschedule)
}
case <-waitTrigger:
// At least one transaction's waiting time ran out, pop all expired ones
// and update the blobpool according to availability
// Availability failure case
for hash, instance := range f.waittime {
if time.Duration(f.clock.Now()-instance)+txGatherSlack > blobAvailabilityTimeout {
// Check if enough peers have announced availability
for peer := range f.waitlist[hash] {
delete(f.waitslots[peer], hash)
if len(f.waitslots[peer]) == 0 {
delete(f.waitslots, peer)
}
}
delete(f.waittime, hash)
delete(f.waitlist, hash)
}
}
// If transactions are still waiting for availability, reschedule the wait timer
if len(f.waittime) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
case <-timeoutTrigger:
// Clean up any expired retrievals and avoid re-requesting them from the
// same peer (either overloaded or malicious, useless in both cases).
// Update blobpool according to availability result.
for peer, requests := range f.requests {
newRequests := make([]*cellRequest, 0)
for _, req := range requests {
if time.Duration(f.clock.Now()-req.time)+txGatherSlack > blobFetchTimeout {
// Reschedule all timeout cells to alternate peers
for _, hash := range req.txs {
// Do not request the same tx from this peer
delete(f.announces[peer], hash)
delete(f.alternates[hash], peer)
// Allow other candidates to be requested these cells
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(req.cells)
// Drop cells if there is no alternate source to fetch cells from
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
delete(f.fetches, hash)
}
}
if len(f.announces[peer]) == 0 {
delete(f.announces, peer)
}
} else {
newRequests = append(newRequests, req)
}
}
f.requests[peer] = newRequests
if len(f.requests[peer]) == 0 {
delete(f.requests, peer)
}
}
// Schedule a new transaction retrieval
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
// Trigger timeout for new schedule
f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
case delivery := <-f.cleanup:
// Remove from announce
addedHashes := make([]common.Hash, 0)
addedCells := make([][]kzg4844.Cell, 0)
var requestId int
request := new(cellRequest)
for _, hash := range delivery.txs {
// Find the request
for i, req := range f.requests[delivery.origin] {
if slices.Contains(req.txs, hash) && *req.cells == *delivery.cellBitmap {
request = req
requestId = i
break
}
}
if request != nil {
break
}
}
if request == nil {
// peer sent cells not requested. ignore
break
}
for i, hash := range delivery.txs {
if !slices.Contains(request.txs, hash) {
// Unexpected hash, ignore
continue
}
// Update fetch status
f.fetches[hash].fetched = append(f.fetches[hash].fetched, delivery.cellBitmap.Indices()...)
f.fetches[hash].cells = append(f.fetches[hash].cells, delivery.cells[i]...)
// Update announces of this peer
delete(f.announces[delivery.origin], hash)
if len(f.announces[delivery.origin]) == 0 {
delete(f.announces, delivery.origin)
}
delete(f.alternates[hash], delivery.origin)
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
}
// Check whether the all required cells are fetched
completed := false
if _, ok := f.full[hash]; ok && len(f.fetches[hash].fetched) >= kzg4844.DataPerBlob {
completed = true
} else if _, ok := f.partial[hash]; ok {
fetched := make([]uint64, len(f.fetches[hash].fetched))
copy(fetched, f.fetches[hash].fetched)
slices.Sort(fetched)
custodyIndices := f.custody.Indices()
completed = slices.Equal(fetched, custodyIndices)
}
if completed {
addedHashes = append(addedHashes, hash)
fetchStatus := f.fetches[hash]
sort.Slice(fetchStatus.cells, func(i, j int) bool {
return fetchStatus.fetched[i] < fetchStatus.fetched[j]
})
addedCells = append(addedCells, fetchStatus.cells)
// remove announces from other peers
for peer, txset := range f.announces {
delete(txset, hash)
if len(txset) == 0 {
delete(f.announces, peer)
}
}
delete(f.alternates, hash)
delete(f.fetches, hash)
}
}
// Update mempool status for arrived hashes
if len(addedHashes) > 0 {
f.addPayload(addedHashes, addedCells, delivery.cellBitmap)
}
// Remove the request
f.requests[delivery.origin][requestId] = f.requests[delivery.origin][len(f.requests[delivery.origin])-1]
f.requests[delivery.origin] = f.requests[delivery.origin][:len(f.requests[delivery.origin])-1]
if len(f.requests[delivery.origin]) == 0 {
delete(f.requests, delivery.origin)
}
// Reschedule missing transactions in the request
// Anything not delivered should be re-scheduled (with or without
// this peer, depending on the response cutoff)
delivered := make(map[common.Hash]struct{})
for _, hash := range delivery.txs {
delivered[hash] = struct{}{}
}
cutoff := len(request.txs)
for i, hash := range request.txs {
if _, ok := delivered[hash]; ok {
cutoff = i
continue
}
}
// Reschedule missing hashes from alternates, not-fulfilled from alt+self
for i, hash := range request.txs {
if _, ok := delivered[hash]; !ok {
// Not delivered
if i < cutoff {
// Remove origin from candidate sources for partial responses
delete(f.alternates[hash], delivery.origin)
delete(f.announces[delivery.origin], hash)
if len(f.announces[delivery.origin]) == 0 {
delete(f.announces, delivery.origin)
}
}
// Mark cells deliverable by other peers
if f.fetches[hash] != nil {
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(delivery.cellBitmap)
}
}
}
// Something was delivered, try to reschedule requests
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too
case drop := <-f.drop:
// A peer was dropped, remove all traces of it
if _, ok := f.waitslots[drop.peer]; ok {
for hash := range f.waitslots[drop.peer] {
delete(f.waitlist[hash], drop.peer)
if len(f.waitlist[hash]) == 0 {
delete(f.waitlist, hash)
delete(f.waittime, hash)
}
}
delete(f.waitslots, drop.peer)
if len(f.waitlist) > 0 {
f.rescheduleWait(waitTimer, waitTrigger)
}
}
// Clean up general announcement tracking
if _, ok := f.announces[drop.peer]; ok {
for hash := range f.announces[drop.peer] {
delete(f.alternates[hash], drop.peer)
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
}
}
delete(f.announces, drop.peer)
}
delete(f.announces, drop.peer)
// Clean up any active requests
if request, ok := f.requests[drop.peer]; ok && len(request) != 0 {
for _, req := range request {
for _, hash := range req.txs {
// Undelivered hash, reschedule if there's an alternative origin available
f.fetches[hash].fetching = f.fetches[hash].fetching.Difference(req.cells)
delete(f.alternates[hash], drop.peer)
if len(f.alternates[hash]) == 0 {
delete(f.alternates, hash)
delete(f.fetches, hash)
}
}
}
delete(f.requests, drop.peer)
// If a request was cancelled, check if anything needs to be rescheduled
f.scheduleFetches(timeoutTimer, timeoutTrigger, nil)
f.rescheduleTimeout(timeoutTimer, timeoutTrigger)
}
case <-f.quit:
return
}
// Loop did something, ping the step notifier if needed (tests)
if f.step != nil {
f.step <- struct{}{}
}
}
}
func (f *BlobFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) {
if *timer != nil {
(*timer).Stop()
}
now := f.clock.Now()
earliest := now
for _, instance := range f.waittime {
if earliest > instance {
earliest = instance
if txArriveTimeout-time.Duration(now-earliest) < txGatherSlack {
break
}
}
}
*timer = f.clock.AfterFunc(txArriveTimeout-time.Duration(now-earliest), func() {
trigger <- struct{}{}
})
}
// Exactly same as the one in TxFetcher
func (f *BlobFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) {
if *timer != nil {
(*timer).Stop()
}
now := f.clock.Now()
earliest := now
for _, requests := range f.requests {
for _, req := range requests {
// If this request already timed out, skip it altogether
if req.txs == nil {
continue
}
if earliest > req.time {
earliest = req.time
if blobFetchTimeout-time.Duration(now-earliest) < txGatherSlack {
break
}
}
}
}
*timer = f.clock.AfterFunc(blobFetchTimeout-time.Duration(now-earliest), func() {
trigger <- struct{}{}
})
}
func (f *BlobFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, whitelist map[string]struct{}) {
// Gather the set of peers we want to retrieve from (default to all)
actives := whitelist
if actives == nil {
actives = make(map[string]struct{})
for peer := range f.announces {
actives[peer] = struct{}{}
}
}
if len(actives) == 0 {
return
}
// For each active peer, try to schedule some payload fetches
idle := len(f.requests) == 0
f.forEachPeer(actives, func(peer string) {
if len(f.announces[peer]) == 0 || len(f.requests[peer]) != 0 {
return // continue
}
var (
hashes = make([]common.Hash, 0, maxTxRetrievals)
custodies = make([]*types.CustodyBitmap, 0, maxTxRetrievals)
)
f.forEachAnnounce(f.announces[peer], func(hash common.Hash, cells *types.CustodyBitmap) bool {
var difference *types.CustodyBitmap
if f.fetches[hash] == nil {
// tx is not being fetched
difference = cells
} else {
difference = cells.Difference(f.fetches[hash].fetching)
}
// Mark fetching for differences
if difference.OneCount() != 0 {
if f.fetches[hash] == nil {
f.fetches[hash] = &fetchStatus{
fetching: difference,
fetched: make([]uint64, 0),
cells: make([]kzg4844.Cell, 0),
}
} else {
f.fetches[hash].fetching = f.fetches[hash].fetching.Union(difference)
}
// Accumulate the hash and stop if the limit was reached
hashes = append(hashes, hash)
custodies = append(custodies, difference)
}
// Mark alternatives
if f.alternates[hash] == nil {
f.alternates[hash] = map[string]*types.CustodyBitmap{
peer: cells,
}
} else {
f.alternates[hash][peer] = cells
}
return len(hashes) < maxPayloadRetrievals
})
// If any hashes were allocated, request them from the peer
if len(hashes) > 0 {
// Group hashes by custody bitmap
requestByCustody := make(map[string]*cellRequest)
for i, hash := range hashes {
custody := custodies[i]
key := string(custody[:])
if _, ok := requestByCustody[key]; !ok {
requestByCustody[key] = &cellRequest{
txs: []common.Hash{},
cells: custody,
time: f.clock.Now(),
}
}
requestByCustody[key].txs = append(requestByCustody[key].txs, hash)
}
// construct request
var request []*cellRequest
for _, cr := range requestByCustody {
request = append(request, cr)
}
f.requests[peer] = request
go func(peer string, request []*cellRequest) {
for _, req := range request {
if err := f.fetchPayloads(peer, req.txs, req.cells); err != nil {
f.Drop(peer)
break
}
}
}(peer, request)
}
})
// If a new request was fired, schedule a timeout timer
if idle && len(f.requests) > 0 {
f.rescheduleTimeout(timer, timeout)
}
}
// forEachAnnounce loops over the given announcements in arrival order, invoking
// the do function for each until it returns false. We enforce an arrival
// ordering to minimize the chances of transaction nonce-gaps, which result in
// transactions being rejected by the txpool.
func (f *BlobFetcher) forEachAnnounce(announces map[common.Hash]*cellWithSeq, do func(hash common.Hash, cells *types.CustodyBitmap) bool) {
type announcement struct {
hash common.Hash
cells *types.CustodyBitmap
seq uint64
}
// Process announcements by their arrival order
list := make([]announcement, 0, len(announces))
for hash, entry := range announces {
list = append(list, announcement{hash: hash, cells: entry.cells, seq: entry.seq})
}
sort.Slice(list, func(i, j int) bool {
return list[i].seq < list[j].seq
})
for i := range list {
if !do(list[i].hash, list[i].cells) {
return
}
}
}
// forEachPeer does a range loop over a map of peers in production, but during
// testing it does a deterministic sorted random to allow reproducing issues.
func (f *BlobFetcher) forEachPeer(peers map[string]struct{}, do func(peer string)) {
// If we're running production(step == nil), use whatever Go's map gives us
if f.step == nil {
for peer := range peers {
do(peer)
}
return
}
// We're running the test suite, make iteration deterministic (sorted by peer id)
list := make([]string, 0, len(peers))
for peer := range peers {
list = append(list, peer)
}
sort.Strings(list)
for _, peer := range list {
do(peer)
}
}

999
eth/fetcher/blob_fetcher_test.go Normal file
View file

@ -0,0 +1,999 @@
// Copyright 2026 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package fetcher
import (
"slices"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
)
// makeTestBlobSidecar is a helper method to create random blob sidecar
// with certain number of blobs.
func makeTestCellSidecar(blobCount int) *types.BlobTxCellSidecar {
var (
blobs []kzg4844.Blob
commitments []kzg4844.Commitment
proofs []kzg4844.Proof
)
for i := 0; i < blobCount; i++ {
blob := &kzg4844.Blob{}
blob[0] = byte(i)
blobs = append(blobs, *blob)
commit, _ := kzg4844.BlobToCommitment(blob)
commitments = append(commitments, commit)
cellProofs, _ := kzg4844.ComputeCellProofs(blob)
proofs = append(proofs, cellProofs...)
}
sidecar, _ := types.NewBlobTxSidecar(types.BlobSidecarVersion1, blobs, commitments, proofs).ToBlobTxCellSidecar()
return sidecar
}
func selectCells(cells []kzg4844.Cell, custody *types.CustodyBitmap) []kzg4844.Cell {
custodyIndices := custody.Indices()
result := make([]kzg4844.Cell, 0)
for _, idx := range custodyIndices {
result = append(result, cells[idx])
}
return result
}
const (
testBlobAvailabilityTimeout = 500 * time.Millisecond
testBlobFetchTimeout = 5 * time.Second
)
var (
testBlobTxHashes = []common.Hash{
{0x01}, {0x02}, {0x03}, {0x04}, {0x05}, {0x06}, {0x07}, {0x08},
}
testBlobSidecars = []*types.BlobTxCellSidecar{
makeTestCellSidecar(1),
makeTestCellSidecar(2),
makeTestCellSidecar(3),
makeTestCellSidecar(4),
}
custody = types.NewCustodyBitmap([]uint64{0, 1, 2, 3, 4, 5, 6, 7})
fullCustody = *types.CustodyBitmapAll
halfCustody = *types.CustodyBitmapData
frontCustody = types.NewCustodyBitmap([]uint64{0, 1, 2, 3, 8, 9, 10, 11})
backCustody = types.NewCustodyBitmap([]uint64{4, 5, 6, 7, 8, 9, 10, 11})
differentCustody = types.NewCustodyBitmap([]uint64{8, 9, 10, 11, 12, 13, 14, 15})
)
type doBlobNotify struct {
peer string
hashes []common.Hash
custody types.CustodyBitmap
}
type doBlobEnqueue struct {
peer string
hashes []common.Hash
cells [][]kzg4844.Cell
custody types.CustodyBitmap
}
type blobDoFunc func(*BlobFetcher)
type isWaitingAvailability map[common.Hash]map[string]struct{}
type isDecidedFull map[common.Hash]struct{}
type isDecidedPartial map[common.Hash]struct{}
type blobAnnounce struct {
hash common.Hash
custody types.CustodyBitmap
}
type isBlobScheduled struct {
announces map[string][]blobAnnounce // announces에 있는 것들 (peer -> hash+custody)
fetching map[string][]blobAnnounce // requests에 있는 것들 (peer -> hash+custody)
}
type isCompleted []common.Hash
type isDropped []string
type isFetching struct {
hashes map[common.Hash]fetchInfo
}
type fetchInfo struct {
fetching *types.CustodyBitmap
fetched []uint64
}
type blobFetcherTest struct {
init func() *BlobFetcher
steps []interface{}
}
type mockRand struct {
value int
}
func (r *mockRand) Intn(n int) int {
return r.value
}
// TestBlobFetcherFullSchedule tests scheduling full payload decision
// Blob should be fetched immediately when its availability is announced
// by idle peer, if the client decided to pull the full payload
// Additional announcements should be recorded as alternates during the fetch
func TestBlobFetcherFullFetch(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 5}, // to force full requests (5 < 15)
)
},
steps: []interface{}{
// A announced full custody blob (should make full decision & start fetching)
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isDecidedFull{testBlobTxHashes[0]: struct{}{}},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// Same hash announced by another peer(B) -> should be added to alternatives
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
// Announce partial custody by C -> should be ignored
doBlobNotify{peer: "C", hashes: []common.Hash{testBlobTxHashes[1]}, custody: custody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
// Additional hashes announced by A -> should not be fetched
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[1]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}, {hash: testBlobTxHashes[1], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
// Announce of multiple transactions
doBlobNotify{peer: "D", hashes: []common.Hash{testBlobTxHashes[2], testBlobTxHashes[3]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}, {hash: testBlobTxHashes[1], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"D": {{hash: testBlobTxHashes[2], custody: halfCustody}, {hash: testBlobTxHashes[3], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"D": {{hash: testBlobTxHashes[2], custody: halfCustody}, {hash: testBlobTxHashes[3], custody: halfCustody}},
},
},
},
})
}
// TestBlobFetcherPartialFetching tests partial request decision and availability check flow
func TestBlobFetcherPartialFetch(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 20}, // Force partial requests (20 >= 15)
)
},
steps: []interface{}{
// First full announce for tx 0, 1 -> should make partial decision and go to waitlist
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0], testBlobTxHashes[1]}, custody: fullCustody},
isDecidedPartial{testBlobTxHashes[0]: struct{}{}, testBlobTxHashes[1]: struct{}{}},
isWaitingAvailability{testBlobTxHashes[0]: map[string]struct{}{"A": {}}, testBlobTxHashes[1]: map[string]struct{}{"A": {}}},
isBlobScheduled{announces: nil, fetching: nil},
// Partial announce for tx 0 (waitlist) -> should be dropped
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: custody},
isWaitingAvailability{testBlobTxHashes[0]: map[string]struct{}{"A": {}}, testBlobTxHashes[1]: map[string]struct{}{"A": {}}},
isBlobScheduled{announces: nil, fetching: nil},
// Second full announce for tx 0 -> should make tx 0 available & fetched
doBlobNotify{peer: "C", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isWaitingAvailability{testBlobTxHashes[1]: map[string]struct{}{"A": {}}},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"C": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &custody,
fetched: []uint64{},
},
},
},
// Partial announce for tx 0, overlapped custody -> overlapping part should be accepted
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: frontCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"B": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"C": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
// Partial announce for tx 0, with additional custody -> don't update
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: custody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"B": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"C": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
// Partial announce for tx 0, without any overlapped custody -> should be dropped
doBlobNotify{peer: "D", hashes: []common.Hash{testBlobTxHashes[0]}, custody: differentCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"B": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"C": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
},
})
}
// todo wait timeout
// todo drop
// TestBlobFetcherFullDelivery tests cell delivery and fetch completion logic (full fetch)
func TestBlobFetcherFullDelivery(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 5}, // Force full requests for simplicity
)
},
steps: []interface{}{
// Full announce by two peers (A, B) -> schedule fetch
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// All alternates should be clean up on delivery
doBlobEnqueue{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, cells: [][]kzg4844.Cell{selectCells(testBlobSidecars[0].Cells, &halfCustody)}, custody: halfCustody},
isBlobScheduled{announces: nil, fetching: nil},
isFetching{hashes: nil}, // fetches should be empty after completion
isCompleted{testBlobTxHashes[0]},
},
})
}
// TestBlobFetcherPartialDelivery tests cell delivery and fetch completion logic (partial fetch)
func TestBlobFetcherPartialDelivery(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 20},
)
},
steps: []interface{}{
// Full announce by two peers (A, B) -> schedule fetch
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isWaitingAvailability(nil),
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"B": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &custody,
fetched: []uint64{},
},
},
},
// Partial announce by C, D -> alternates
doBlobNotify{peer: "C", hashes: []common.Hash{testBlobTxHashes[0]}, custody: frontCustody},
doBlobNotify{peer: "D", hashes: []common.Hash{testBlobTxHashes[0]}, custody: backCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
"B": {{hash: testBlobTxHashes[0], custody: custody}},
"C": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: custody}},
},
},
// Drop A, B -> schedule fetch from C, D
doDrop("A"),
doDrop("B"),
isBlobScheduled{
announces: map[string][]blobAnnounce{
"C": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
fetching: map[string][]blobAnnounce{
"C": {{hash: testBlobTxHashes[0], custody: *frontCustody.Intersection(&custody)}},
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
},
// Delivery from C -> wait for D
doBlobEnqueue{
peer: "C",
hashes: []common.Hash{testBlobTxHashes[0]},
cells: [][]kzg4844.Cell{selectCells(testBlobSidecars[0].Cells, frontCustody.Intersection(&custody))},
custody: *frontCustody.Intersection(&custody),
},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
fetching: map[string][]blobAnnounce{
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &custody,
fetched: frontCustody.Intersection(&custody).Indices(),
},
},
},
// Announce already delivered cells + fetching cells -> leave fetching cells only
doBlobNotify{peer: "E", hashes: []common.Hash{testBlobTxHashes[0]}, custody: custody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
"E": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"D": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
},
// Not delivered -> reschedule to E
doWait{time: blobFetchTimeout, step: true},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"E": {{hash: testBlobTxHashes[0], custody: custody}},
},
fetching: map[string][]blobAnnounce{
"E": {{hash: testBlobTxHashes[0], custody: *backCustody.Intersection(&custody)}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &custody,
fetched: frontCustody.Intersection(&custody).Indices(),
},
},
},
// Delivery from E -> complete
doWait{time: blobFetchTimeout / 2, step: true},
doBlobEnqueue{
peer: "E",
hashes: []common.Hash{testBlobTxHashes[0]},
cells: [][]kzg4844.Cell{selectCells(testBlobSidecars[0].Cells, backCustody.Intersection(&custody))},
custody: *backCustody.Intersection(&custody),
},
isCompleted{testBlobTxHashes[0]},
},
})
}
// TestBlobFetcherAvailabilityTimeout tests availability timeout for partial requests
func TestBlobFetcherAvailabilityTimeout(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 20},
)
},
steps: []interface{}{
// First full announce for tx 0 -> should make partial decision and go to waitlist
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isDecidedPartial{testBlobTxHashes[0]: struct{}{}},
isWaitingAvailability{testBlobTxHashes[0]: map[string]struct{}{"A": {}}},
isBlobScheduled{announces: nil, fetching: nil},
// Run clock for timeout
doWait{time: testBlobAvailabilityTimeout, step: true},
// After timeout, waitlist should be empty
isWaitingAvailability{},
isBlobScheduled{announces: nil, fetching: nil},
},
})
}
// TestBlobFetcherPeerDrop tests peer drop scenarios
func TestBlobFetcherPeerDrop(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 5},
)
},
steps: []interface{}{
// Full announce by peer A -> should schedule fetch
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isDecidedFull{testBlobTxHashes[0]: struct{}{}},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// Another peer B announces same hash -> should be added to alternates
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
// Drop peer A -> should reschedule fetch to peer B
doDrop("A"),
isBlobScheduled{
announces: map[string][]blobAnnounce{
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// Drop peer B -> should drop the transaction, remove all traces
doDrop("B"),
isBlobScheduled{announces: nil, fetching: nil},
isFetching{hashes: nil},
},
})
}
// TestBlobFetcherFetchTimeout tests fetch timeout and rescheduling, full request case
func TestBlobFetcherFetchTimeout(t *testing.T) {
testBlobFetcher(t, blobFetcherTest{
init: func() *BlobFetcher {
return NewBlobFetcher(
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(txs []common.Hash, _ [][]kzg4844.Cell, _ *types.CustodyBitmap) []error {
return make([]error, len(txs))
},
func(string, []common.Hash, *types.CustodyBitmap) error { return nil },
func(string) {},
&custody,
&mockRand{value: 5},
)
},
steps: []interface{}{
// Full announce by peer A -> schedule fetch
doBlobNotify{peer: "A", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isDecidedFull{testBlobTxHashes[0]: struct{}{}},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// Another peer announces same hash -> should be added to alternates
doBlobNotify{peer: "B", hashes: []common.Hash{testBlobTxHashes[0]}, custody: fullCustody},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"A": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
// Wait for fetch timeout -> should reschedule to peer B
doWait{time: testBlobFetchTimeout, step: true},
isBlobScheduled{
announces: map[string][]blobAnnounce{
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
fetching: map[string][]blobAnnounce{
"B": {{hash: testBlobTxHashes[0], custody: halfCustody}},
},
},
isFetching{
hashes: map[common.Hash]fetchInfo{
testBlobTxHashes[0]: {
fetching: &halfCustody,
fetched: []uint64{},
},
},
},
// Wait for timeout -> should drop transaction
doWait{time: testBlobFetchTimeout, step: true},
isBlobScheduled{announces: nil, fetching: nil},
isFetching{hashes: nil},
},
})
}
// testBlobFetcher is the generic test runner for blob fetcher tests
func testBlobFetcher(t *testing.T, tt blobFetcherTest) {
clock := new(mclock.Simulated)
wait := make(chan struct{})
// Create a fetcher and boot it up
fetcher := tt.init()
fetcher.clock = clock
fetcher.step = wait
fetcher.Start()
defer fetcher.Stop()
defer func() {
for {
select {
case <-wait:
default:
return
}
}
}()
// Iterate through all the test steps and execute them
for i, step := range tt.steps {
// Clear the channel if anything is left over
for len(wait) > 0 {
<-wait
}
// Process the next step of the test
switch step := step.(type) {
case doBlobNotify:
if err := fetcher.Notify(step.peer, step.hashes, step.custody); err != nil {
t.Errorf("step %d: failed to notify fetcher: %v", i, err)
return
}
<-wait
case doBlobEnqueue:
if err := fetcher.Enqueue(step.peer, step.hashes, step.cells, step.custody); err != nil {
t.Errorf("step %d: failed to enqueue blobs: %v", i, err)
return
}
<-wait
case blobDoFunc:
step(fetcher)
case isWaitingAvailability:
// Check expected hashes and peers are present
for hash, peers := range step {
if waitPeers, ok := fetcher.waitlist[hash]; !ok {
t.Errorf("step %d: hash %x not in waitlist", i, hash)
return
} else {
// Check expected peers are present
for peer := range peers {
if _, ok := waitPeers[peer]; !ok {
t.Errorf("step %d: peer %s not waiting for hash %x", i, peer, hash)
return
}
}
// Check no unexpected peers are present
for peer := range waitPeers {
if _, ok := peers[peer]; !ok {
t.Errorf("step %d: unexpected peer %s waiting for hash %x", i, peer, hash)
return
}
}
}
}
// Check no unexpected hashes in waitlist
for hash := range fetcher.waitlist {
if _, ok := step[hash]; !ok {
t.Errorf("step %d: unexpected hash %x in waitlist", i, hash)
return
}
}
case isDecidedFull:
for hash := range step {
if _, ok := fetcher.full[hash]; !ok {
t.Errorf("step %d: hash %x not decided for full request", i, hash)
return
}
}
case isDecidedPartial:
for hash := range step {
if _, ok := fetcher.partial[hash]; !ok {
t.Errorf("step %d: hash %x not decided for partial request", i, hash)
return
}
}
case isBlobScheduled:
// todo fetches
// Check tracking (announces) - bidirectional verification
for peer, announces := range step.announces {
peerAnnounces := fetcher.announces[peer]
if peerAnnounces == nil {
t.Errorf("step %d: peer %s missing from announces", i, peer)
continue
}
// Check expected announces are present
for _, ann := range announces {
if cellWithSeq, ok := peerAnnounces[ann.hash]; !ok {
t.Errorf("step %d, peer %s: hash %x missing from announces", i, peer, ann.hash)
} else if *cellWithSeq.cells != ann.custody {
t.Errorf("step %d, peer %s, hash %x: custody mismatch in announces", i, peer, ann.hash)
}
}
// Check no unexpected announces are present
for hash := range peerAnnounces {
found := false
for _, ann := range announces {
if ann.hash == hash {
found = true
break
}
}
if !found {
t.Errorf("step %d, peer %s: unexpected hash %x in announces", i, peer, hash)
}
}
}
// Check no unexpected peers in announces
for peer := range fetcher.announces {
if _, ok := step.announces[peer]; !ok {
t.Errorf("step %d: unexpected peer %s in announces", i, peer)
}
}
// Check fetching (requests)
for peer, requests := range step.fetching {
peerRequests := fetcher.requests[peer]
if peerRequests == nil {
t.Errorf("step %d: peer %s missing from requests", i, peer)
continue
}
// Check expected requests are present
for _, req := range requests {
found := false
for _, cellReq := range peerRequests {
for _, hash := range cellReq.txs {
if hash == req.hash && *cellReq.cells == req.custody {
found = true
break
}
}
if found {
break
}
}
if !found {
t.Errorf("step %d, peer %s: hash %x with custody not found in requests", i, peer, req.hash)
}
}
// (bidirectional) Check no unexpected requests are present
for _, cellReq := range peerRequests {
for _, hash := range cellReq.txs {
found := false
for _, req := range requests {
if req.hash == hash && *cellReq.cells == req.custody {
found = true
break
}
}
if !found {
t.Errorf("step %d, peer %s: unexpected hash %x in requests", i, peer, hash)
}
}
}
}
// Check no unexpected peers in requests
for peer := range fetcher.requests {
if _, ok := step.fetching[peer]; !ok {
t.Errorf("step %d: unexpected peer %s in requests", i, peer)
}
}
// Check internal consistency: alternates should match announces
// For every hash being fetched, alternates should contain all peers who announced it
for _, announces := range step.fetching {
for _, announce := range announces {
hash := announce.hash
alternates := fetcher.alternates[hash]
if alternates == nil {
t.Errorf("step %d: hash %x missing from alternates", i, hash)
continue
}
// Check that all peers with this hash in announces are in alternates with matching custody
for peer, peerAnnounces := range fetcher.announces {
if cellWithSeq := peerAnnounces[hash]; cellWithSeq != nil {
if altCustody, ok := alternates[peer]; !ok {
t.Errorf("step %d, hash %x: peer %s missing from alternates", i, hash, peer)
} else if *altCustody != *cellWithSeq.cells {
t.Errorf("step %d, hash %x, peer %s: custody bitmap mismatch in alternates", i, hash, peer)
}
}
}
// Check that all peers in alternates actually have this hash announced with matching custody
for peer, altCustody := range alternates {
if fetcher.announces[peer] == nil || fetcher.announces[peer][hash] == nil {
t.Errorf("step %d, hash %x: peer %s extra in alternates", i, hash, peer)
} else if cellWithSeq := fetcher.announces[peer][hash]; *cellWithSeq.cells != *altCustody {
t.Errorf("step %d, hash %x, peer %s: custody bitmap mismatch between announces and alternates", i, hash, peer)
}
}
}
}
case isFetching:
// Check expected hashes are present in fetches
for hash, expected := range step.hashes {
if fetchStatus, ok := fetcher.fetches[hash]; !ok {
t.Errorf("step %d: hash %x missing from fetches", i, hash)
} else {
// Check fetching bitmap
if expected.fetching != nil {
if fetchStatus.fetching == nil {
t.Errorf("step %d, hash %x: fetching bitmap is nil", i, hash)
} else if *fetchStatus.fetching != *expected.fetching {
t.Errorf("step %d, hash %x: fetching bitmap mismatch", i, hash)
}
}
// Check fetched indices
if expected.fetched != nil {
if len(fetchStatus.fetched) != len(expected.fetched) {
t.Errorf("step %d, hash %x: fetched length mismatch, got %d, want %d", i, hash, len(fetchStatus.fetched), len(expected.fetched))
} else {
// Sort both slices before comparing
gotFetched := make([]uint64, len(fetchStatus.fetched))
copy(gotFetched, fetchStatus.fetched)
slices.Sort(gotFetched)
expectedFetched := make([]uint64, len(expected.fetched))
copy(expectedFetched, expected.fetched)
slices.Sort(expectedFetched)
if !slices.Equal(gotFetched, expectedFetched) {
t.Errorf("step %d, hash %x: fetched indices mismatch", i, hash)
}
}
}
}
}
// Check no unexpected hashes in fetches
for hash := range fetcher.fetches {
if _, ok := step.hashes[hash]; !ok {
t.Errorf("step %d: unexpected hash %x in fetches", i, hash)
}
}
case isCompleted:
for _, hash := range step {
if _, ok := fetcher.fetches[hash]; ok {
t.Errorf("step %d: hash %x still in fetches (should be completed)", i, hash)
return
}
}
case isDropped:
for _, peer := range step {
if _, ok := fetcher.announces[peer]; ok {
t.Errorf("step %d: peer %s still has announces (should be dropped)", i, peer)
return
}
}
case doWait:
clock.Run(step.time)
if step.step {
<-wait
}
case doDrop:
if err := fetcher.Drop(string(step)); err != nil {
t.Errorf("step %d: %v", i, err)
}
<-wait
default:
t.Errorf("step %d: unknown step type %T", i, step)
return
}
}
}

21
eth/fetcher/metrics.go
View file

@ -57,4 +57,25 @@ var (
// to become "unfrozen", either by eventually replying to the request
// or by being dropped, measuring from the moment the request was sent.
txFetcherSlowWait = metrics.NewRegisteredHistogram("eth/fetcher/transaction/slow/wait", nil, metrics.NewExpDecaySample(1028, 0.015))
blobAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/in", nil)
blobAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/announces/dos", nil)
blobRequestOutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/out", nil)
blobRequestFailMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/fail", nil)
blobRequestDoneMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/done", nil)
blobRequestTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/request/timeout", nil)
blobReplyInMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/replies/in", nil)
blobReplyRejectMeter = metrics.NewRegisteredMeter("eth/fetcher/blob/replies/reject", nil)
blobFetcherWaitingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/waiting/peers", nil)
blobFetcherWaitingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/waiting/hashes", nil)
blobFetcherQueueingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/queueing/peers", nil)
blobFetcherQueueingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/queueing/hashes", nil)
blobFetcherFetchingPeers = metrics.NewRegisteredGauge("eth/fetcher/blob/fetching/peers", nil)
blobFetcherFetchingHashes = metrics.NewRegisteredGauge("eth/fetcher/blob/fetching/hashes", nil)
blobFetcherWaitTime = metrics.NewRegisteredHistogram("eth/fetcher/blob/wait/time", nil, metrics.NewExpDecaySample(1028, 0.015))
blobFetcherFetchTime = metrics.NewRegisteredHistogram("eth/fetcher/blob/fetch/time", nil, metrics.NewExpDecaySample(1028, 0.015))
)

17
eth/fetcher/tx_fetcher.go
View file

@ -232,7 +232,7 @@ func NewTxFetcherForTests(
// Notify announces the fetcher of the potential availability of a new batch of
// transactions in the network.
func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []common.Hash) error {
func (f *TxFetcher) Notify(peer string, kinds []byte, sizes []uint32, hashes []common.Hash) ([]common.Hash, error) {
// Keep track of all the announced transactions
txAnnounceInMeter.Mark(int64(len(hashes)))
@ -245,12 +245,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
unknownHashes = make([]common.Hash, 0, len(hashes))
unknownMetas = make([]txMetadata, 0, len(hashes))
blobFetchHashes = make([]common.Hash, 0, len(hashes))
duplicate int64
onchain int64
underpriced int64
)
for i, hash := range hashes {
err := f.validateMeta(hash, types[i])
err := f.validateMeta(hash, kinds[i])
if errors.Is(err, txpool.ErrAlreadyKnown) {
duplicate++
continue
@ -271,11 +273,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
}
unknownHashes = append(unknownHashes, hash)
if kinds[i] == types.BlobTxType {
blobFetchHashes = append(blobFetchHashes, hash)
}
// Transaction metadata has been available since eth68, and all
// legacy eth protocols (prior to eth68) have been deprecated.
// Therefore, metadata is always expected in the announcement.
unknownMetas = append(unknownMetas, txMetadata{kind: types[i], size: sizes[i]})
unknownMetas = append(unknownMetas, txMetadata{kind: kinds[i], size: sizes[i]})
}
txAnnounceKnownMeter.Mark(duplicate)
txAnnounceUnderpricedMeter.Mark(underpriced)
@ -283,14 +288,14 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c
// If anything's left to announce, push it into the internal loop
if len(unknownHashes) == 0 {
return nil
return blobFetchHashes, nil
}
announce := &txAnnounce{origin: peer, hashes: unknownHashes, metas: unknownMetas}
select {
case f.notify <- announce:
return nil
return blobFetchHashes, nil
case <-f.quit:
return errTerminated
return nil, errTerminated
}
}

2
eth/fetcher/tx_fetcher_test.go
View file

@ -1888,7 +1888,7 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) {
// Process the original or expanded steps
switch step := step.(type) {
case doTxNotify:
if err := fetcher.Notify(step.peer, step.types, step.sizes, step.hashes); err != nil {
if _, err := fetcher.Notify(step.peer, step.types, step.sizes, step.hashes); err != nil {
t.Errorf("step %d: %v", i, err)
}
<-wait // Fetcher needs to process this, wait until it's done

31
eth/handler.go
View file

@ -33,6 +33,7 @@ import (
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/fetcher"
@ -75,7 +76,7 @@ type txPool interface {
// GetRLP retrieves the RLP-encoded transaction from local txpool
// with given tx hash.
GetRLP(hash common.Hash) []byte
GetRLP(hash common.Hash, includeBlob bool) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
@ -97,6 +98,16 @@ type txPool interface {
FilterType(kind byte) bool
}
// blobPool defines the methods needed from a blob pool implementation to
// support cell-based blob data availability.
type blobPool interface {
Has(hash common.Hash) bool
GetCells(hash common.Hash, mask types.CustodyBitmap) ([]kzg4844.Cell, error)
ValidateCells([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error
AddPayload([]common.Hash, [][]kzg4844.Cell, *types.CustodyBitmap) []error
GetCustody(hash common.Hash) *types.CustodyBitmap
}
// handlerConfig is the collection of initialization parameters to create a full
// node network handler.
type handlerConfig struct {
@ -104,11 +115,13 @@ type handlerConfig struct {
Database ethdb.Database // Database for direct sync insertions
Chain *core.BlockChain // Blockchain to serve data from
TxPool txPool // Transaction pool to propagate from
BlobPool blobPool // Blob pool for cell-based blob data availability
Network uint64 // Network identifier to advertise
Sync ethconfig.SyncMode // Whether to snap or full sync
BloomCache uint64 // Megabytes to alloc for snap sync bloom
EventMux *event.TypeMux // Legacy event mux, deprecate for `feed`
RequiredBlocks map[uint64]common.Hash // Hard coded map of required block hashes for sync challenges
Custody types.CustodyBitmap
}
type handler struct {
@ -118,11 +131,13 @@ type handler struct {
database ethdb.Database
txpool txPool
blobpool blobPool
chain *core.BlockChain
maxPeers int
downloader *downloader.Downloader
txFetcher *fetcher.TxFetcher
blobFetcher *fetcher.BlobFetcher
peers *peerSet
txBroadcastKey [16]byte
@ -154,6 +169,7 @@ func newHandler(config *handlerConfig) (*handler, error) {
eventMux: config.EventMux,
database: config.Database,
txpool: config.TxPool,
blobpool: config.BlobPool,
chain: config.Chain,
peers: newPeerSet(),
txBroadcastKey: newBroadcastChoiceKey(),
@ -189,6 +205,16 @@ func newHandler(config *handlerConfig) (*handler, error) {
return nil
}
h.txFetcher = fetcher.NewTxFetcher(h.chain, validateMeta, addTxs, fetchTx, h.removePeer)
// Construct the blob fetcher for cell-based blob data availability
fetchPayloads := func(peer string, hashes []common.Hash, cells *types.CustodyBitmap) error {
p := h.peers.peer(peer)
if p == nil {
return errors.New("unknown peer")
}
return p.RequestPayload(hashes, cells)
}
h.blobFetcher = fetcher.NewBlobFetcher(h.blobpool.ValidateCells, h.blobpool.AddPayload, fetchPayloads, h.removePeer, &config.Custody, nil)
return h, nil
}
@ -403,6 +429,7 @@ func (h *handler) unregisterPeer(id string) {
}
h.downloader.UnregisterPeer(id)
h.txFetcher.Drop(id)
h.blobFetcher.Drop(id)
if err := h.peers.unregisterPeer(id); err != nil {
logger.Error("Ethereum peer removal failed", "err", err)
@ -425,6 +452,7 @@ func (h *handler) Start(maxPeers int) {
// start sync handlers
h.txFetcher.Start()
h.blobFetcher.Start()
// start peer handler tracker
h.wg.Add(1)
@ -435,6 +463,7 @@ func (h *handler) Stop() {
h.txsSub.Unsubscribe() // quits txBroadcastLoop
h.blockRange.stop()
h.txFetcher.Stop()
h.blobFetcher.Stop()
h.downloader.Terminate()
// Quit chainSync and txsync64.

35
eth/handler_eth.go
View file

@ -33,6 +33,7 @@ type ethHandler handler
func (h *ethHandler) Chain() *core.BlockChain { return h.chain }
func (h *ethHandler) TxPool() eth.TxPool { return h.txpool }
func (h *ethHandler) BlobPool() eth.BlobPool { return h.blobpool }
// RunPeer is invoked when a peer joins on the `eth` protocol.
func (h *ethHandler) RunPeer(peer *eth.Peer, hand eth.Handler) error {
@ -58,8 +59,19 @@ func (h *ethHandler) AcceptTxs() bool {
func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
// Consume any broadcasts and announces, forwarding the rest to the downloader
switch packet := packet.(type) {
case *eth.NewPooledTransactionHashesPacket:
return h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
case *eth.NewPooledTransactionHashesPacket71:
hashes, err := h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
if err != nil {
return err
}
if len(hashes) != 0 {
return h.blobFetcher.Notify(peer.ID(), hashes, packet.Mask)
}
return nil
case *eth.NewPooledTransactionHashesPacket70:
_, err := h.txFetcher.Notify(peer.ID(), packet.Types, packet.Sizes, packet.Hashes)
return err
case *eth.TransactionsPacket:
txs, err := packet.Items()
@ -81,6 +93,9 @@ func (h *ethHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
}
return h.txFetcher.Enqueue(peer.ID(), txs, true)
case *eth.CellsResponse:
return h.blobFetcher.Enqueue(peer.ID(), packet.Hashes, packet.Cells, packet.Mask)
default:
return fmt.Errorf("unexpected eth packet type: %T", packet)
}
@ -98,11 +113,17 @@ func handleTransactions(peer *eth.Peer, list []*types.Transaction, directBroadca
// If we receive any blob transactions missing sidecars, or with
// sidecars that don't correspond to the versioned hashes reported
// in the header, disconnect from the sending peer.
if tx.BlobTxSidecar() == nil {
return errors.New("received sidecar-less blob transaction")
}
if err := tx.BlobTxSidecar().ValidateBlobCommitmentHashes(tx.BlobHashes()); err != nil {
return err
if peer.Version() >= eth.ETH71 {
if tx.BlobTxSidecar() != nil && len(tx.BlobTxSidecar().Blobs) != 0 {
return fmt.Errorf("not allowed to respond with full-blob transaction under eth71")
}
} else {
if tx.BlobTxSidecar() == nil {
return errors.New("received sidecar-less blob transaction")
}
if err := tx.BlobTxSidecar().ValidateBlobCommitmentHashes(tx.BlobHashes()); err != nil {
return err
}
}
}
}

34
eth/handler_eth_test.go
View file

@ -44,13 +44,14 @@ type testEthHandler struct {
func (h *testEthHandler) Chain() *core.BlockChain { panic("no backing chain") }
func (h *testEthHandler) TxPool() eth.TxPool { panic("no backing tx pool") }
func (h *testEthHandler) BlobPool() eth.BlobPool { return nil }
func (h *testEthHandler) AcceptTxs() bool { return true }
func (h *testEthHandler) RunPeer(*eth.Peer, eth.Handler) error { panic("not used in tests") }
func (h *testEthHandler) PeerInfo(enode.ID) interface{} { panic("not used in tests") }
func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error {
switch packet := packet.(type) {
case *eth.NewPooledTransactionHashesPacket:
case *eth.NewPooledTransactionHashesPacket70:
h.txAnnounces.Send(packet.Hashes)
return nil
@ -105,10 +106,12 @@ func testForkIDSplit(t *testing.T, protocol uint) {
_, blocksNoFork, _ = core.GenerateChainWithGenesis(gspecNoFork, engine, 2, nil)
_, blocksProFork, _ = core.GenerateChainWithGenesis(gspecProFork, engine, 2, nil)
txPool = newTestTxPool()
ethNoFork, _ = newHandler(&handlerConfig{
Database: dbNoFork,
Chain: chainNoFork,
TxPool: newTestTxPool(),
TxPool: txPool,
BlobPool: txPool,
Network: 1,
Sync: ethconfig.FullSync,
BloomCache: 1,
@ -116,7 +119,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
ethProFork, _ = newHandler(&handlerConfig{
Database: dbProFork,
Chain: chainProFork,
TxPool: newTestTxPool(),
TxPool: txPool,
BlobPool: txPool,
Network: 1,
Sync: ethconfig.FullSync,
BloomCache: 1,
@ -137,8 +141,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
peerNoFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil)
peerProFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil)
peerNoFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil)
peerProFork := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@ -168,8 +172,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil)
peerNoFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@ -199,8 +203,8 @@ func testForkIDSplit(t *testing.T, protocol uint) {
defer p2pNoFork.Close()
defer p2pProFork.Close()
peerNoFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil)
peerNoFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pNoFork), p2pNoFork, nil, nil)
peerProFork = eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pProFork), p2pProFork, nil, nil)
defer peerNoFork.Close()
defer peerProFork.Close()
@ -249,8 +253,8 @@ func testRecvTransactions(t *testing.T, protocol uint) {
defer p2pSrc.Close()
defer p2pSink.Close()
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool)
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, handler.txpool)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, handler.txpool)
defer src.Close()
defer sink.Close()
@ -305,8 +309,8 @@ func testSendTransactions(t *testing.T, protocol uint) {
defer p2pSrc.Close()
defer p2pSink.Close()
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool)
src := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pSrc), p2pSrc, handler.txpool, handler.blobpool)
sink := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pSink), p2pSink, handler.txpool, handler.blobpool)
defer src.Close()
defer sink.Close()
@ -380,8 +384,8 @@ func testTransactionPropagation(t *testing.T, protocol uint) {
defer sourcePipe.Close()
defer sinkPipe.Close()
sourcePeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{byte(i + 1)}, "", nil, sourcePipe), sourcePipe, source.txpool)
sinkPeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, sink.txpool)
sourcePeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{byte(i + 1)}, "", nil, sourcePipe), sourcePipe, source.txpool, source.txpool)
sinkPeer := eth.NewPeer(protocol, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, sink.txpool, sink.txpool)
defer sourcePeer.Close()
defer sinkPeer.Close()

108
eth/handler_test.go
View file

@ -17,6 +17,7 @@
package eth
import (
"errors"
"maps"
"math/big"
"math/rand"
@ -31,6 +32,7 @@ import (
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/eth/protocols/eth"
"github.com/ethereum/go-ethereum/ethdb"
@ -54,7 +56,10 @@ var (
// Its goal is to get around setting up a valid statedb for the balance and nonce
// checks.
type testTxPool struct {
pool map[common.Hash]*types.Transaction // Hash map of collected transactions
txPool map[common.Hash]*types.Transaction // Hash map of collected transactions
cellPool map[common.Hash][]kzg4844.Cell
custody map[common.Hash]types.CustodyBitmap
txFeed event.Feed // Notification feed to allow waiting for inclusion
lock sync.RWMutex // Protects the transaction pool
@ -63,7 +68,9 @@ type testTxPool struct {
// newTestTxPool creates a mock transaction pool.
func newTestTxPool() *testTxPool {
return &testTxPool{
pool: make(map[common.Hash]*types.Transaction),
txPool: make(map[common.Hash]*types.Transaction),
cellPool: make(map[common.Hash][]kzg4844.Cell),
custody: make(map[common.Hash]types.CustodyBitmap),
}
}
@ -73,7 +80,7 @@ func (p *testTxPool) Has(hash common.Hash) bool {
p.lock.Lock()
defer p.lock.Unlock()
return p.pool[hash] != nil
return p.txPool[hash] != nil
}
// Get retrieves the transaction from local txpool with given
@ -81,16 +88,16 @@ func (p *testTxPool) Has(hash common.Hash) bool {
func (p *testTxPool) Get(hash common.Hash) *types.Transaction {
p.lock.Lock()
defer p.lock.Unlock()
return p.pool[hash]
return p.txPool[hash]
}
// Get retrieves the transaction from local txpool with given
// tx hash.
func (p *testTxPool) GetRLP(hash common.Hash) []byte {
func (p *testTxPool) GetRLP(hash common.Hash, includeBlob bool) []byte {
p.lock.Lock()
defer p.lock.Unlock()
tx := p.pool[hash]
tx := p.txPool[hash]
if tx != nil {
blob, _ := rlp.EncodeToBytes(tx)
return blob
@ -104,7 +111,7 @@ func (p *testTxPool) GetMetadata(hash common.Hash) *txpool.TxMetadata {
p.lock.Lock()
defer p.lock.Unlock()
tx := p.pool[hash]
tx := p.txPool[hash]
if tx != nil {
return &txpool.TxMetadata{
Type: tx.Type(),
@ -121,7 +128,7 @@ func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error {
defer p.lock.Unlock()
for _, tx := range txs {
p.pool[tx.Hash()] = tx
p.txPool[tx.Hash()] = tx
}
p.txFeed.Send(core.NewTxsEvent{Txs: txs})
return make([]error, len(txs))
@ -134,7 +141,7 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]
var count int
batches := make(map[common.Address][]*types.Transaction)
for _, tx := range p.pool {
for _, tx := range p.txPool {
from, _ := types.Sender(types.HomesteadSigner{}, tx)
batches[from] = append(batches[from], tx)
}
@ -164,6 +171,68 @@ func (p *testTxPool) Pending(filter txpool.PendingFilter) (map[common.Address][]
func (p *testTxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool) event.Subscription {
return p.txFeed.Subscribe(ch)
}
func (p *testTxPool) GetCells(hash common.Hash, mask types.CustodyBitmap) ([]kzg4844.Cell, error) {
p.lock.RLock()
defer p.lock.RUnlock()
_, exists := p.txPool[hash]
if !exists {
return nil, errors.New("Requested tx does not exist")
}
var cells []kzg4844.Cell
if cells, exists = p.cellPool[hash]; !exists {
return nil, errors.New("Requested cells do not exist")
}
result := make([]kzg4844.Cell, 0, mask.OneCount())
for _, idx := range mask.Indices() {
if int(idx) < len(cells) {
result = append(result, cells[idx])
}
}
return result, nil
}
func (p *testTxPool) GetCustody(hash common.Hash) *types.CustodyBitmap {
p.lock.RLock()
defer p.lock.RUnlock()
mask, ok := p.custody[hash]
if !ok {
return nil
}
return &mask
}
// AddCells adds cells for a specific transaction hash (for testing)
func (p *testTxPool) AddCells(hash common.Hash, cells []kzg4844.Cell, mask types.CustodyBitmap) {
p.lock.Lock()
defer p.lock.Unlock()
p.cellPool[hash] = cells
p.custody[hash] = mask
}
func (p *testTxPool) AddPayload(txs []common.Hash, cells [][]kzg4844.Cell, custody *types.CustodyBitmap) []error {
p.lock.Lock()
defer p.lock.Unlock()
for i, tx := range txs {
p.cellPool[tx] = cells[i]
}
return nil
}
func (p *testTxPool) ValidateCells(txs []common.Hash, cells [][]kzg4844.Cell, custody *types.CustodyBitmap) []error {
p.lock.Lock()
defer p.lock.Unlock()
errors := make([]error, len(txs))
for i, tx := range txs {
errors[i] = kzg4844.VerifyCells(cells[i], p.txPool[tx].BlobTxSidecar().Commitments, p.txPool[tx].BlobTxSidecar().Proofs, custody.Indices())
}
return errors
}
// FilterType should check whether the pool supports the given type of transactions.
func (p *testTxPool) FilterType(kind byte) bool {
@ -178,10 +247,11 @@ func (p *testTxPool) FilterType(kind byte) bool {
// preinitialized with some sane testing defaults and the transaction pool mocked
// out.
type testHandler struct {
db ethdb.Database
chain *core.BlockChain
txpool *testTxPool
handler *handler
db ethdb.Database
chain *core.BlockChain
txpool *testTxPool
blobpool *testTxPool
handler *handler
}
// newTestHandler creates a new handler for testing purposes with no blocks.
@ -210,6 +280,7 @@ func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler
Database: db,
Chain: chain,
TxPool: txpool,
BlobPool: txpool,
Network: 1,
Sync: mode,
BloomCache: 1,
@ -217,10 +288,11 @@ func newTestHandlerWithBlocks(blocks int, mode ethconfig.SyncMode) *testHandler
handler.Start(1000)
return &testHandler{
db: db,
chain: chain,
txpool: txpool,
handler: handler,
db: db,
chain: chain,
txpool: txpool,
blobpool: txpool,
handler: handler,
}
}
@ -317,7 +389,7 @@ func createTestPeers(rand *rand.Rand, n int) []*ethPeer {
var id enode.ID
rand.Read(id[:])
p2pPeer := p2p.NewPeer(id, "test", nil)
ep := eth.NewPeer(eth.ETH69, p2pPeer, nil, nil)
ep := eth.NewPeer(eth.ETH69, p2pPeer, nil, nil, nil)
peers[i] = &ethPeer{Peer: ep}
}
return peers

30
eth/protocols/eth/broadcast.go
View file

@ -113,29 +113,51 @@ func (p *Peer) announceTransactions() {
pending []common.Hash
pendingTypes []byte
pendingSizes []uint32
mask types.CustodyBitmap
size common.StorageSize
processed = make(map[int]bool)
)
for count = 0; count < len(queue) && size < maxTxPacketSize; count++ {
if meta := p.txpool.GetMetadata(queue[count]); meta != nil {
custody := p.blobpool.GetCustody(queue[count])
if custody != nil {
// blob tx
if mask.OneCount() == 0 {
mask = *custody
} else {
if mask != *custody {
// group by mask
continue
}
}
}
pending = append(pending, queue[count])
pendingTypes = append(pendingTypes, meta.Type)
pendingSizes = append(pendingSizes, uint32(meta.Size))
size += common.HashLength
processed[count] = true
}
}
// Shift and trim queue
queue = queue[:copy(queue, queue[count:])]
// Shift and trim queue using processed map
var remaining []common.Hash
for i, h := range queue {
if !processed[i] {
remaining = append(remaining, h)
}
}
queue = remaining
// If there's anything available to transfer, fire up an async writer
if len(pending) > 0 {
done = make(chan struct{})
go func() {
if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes); err != nil {
if err := p.sendPooledTransactionHashes(pending, pendingTypes, pendingSizes, mask); err != nil {
fail <- err
return
}
close(done)
p.Log().Trace("Sent transaction announcements", "count", len(pending))
p.Log().Trace("Sent transaction announcements", "count", len(pending), "mask", mask, "tx", pending)
}()
}
}

41
eth/protocols/eth/handler.go
View file

@ -24,6 +24,7 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/txpool"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/enode"
@ -64,6 +65,9 @@ type Backend interface {
// TxPool retrieves the transaction pool object to serve data.
TxPool() TxPool
// BlobPool retrieves the blob pool object to serve cell requests.
BlobPool() BlobPool
// AcceptTxs retrieves whether transaction processing is enabled on the node
// or if inbound transactions should simply be dropped.
AcceptTxs() bool
@ -83,6 +87,16 @@ type Backend interface {
Handle(peer *Peer, packet Packet) error
}
// BlobPool defines the methods needed by the protocol handler to serve cell requests.
type BlobPool interface {
// GetCells retrieves cells for a given transaction hash filtered by the custody bitmap.
GetCells(hash common.Hash, mask types.CustodyBitmap) ([]kzg4844.Cell, error)
// GetCustody returns the custody bitmap for a given transaction hash.
GetCustody(hash common.Hash) *types.CustodyBitmap
// Has returns whether the blob pool contains a transaction with the given hash.
Has(hash common.Hash) bool
}
// TxPool defines the methods needed by the protocol handler to serve transactions.
type TxPool interface {
// Get retrieves the transaction from the local txpool with the given hash.
@ -90,7 +104,7 @@ type TxPool interface {
// GetRLP retrieves the RLP-encoded transaction from the local txpool with
// the given hash.
GetRLP(hash common.Hash) []byte
GetRLP(hash common.Hash, includeBlob bool) []byte
// GetMetadata returns the transaction type and transaction size with the
// given transaction hash.
@ -106,7 +120,7 @@ func MakeProtocols(backend Backend, network uint64, disc enode.Iterator) []p2p.P
Version: version,
Length: protocolLengths[version],
Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error {
peer := NewPeer(version, p, rw, backend.TxPool())
peer := NewPeer(version, p, rw, backend.TxPool(), backend.BlobPool())
defer peer.Close()
return backend.RunPeer(peer, func(peer *Peer) error {
@ -180,6 +194,22 @@ var eth69 = map[uint64]msgHandler{
BlockRangeUpdateMsg: handleBlockRangeUpdate,
}
var eth71 = map[uint64]msgHandler{
TransactionsMsg: handleTransactions,
NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes71,
GetBlockHeadersMsg: handleGetBlockHeaders,
BlockHeadersMsg: handleBlockHeaders,
GetBlockBodiesMsg: handleGetBlockBodies,
BlockBodiesMsg: handleBlockBodies,
GetReceiptsMsg: handleGetReceipts,
ReceiptsMsg: handleReceipts,
GetPooledTransactionsMsg: handleGetPooledTransactions,
PooledTransactionsMsg: handlePooledTransactions,
BlockRangeUpdateMsg: handleBlockRangeUpdate,
GetCellsMsg: handleGetCells,
CellsMsg: handleCells,
}
// handleMessage is invoked whenever an inbound message is received from a remote
// peer. The remote connection is torn down upon returning any error.
func handleMessage(backend Backend, peer *Peer) error {
@ -194,9 +224,12 @@ func handleMessage(backend Backend, peer *Peer) error {
defer msg.Discard()
var handlers map[uint64]msgHandler
if peer.version == ETH69 {
switch peer.version {
case ETH69:
handlers = eth69
} else {
case ETH71:
handlers = eth71
default:
return fmt.Errorf("unknown eth protocol version: %v", peer.version)
}

15
eth/protocols/eth/handler_test.go
View file

@ -62,9 +62,10 @@ func u64(val uint64) *uint64 { return &val }
// purpose is to allow testing the request/reply workflows and wire serialization
// in the `eth` protocol without actually doing any data processing.
type testBackend struct {
db ethdb.Database
chain *core.BlockChain
txpool *txpool.TxPool
db ethdb.Database
chain *core.BlockChain
txpool *txpool.TxPool
blobpool *blobpool.BlobPool
}
// newTestBackend creates an empty chain and wraps it into a mock backend.
@ -142,9 +143,10 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, cancun bool, generat
txpool, _ := txpool.New(txconfig.PriceLimit, chain, []txpool.SubPool{legacyPool, blobPool})
return &testBackend{
db: db,
chain: chain,
txpool: txpool,
db: db,
chain: chain,
txpool: txpool,
blobpool: blobPool,
}
}
@ -156,6 +158,7 @@ func (b *testBackend) close() {
func (b *testBackend) Chain() *core.BlockChain { return b.chain }
func (b *testBackend) TxPool() TxPool { return b.txpool }
func (b *testBackend) BlobPool() BlobPool { return b.blobpool }
func (b *testBackend) RunPeer(peer *Peer, handler Handler) error {
// Normally the backend would do peer maintenance and handshakes. All that

78
eth/protocols/eth/handlers.go
View file

@ -26,6 +26,7 @@ import (
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/tracker"
"github.com/ethereum/go-ethereum/rlp"
@ -482,7 +483,27 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer)
if !backend.AcceptTxs() {
return nil
}
ann := new(NewPooledTransactionHashesPacket)
ann := new(NewPooledTransactionHashesPacket70)
if err := msg.Decode(ann); err != nil {
return err
}
if len(ann.Hashes) != len(ann.Types) || len(ann.Hashes) != len(ann.Sizes) {
return fmt.Errorf("NewPooledTransactionHashes: invalid len of fields in %v %v %v", len(ann.Hashes), len(ann.Types), len(ann.Sizes))
}
// Schedule all the unknown hashes for retrieval
for _, hash := range ann.Hashes {
peer.MarkTransaction(hash)
}
return backend.Handle(peer, ann)
}
func handleNewPooledTransactionHashes71(backend Backend, msg Decoder, peer *Peer) error {
// New transaction announcement arrived, make sure we have
// a valid and fresh chain to handle them
if !backend.AcceptTxs() {
return nil
}
ann := new(NewPooledTransactionHashesPacket71)
if err := msg.Decode(ann); err != nil {
return err
}
@ -502,11 +523,11 @@ func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error
if err := msg.Decode(&query); err != nil {
return err
}
hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsRequest)
hashes, txs := answerGetPooledTransactions(backend, query.GetPooledTransactionsRequest, peer.version < ETH71)
return peer.ReplyPooledTransactionsRLP(query.RequestId, hashes, txs)
}
func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsRequest) ([]common.Hash, []rlp.RawValue) {
func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsRequest, includeBlob bool) ([]common.Hash, []rlp.RawValue) {
// Gather transactions until the fetch or network limits is reached
var (
bytes int
@ -518,7 +539,7 @@ func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsReq
break
}
// Retrieve the requested transaction, skipping if unknown to us
encoded := backend.TxPool().GetRLP(hash)
encoded := backend.TxPool().GetRLP(hash, includeBlob)
if len(encoded) == 0 {
continue
}
@ -580,3 +601,52 @@ func handleBlockRangeUpdate(backend Backend, msg Decoder, peer *Peer) error {
peer.lastRange.Store(&update)
return nil
}
func handleGetCells(backend Backend, msg Decoder, peer *Peer) error {
// Decode the cell retrieval message
var query GetCellsRequestPacket
if err := msg.Decode(&query); err != nil {
return err
}
hashes, cells, custody := answerGetCells(backend, query.GetCellsRequest)
return peer.ReplyCells(query.RequestId, hashes, cells, custody)
}
func answerGetCells(backend Backend, query GetCellsRequest) ([]common.Hash, [][]kzg4844.Cell, types.CustodyBitmap) {
var (
cellCounts int
hashes []common.Hash
cells [][]kzg4844.Cell
)
maxCells := softResponseLimit / 2048
for _, hash := range query.Hashes {
if cellCounts >= maxCells {
break
}
cell, _ := backend.BlobPool().GetCells(hash, query.Mask)
if len(cell) == 0 {
// skip this tx
continue
}
hashes = append(hashes, hash)
cells = append(cells, cell)
cellCounts += len(cell)
}
return hashes, cells, query.Mask
}
func handleCells(backend Backend, msg Decoder, peer *Peer) error {
var cellsResponse CellsPacket
if err := msg.Decode(&cellsResponse); err != nil {
return err
}
tresp := tracker.Response{
ID: cellsResponse.RequestId,
MsgCode: CellsMsg,
Size: len(cellsResponse.CellsResponse.Hashes),
}
if err := peer.tracker.Fulfil(tresp); err != nil {
return fmt.Errorf("Cells: %w", err)
}
return backend.Handle(peer, &cellsResponse.CellsResponse)
}

2
eth/protocols/eth/handshake_test.go
View file

@ -77,7 +77,7 @@ func testHandshake(t *testing.T, protocol uint) {
defer app.Close()
defer net.Close()
peer := NewPeer(protocol, p2p.NewPeer(enode.ID{}, "peer", nil), net, nil)
peer := NewPeer(protocol, p2p.NewPeer(enode.ID{}, "peer", nil), net, nil, nil)
defer peer.Close()
// Send the junk test with one peer, check the handshake failure

49
eth/protocols/eth/peer.go
View file

@ -24,6 +24,7 @@ import (
mapset "github.com/deckarep/golang-set/v2"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/tracker"
"github.com/ethereum/go-ethereum/rlp"
@ -53,7 +54,8 @@ type Peer struct {
version uint // Protocol version negotiated
lastRange atomic.Pointer[BlockRangeUpdatePacket]
txpool TxPool // Transaction pool used by the broadcasters for liveness checks
txpool TxPool // Transaction pool used by the broadcasters for liveness checks
blobpool BlobPool
knownTxs *knownCache // Set of transaction hashes known to be known by this peer
txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests
txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests
@ -68,7 +70,7 @@ type Peer struct {
// NewPeer creates a wrapper for a network connection and negotiated protocol
// version.
func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Peer {
func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool, blobpool BlobPool) *Peer {
cap := p2p.Cap{Name: ProtocolName, Version: version}
id := p.ID().String()
peer := &Peer{
@ -84,6 +86,7 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe
reqCancel: make(chan *cancel),
resDispatch: make(chan *response),
txpool: txpool,
blobpool: blobpool,
term: make(chan struct{}),
}
// Start up all the broadcasters
@ -166,10 +169,13 @@ func (p *Peer) AsyncSendTransactions(hashes []common.Hash) {
// This method is a helper used by the async transaction announcer. Don't call it
// directly as the queueing (memory) and transmission (bandwidth) costs should
// not be managed directly.
func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32) error {
func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash, types []byte, sizes []uint32, cells types.CustodyBitmap) error {
// Mark all the transactions as known, but ensure we don't overflow our limits
p.knownTxs.Add(hashes...)
return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket{Types: types, Sizes: sizes, Hashes: hashes})
if p.version >= ETH71 {
return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket71{Types: types, Sizes: sizes, Hashes: hashes, Mask: cells})
}
return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket70{Types: types, Sizes: sizes, Hashes: hashes})
}
// AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually
@ -222,6 +228,41 @@ func (p *Peer) ReplyReceiptsRLP(id uint64, receipts rlp.RawList[*ReceiptList]) e
})
}
// ReplyCells is the response to GetCells.
func (p *Peer) ReplyCells(id uint64, hashes []common.Hash, cells [][]kzg4844.Cell, mask types.CustodyBitmap) error {
return p2p.Send(p.rw, CellsMsg, &CellsPacket{
RequestId: id,
CellsResponse: CellsResponse{
Hashes: hashes,
Cells: cells,
Mask: mask,
},
})
}
// RequestPayload fetches a batch of cells from a remote node.
func (p *Peer) RequestPayload(hashes []common.Hash, cell *types.CustodyBitmap) error {
p.Log().Debug("Fetching batch of cells", "txcount", len(hashes), "cellcount", cell.OneCount())
id := rand.Uint64()
err := p.tracker.Track(tracker.Request{
ID: id,
ReqCode: GetCellsMsg,
RespCode: CellsMsg,
Size: len(hashes),
})
if err != nil {
return err
}
return p2p.Send(p.rw, GetCellsMsg, &GetCellsRequestPacket{
RequestId: id,
GetCellsRequest: GetCellsRequest{
Hashes: hashes,
Mask: *cell,
},
})
}
// RequestOneHeader is a wrapper around the header query functions to fetch a
// single header. It is used solely by the fetcher.
func (p *Peer) RequestOneHeader(hash common.Hash, sink chan *Response) (*Request, error) {

2
eth/protocols/eth/peer_test.go
View file

@ -45,7 +45,7 @@ func newTestPeer(name string, version uint, backend Backend) (*testPeer, <-chan
var id enode.ID
rand.Read(id[:])
peer := NewPeer(version, p2p.NewPeer(id, name, nil), net, backend.TxPool())
peer := NewPeer(version, p2p.NewPeer(id, name, nil), net, backend.TxPool(), backend.BlobPool())
errc := make(chan error, 1)
go func() {
defer app.Close()

59
eth/protocols/eth/protocol.go
View file

@ -24,12 +24,14 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/forkid"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto/kzg4844"
"github.com/ethereum/go-ethereum/rlp"
)
// Constants to match up protocol versions and messages
const (
ETH69 = 69
ETH71 = 71
)
// ProtocolName is the official short name of the `eth` protocol used during
@ -38,11 +40,11 @@ const ProtocolName = "eth"
// ProtocolVersions are the supported versions of the `eth` protocol (first
// is primary).
var ProtocolVersions = []uint{ETH69}
var ProtocolVersions = []uint{ETH71, ETH69}
// protocolLengths are the number of implemented message corresponding to
// different protocol versions.
var protocolLengths = map[uint]uint64{ETH69: 18}
var protocolLengths = map[uint]uint64{ETH69: 18, ETH71: 20}
// maxMessageSize is the maximum cap on the size of a protocol message.
const maxMessageSize = 10 * 1024 * 1024
@ -65,6 +67,8 @@ const (
GetReceiptsMsg = 0x0f
ReceiptsMsg = 0x10
BlockRangeUpdateMsg = 0x11
GetCellsMsg = 0x12
CellsMsg = 0x13
)
var (
@ -230,13 +234,22 @@ type ReceiptsPacket struct {
// ReceiptsRLPResponse is used for receipts, when we already have it encoded
type ReceiptsRLPResponse []rlp.RawValue
// NewPooledTransactionHashesPacket represents a transaction announcement packet on eth/68 and newer.
type NewPooledTransactionHashesPacket struct {
// NewPooledTransactionHashesPacket70 represents a transaction announcement packet on eth/69.
type NewPooledTransactionHashesPacket70 struct {
Types []byte
Sizes []uint32
Hashes []common.Hash
}
// NewPooledTransactionHashesPacket71 represents a transaction announcement packet on eth/71
// with an additional custody bitmap field for cell-based blob data availability.
type NewPooledTransactionHashesPacket71 struct {
Types []byte
Sizes []uint32
Hashes []common.Hash
Mask types.CustodyBitmap
}
// GetPooledTransactionsRequest represents a transaction query.
type GetPooledTransactionsRequest []common.Hash
@ -273,6 +286,31 @@ type BlockRangeUpdatePacket struct {
LatestBlockHash common.Hash
}
// GetCellsRequest represents a request for cells of blob transactions.
type GetCellsRequest struct {
Hashes []common.Hash
Mask types.CustodyBitmap
}
// GetCellsRequestPacket represents a cell request with request ID wrapping.
type GetCellsRequestPacket struct {
RequestId uint64
GetCellsRequest
}
// CellsResponse represents a response containing cells for blob transactions.
type CellsResponse struct {
Hashes []common.Hash
Cells [][]kzg4844.Cell
Mask types.CustodyBitmap
}
// CellsPacket represents a cells response with request ID wrapping.
type CellsPacket struct {
RequestId uint64
CellsResponse
}
func (*StatusPacket) Name() string { return "Status" }
func (*StatusPacket) Kind() byte { return StatusMsg }
@ -291,8 +329,11 @@ func (*GetBlockBodiesRequest) Kind() byte { return GetBlockBodiesMsg }
func (*BlockBodiesResponse) Name() string { return "BlockBodies" }
func (*BlockBodiesResponse) Kind() byte { return BlockBodiesMsg }
func (*NewPooledTransactionHashesPacket) Name() string { return "NewPooledTransactionHashes" }
func (*NewPooledTransactionHashesPacket) Kind() byte { return NewPooledTransactionHashesMsg }
func (*NewPooledTransactionHashesPacket70) Name() string { return "NewPooledTransactionHashes" }
func (*NewPooledTransactionHashesPacket70) Kind() byte { return NewPooledTransactionHashesMsg }
func (*NewPooledTransactionHashesPacket71) Name() string { return "NewPooledTransactionHashes" }
func (*NewPooledTransactionHashesPacket71) Kind() byte { return NewPooledTransactionHashesMsg }
func (*GetPooledTransactionsRequest) Name() string { return "GetPooledTransactions" }
func (*GetPooledTransactionsRequest) Kind() byte { return GetPooledTransactionsMsg }
@ -311,3 +352,9 @@ func (*ReceiptsRLPResponse) Kind() byte { return ReceiptsMsg }
func (*BlockRangeUpdatePacket) Name() string { return "BlockRangeUpdate" }
func (*BlockRangeUpdatePacket) Kind() byte { return BlockRangeUpdateMsg }
func (*GetCellsRequest) Name() string { return "GetCells" }
func (*GetCellsRequest) Kind() byte { return GetCellsMsg }
func (*CellsResponse) Name() string { return "Cells" }
func (*CellsResponse) Kind() byte { return CellsMsg }

4
eth/sync_test.go
View file

@ -50,8 +50,8 @@ func testSnapSyncDisabling(t *testing.T, ethVer uint, snapVer uint) {
defer emptyPipeEth.Close()
defer fullPipeEth.Close()
emptyPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{1}, "", caps), emptyPipeEth, empty.txpool)
fullPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{2}, "", caps), fullPipeEth, full.txpool)
emptyPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{1}, "", caps), emptyPipeEth, empty.txpool, empty.blobpool)
fullPeerEth := eth.NewPeer(ethVer, p2p.NewPeer(enode.ID{2}, "", caps), fullPipeEth, full.txpool, full.blobpool)
defer emptyPeerEth.Close()
defer fullPeerEth.Close()

2
tests/fuzzers/txfetcher/txfetcher_fuzzer.go
View file

@ -139,7 +139,7 @@ func fuzz(input []byte) int {
if verbose {
fmt.Println("Notify", peer, announceIdxs)
}
if err := f.Notify(peer, types, sizes, announces); err != nil {
if _, err := f.Notify(peer, types, sizes, announces); err != nil {
panic(err)
}