core/txpool/blobpool: improve gapped queue observability

The gapped transaction queue had limited observability: addGappedMeter
fired before the allowance/maxGapped check, so it counted all nonce-
too-high attempts rather than actual additions.

This change:
- Moves addGappedMeter to fire only after a tx is actually added to
  the gapped queue
- Adds addGappedRejectMeter for rejected gapped txs (no allowance or
  queue full)
- Adds gappedPromotedMeter for txs promoted from gapped queue to main
  pool
- Adds gappedStaleMeter for txs that became stale during promotion
- Adds gappedEvictedMeter for txs evicted by timeout or nonce staleness
- Adds gappedGauge to track the current gapped queue size

Fixes #33811

core/txpool/blobpool/blobpool.go

@@ -1592,13 +1592,14 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
 		case errors.Is(err, core.ErrNonceTooLow):
 			addStaleMeter.Mark(1)
 		case errors.Is(err, core.ErrNonceTooHigh):
-			addGappedMeter.Mark(1)
 			// Store the tx in memory, and revalidate later
 			from, _ := types.Sender(p.signer, tx)
 			allowance := p.gappedAllowance(from)
 			if allowance >= 1 && len(p.gapped) < maxGapped {
 				p.gapped[from] = append(p.gapped[from], tx)
 				p.gappedSource[tx.Hash()] = from
+				addGappedMeter.Mark(1)
+				gappedGauge.Update(int64(len(p.gapped)))
 				log.Trace("added tx to gapped blob queue", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
 				return nil
 			} else {
@@ -1606,6 +1607,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
 				// transactions by keeping the old and dropping this one.
 				// Thus replacing a gapped transaction with another gapped transaction
 				// is discouraged.
+				addGappedRejectMeter.Mark(1)
 				log.Trace("no gapped blob queue allowance", "allowance", allowance, "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
 			}
 		case errors.Is(err, core.ErrInsufficientFunds):
@@ -1782,6 +1784,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
 
 			if tx.Nonce() < stateNonce {
 				// Stale, drop it. Eventually we could add to limbo here if hash matches.
+				gappedStaleMeter.Mark(1)
 				log.Trace("Gapped blob transaction became stale", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "state", stateNonce, "qlen", len(p.gapped[from]))
 				continue
 			}
@@ -1791,6 +1794,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
 				// We do not recurse here, but continue to loop instead.
 				// We are under lock, so we can add the transaction directly.
 				if err := p.addLocked(tx, false); err == nil {
+					gappedPromotedMeter.Mark(1)
 					log.Trace("Gapped blob transaction added to pool", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "qlen", len(p.gapped[from]))
 				} else {
 					log.Trace("Gapped blob transaction not accepted", "hash", tx.Hash(), "from", from, "nonce", tx.Nonce(), "err", err)
@@ -1802,6 +1806,7 @@ func (p *BlobPool) addLocked(tx *types.Transaction, checkGapped bool) (err error
 		} else {
 			p.gapped[from] = gtxs
 		}
+		gappedGauge.Update(int64(len(p.gapped)))
 	}
 	return nil
 }
@@ -2069,8 +2074,10 @@ func (p *BlobPool) evictGapped() {
 				keep = append(keep, gtx)
 			}
 		}
-		if len(keep) < len(txs) {
+		evicted := len(txs) - len(keep)
-			log.Trace("Evicting old gapped blob transactions", "count", len(txs)-len(keep), "from", from)
+		if evicted > 0 {
+			gappedEvictedMeter.Mark(int64(evicted))
+			log.Trace("Evicting old gapped blob transactions", "count", evicted, "from", from)
 		}
 		if len(keep) == 0 {
 			delete(p.gapped, from)
@@ -2078,6 +2085,7 @@ func (p *BlobPool) evictGapped() {
 			p.gapped[from] = keep
 		}
 	}
+	gappedGauge.Update(int64(len(p.gapped)))
 }
 
 // isAnnouncable checks whether a transaction is announcable based on its

core/txpool/blobpool/metrics.go

@@ -96,10 +96,17 @@ var (
 	addInvalidMeter      = metrics.NewRegisteredMeter("blobpool/add/invalid", nil)      // Invalid transaction, reject, neutral
 	addUnderpricedMeter  = metrics.NewRegisteredMeter("blobpool/add/underpriced", nil)  // Gas tip too low, neutral
 	addStaleMeter        = metrics.NewRegisteredMeter("blobpool/add/stale", nil)        // Nonce already filled, reject, bad-ish
-	addGappedMeter       = metrics.NewRegisteredMeter("blobpool/add/gapped", nil)       // Nonce gapped, reject, bad-ish
+	addGappedMeter       = metrics.NewRegisteredMeter("blobpool/add/gapped", nil)       // Nonce gapped, added to gapped queue
+	addGappedRejectMeter = metrics.NewRegisteredMeter("blobpool/add/gappedreject", nil) // Nonce gapped, rejected (queue full or no allowance)
 	addOverdraftedMeter  = metrics.NewRegisteredMeter("blobpool/add/overdrafted", nil)  // Balance exceeded, reject, neutral
 	addOvercappedMeter   = metrics.NewRegisteredMeter("blobpool/add/overcapped", nil)   // Per-account cap exceeded, reject, neutral
 	addNoreplaceMeter    = metrics.NewRegisteredMeter("blobpool/add/noreplace", nil)    // Replacement fees or tips too low, neutral
 	addNonExclusiveMeter = metrics.NewRegisteredMeter("blobpool/add/nonexclusive", nil) // Plain transaction from same account exists, reject, neutral
 	addValidMeter        = metrics.NewRegisteredMeter("blobpool/add/valid", nil)        // Valid transaction, add, neutral
+
+	// The below metrics track the gapped transaction reorder buffer.
+	gappedPromotedMeter = metrics.NewRegisteredMeter("blobpool/gapped/promoted", nil) // Gapped tx promoted to main pool
+	gappedStaleMeter    = metrics.NewRegisteredMeter("blobpool/gapped/stale", nil)    // Gapped tx became stale during promotion
+	gappedEvictedMeter  = metrics.NewRegisteredMeter("blobpool/gapped/evicted", nil)  // Gapped tx evicted (timeout or nonce stale)
+	gappedGauge         = metrics.NewRegisteredGauge("blobpool/gapped/count", nil)    // Current gapped queue size
 )