sync GPO changes (#954)

* feat: congestion-aware gas price oracle (#790)

* Implement functionality to reset gas price / suggested tip to minimal value when there's no congestion

* Add flags to configure congestion value and initialize gas price oracle accordingly

* Fix and add tests to make sure GPO works as expected depending on pre- or post-Curie (EIP 1559) upgrade

* Apply review suggestions

* chore: auto version bump [bot]

---------

Co-authored-by: omerfirmak <omerfirmak@users.noreply.github.com>

* fix eth/gasprice/gasprice_test.go

* minor

* fix(GPO): min suggested tip cap if there's congestion to avoid filtering through `DefaultIgnorePrice` (#883)

* fix tests

* fix tests

---------

Co-authored-by: Jonas Theis <4181434+jonastheis@users.noreply.github.com>
Co-authored-by: omerfirmak <omerfirmak@users.noreply.github.com>

cmd/geth/main.go

@@ -145,6 +145,7 @@ var (
 		utils.GpoPercentileFlag,
 		utils.GpoMaxGasPriceFlag,
 		utils.GpoIgnoreGasPriceFlag,
+		utils.GpoCongestionThresholdFlag,
 		configFileFlag,
 		utils.L1EndpointFlag,
 		utils.L1ConfirmationsFlag,

cmd/utils/flags.go

@@ -883,6 +883,12 @@ var (
 		Value:    ethconfig.Defaults.GPO.IgnorePrice.Int64(),
 		Category: flags.GasPriceCategory,
 	}
+	GpoCongestionThresholdFlag = &cli.IntFlag{
+		Name:     "gpo.congestionthreshold",
+		Usage:    "Number of pending transactions to consider the network congested and suggest a minimum tip cap",
+		Value:    ethconfig.Defaults.GPO.CongestedThreshold,
+		Category: flags.GasPriceCategory,
+	}
 
 	// Metrics flags
 	MetricsEnabledFlag = &cli.BoolFlag{
@@ -1640,6 +1646,9 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config, light bool) {
 	if ctx.IsSet(GpoIgnoreGasPriceFlag.Name) {
 		cfg.IgnorePrice = big.NewInt(ctx.Int64(GpoIgnoreGasPriceFlag.Name))
 	}
+	if ctx.IsSet(GpoCongestionThresholdFlag.Name) {
+		cfg.CongestedThreshold = ctx.Int(GpoCongestionThresholdFlag.Name)
+	}
 }
 
 func setTxPool(ctx *cli.Context, cfg *legacypool.Config) {

core/txpool/blobpool/blobpool.go

@@ -1538,6 +1538,10 @@ func (p *BlobPool) Stats() (int, int) {
 	return pending, 0 // No non-executable txs in the blob pool
 }
 
+func (p *BlobPool) StatsWithMinBaseFee(minBaseFee *big.Int) (int, int) {
+	return p.Stats()
+}
+
 // Content retrieves the data content of the transaction pool, returning all the
 // pending as well as queued transactions, grouped by account and sorted by nonce.
 //

core/txpool/legacypool/legacypool.go

@@ -483,6 +483,40 @@ func (pool *LegacyPool) stats() (int, int) {
 	return pending, queued
 }
 
+// StatsWithMinBaseFee retrieves the current pool stats, namely the number of pending and the
+// number of queued (non-executable) transactions greater equal minBaseFee.
+func (pool *LegacyPool) StatsWithMinBaseFee(minBaseFee *big.Int) (int, int) {
+	pool.mu.RLock()
+	defer pool.mu.RUnlock()
+
+	return pool.statsWithMinBaseFee(minBaseFee)
+}
+
+// statsWithMinBaseFee retrieves the current pool stats, namely the number of pending and the
+// number of queued (non-executable) transactions greater equal minBaseFee.
+func (pool *LegacyPool) statsWithMinBaseFee(minBaseFee *big.Int) (int, int) {
+	pending := 0
+	for _, list := range pool.pending {
+		for _, tx := range list.txs.flatten() {
+			if _, err := tx.EffectiveGasTip(minBaseFee); err != nil {
+				break // basefee too low, discard rest of txs with higher nonces from the account
+			}
+			pending++
+		}
+	}
+
+	queued := 0
+	for _, list := range pool.queue {
+		for _, tx := range list.txs.flatten() {
+			if _, err := tx.EffectiveGasTip(minBaseFee); err != nil {
+				break // basefee too low, discard rest of txs with higher nonces from the account
+			}
+			queued++
+		}
+	}
+	return pending, queued
+}
+
 // Content retrieves the data content of the transaction pool, returning all the
 // pending as well as queued transactions, grouped by account and sorted by nonce.
 func (pool *LegacyPool) Content() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) {

core/txpool/legacypool/legacypool_test.go

@@ -2654,3 +2654,74 @@ func TestPoolPending(t *testing.T) {
 	maxAccounts := 10
 	assert.Len(t, pool.PendingWithMax(false, maxAccounts), maxAccounts)
 }
+
+func TestStatsWithMinBaseFee(t *testing.T) {
+	// Create the pool to test the pricing enforcement with
+	pool, _ := setupPool()
+	defer pool.Close()
+
+	// Keep track of transaction events to ensure all executables get announced
+	events := make(chan core.NewTxsEvent, 32)
+	sub := pool.txFeed.Subscribe(events)
+	defer sub.Unsubscribe()
+
+	// Create a number of test accounts and fund them
+	keys := make([]*ecdsa.PrivateKey, 4)
+	for i := 0; i < len(keys); i++ {
+		keys[i], _ = crypto.GenerateKey()
+		testAddBalance(pool, crypto.PubkeyToAddress(keys[i].PublicKey), big.NewInt(1000000))
+	}
+	// Generate and queue a batch of transactions, both pending and queued
+	txs := types.Transactions{}
+
+	txs = append(txs, pricedTransaction(0, 100000, big.NewInt(5), keys[0])) // will stay pending
+	txs = append(txs, pricedTransaction(1, 100000, big.NewInt(1), keys[0]))
+	txs = append(txs, pricedTransaction(2, 100000, big.NewInt(2), keys[0]))
+
+	txs = append(txs, dynamicFeeTx(0, 100000, big.NewInt(5), big.NewInt(1), keys[1])) // will stay pending
+	txs = append(txs, dynamicFeeTx(1, 100000, big.NewInt(3), big.NewInt(2), keys[1])) // will stay pending
+	txs = append(txs, dynamicFeeTx(2, 100000, big.NewInt(2), big.NewInt(1), keys[1]))
+	txs = append(txs, dynamicFeeTx(3, 100000, big.NewInt(4), big.NewInt(1), keys[1]))
+
+	localTx := dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[3])
+
+	// queued
+	txs = append(txs, dynamicFeeTx(1, 100000, big.NewInt(3), big.NewInt(2), keys[2])) // will stay queued
+	txs = append(txs, dynamicFeeTx(2, 100000, big.NewInt(1), big.NewInt(1), keys[2]))
+	txs = append(txs, dynamicFeeTx(3, 100000, big.NewInt(2), big.NewInt(2), keys[2]))
+
+	// Import the batch and that both pending and queued transactions match up
+	pool.addRemotesSync(txs)
+	pool.addLocal(localTx)
+
+	minBaseFee := big.NewInt(3)
+	pool.priced.SetBaseFee(minBaseFee)
+
+	// Check pool.Stats(), all tx should be counted
+	{
+		pending, queued := pool.Stats()
+		if pending != 8 {
+			t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 8)
+		}
+		if queued != 3 {
+			t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3)
+		}
+		if err := validateEvents(events, 8); err != nil {
+			t.Fatalf("original event firing failed: %v", err)
+		}
+		if err := validatePoolInternals(pool); err != nil {
+			t.Fatalf("pool internal state corrupted: %v", err)
+		}
+	}
+
+	// Check pool.StatsWithMinBaseFee(), only tx with base fee >= minBaseFee should be counted
+	{
+		pending, queued := pool.StatsWithMinBaseFee(minBaseFee)
+		if pending != 3 {
+			t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 3)
+		}
+		if queued != 1 {
+			t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 1)
+		}
+	}
+}

core/txpool/subpool.go

@@ -125,6 +125,8 @@ type SubPool interface {
 	// number of queued (non-executable) transactions.
 	Stats() (int, int)
 
+	StatsWithMinBaseFee(minBaseFee *big.Int) (pending int, queued int)
+
 	// Content retrieves the data content of the transaction pool, returning all the
 	// pending as well as queued transactions, grouped by account and sorted by nonce.
 	Content() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction)

core/txpool/txpool.go

@@ -446,3 +446,12 @@ func (pool *TxPool) ResumeReorgs() {
 		subpool.ResumeReorgs()
 	}
 }
+
+func (pool *TxPool) StatsWithMinBaseFee(minBaseFee *big.Int) (pending int, queued int) {
+	for _, subpool := range pool.subpools {
+		p, q := subpool.StatsWithMinBaseFee(minBaseFee)
+		pending += p
+		queued += q
+	}
+	return pending, queued
+}

eth/api_backend.go

@@ -344,6 +344,10 @@ func (b *EthAPIBackend) Stats() (runnable int, blocked int) {
 	return b.eth.txPool.Stats()
 }
 
+func (b *EthAPIBackend) StatsWithMinBaseFee(minBaseFee *big.Int) (pending int, queued int) {
+	return b.eth.txPool.StatsWithMinBaseFee(minBaseFee)
+}
+
 func (b *EthAPIBackend) TxPoolContent() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) {
 	return b.eth.txPool.Content()
 }

eth/backend.go

@@ -282,6 +282,7 @@ func New(stack *node.Node, config *ethconfig.Config, l1Client sync_service.EthCl
 	if gpoParams.Default == nil {
 		gpoParams.Default = config.Miner.GasPrice
 	}
+	gpoParams.DefaultBasePrice = new(big.Int).SetUint64(config.TxPool.PriceLimit)
 	eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams)
 
 	// Setup DNS discovery iterators.

eth/gasprice/feehistory_test.go

@@ -58,7 +58,7 @@ func TestFeeHistory(t *testing.T) {
 			MaxHeaderHistory: c.maxHeader,
 			MaxBlockHistory:  c.maxBlock,
 		}
-		backend := newTestBackend(t, big.NewInt(16), c.pending)
+		backend := newTestBackend(t, big.NewInt(16), c.pending, 0)
 		oracle := NewOracle(backend, config)
 
 		first, reward, baseFee, ratio, err := oracle.FeeHistory(context.Background(), c.count, c.last, c.percent)

eth/gasprice/gasprice.go

@@ -37,7 +37,8 @@ const sampleNumber = 3 // Number of transactions sampled in a block
 
 var (
 	DefaultMaxPrice    = big.NewInt(500 * params.GWei)
-	DefaultIgnorePrice = big.NewInt(2 * params.Wei)
+	DefaultIgnorePrice = big.NewInt(1 * params.Wei)
+	DefaultBasePrice   = big.NewInt(0)
 )
 
 type Config struct {
@@ -48,6 +49,9 @@ type Config struct {
 	Default          *big.Int `toml:",omitempty"`
 	MaxPrice         *big.Int `toml:",omitempty"`
 	IgnorePrice      *big.Int `toml:",omitempty"`
+
+	CongestedThreshold int      // Number of pending transactions to consider the network congested and suggest a minimum tip cap.
+	DefaultBasePrice   *big.Int `toml:",omitempty"` // Base price to set when CongestedThreshold is reached before Curie (EIP 1559).
 }
 
 // OracleBackend includes all necessary background APIs for oracle.
@@ -59,6 +63,8 @@ type OracleBackend interface {
 	ChainConfig() *params.ChainConfig
 	SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription
 	StateAt(root common.Hash) (*state.StateDB, error)
+	Stats() (pending int, queued int)
+	StatsWithMinBaseFee(minBaseFee *big.Int) (pending int, queued int)
 }
 
 // Oracle recommends gas prices based on the content of recent
@@ -76,6 +82,9 @@ type Oracle struct {
 	maxHeaderHistory, maxBlockHistory uint64
 
 	historyCache *lru.Cache[cacheKey, processedFees]
+
+	congestedThreshold int      // Number of pending transactions to consider the network congested and suggest a minimum tip cap.
+	defaultBasePrice   *big.Int // Base price to set when CongestedThreshold is reached before Curie (EIP 1559).
 }
 
 // NewOracle returns a new gasprice oracle which can recommend suitable
@@ -116,6 +125,16 @@ func NewOracle(backend OracleBackend, params Config) *Oracle {
 		maxBlockHistory = 1
 		log.Warn("Sanitizing invalid gasprice oracle max block history", "provided", params.MaxBlockHistory, "updated", maxBlockHistory)
 	}
+	congestedThreshold := params.CongestedThreshold
+	if congestedThreshold < 0 {
+		congestedThreshold = 0
+		log.Warn("Sanitizing invalid gasprice oracle congested threshold", "provided", params.CongestedThreshold, "updated", congestedThreshold)
+	}
+	defaultBasePrice := params.DefaultBasePrice
+	if defaultBasePrice == nil || defaultBasePrice.Int64() < 0 {
+		defaultBasePrice = DefaultBasePrice
+		log.Warn("Sanitizing invalid gasprice oracle default base price", "provided", params.DefaultBasePrice, "updated", defaultBasePrice)
+	}
 
 	cache := lru.NewCache[cacheKey, processedFees](2048)
 	headEvent := make(chan core.ChainHeadEvent, 1)
@@ -131,15 +150,17 @@ func NewOracle(backend OracleBackend, params Config) *Oracle {
 	}()
 
 	return &Oracle{
-		backend:          backend,
-		lastPrice:        params.Default,
-		maxPrice:         maxPrice,
-		ignorePrice:      ignorePrice,
-		checkBlocks:      blocks,
-		percentile:       percent,
-		maxHeaderHistory: maxHeaderHistory,
-		maxBlockHistory:  maxBlockHistory,
-		historyCache:     cache,
+		backend:            backend,
+		lastPrice:          params.Default,
+		maxPrice:           maxPrice,
+		ignorePrice:        ignorePrice,
+		checkBlocks:        blocks,
+		percentile:         percent,
+		maxHeaderHistory:   maxHeaderHistory,
+		maxBlockHistory:    maxBlockHistory,
+		historyCache:       cache,
+		congestedThreshold: congestedThreshold,
+		defaultBasePrice:   defaultBasePrice,
 	}
 }
 
@@ -170,6 +191,31 @@ func (oracle *Oracle) SuggestTipCap(ctx context.Context) (*big.Int, error) {
 	if headHash == lastHead {
 		return new(big.Int).Set(lastPrice), nil
 	}
+
+	// If pending txs are less than oracle.congestedThreshold, we consider the network to be non-congested and suggest
+	// a minimal tip cap. This is to prevent users from overpaying for gas when the network is not congested and a few
+	// high-priced txs are causing the suggested tip cap to be high.
+	pendingTxCount, _ := oracle.backend.StatsWithMinBaseFee(head.BaseFee)
+	if pendingTxCount < oracle.congestedThreshold {
+		// Before Curie (EIP-1559), we need to return the total suggested gas price. After Curie we return 2 wei as the tip cap,
+		// as the base fee is set separately or added manually for legacy transactions.
+		// 1. Set price to at least 1 as otherwise tx with a 0 tip might be filtered out by the default mempool config.
+		// 2. Since oracle.ignoreprice was set to 2 (DefaultIgnorePrice) before by default, we need to set the price
+		//    to 2 to avoid filtering in oracle.getBlockValues() by nodes that did not yet update to this version.
+		//    In the future we can set the price to 1 wei.
+		price := big.NewInt(2)
+		if !oracle.backend.ChainConfig().IsCurie(head.Number) {
+			price = oracle.defaultBasePrice
+		}
+
+		oracle.cacheLock.Lock()
+		oracle.lastHead = headHash
+		oracle.lastPrice = price
+		oracle.cacheLock.Unlock()
+
+		return new(big.Int).Set(price), nil
+	}
+
 	var (
 		sent, exp int
 		number    = head.Number.Uint64()

eth/gasprice/gasprice_test.go

@@ -38,8 +38,9 @@ import (
 const testHead = 32
 
 type testBackend struct {
-	chain   *core.BlockChain
-	pending bool // pending block available
+	chain          *core.BlockChain
+	pending        bool // pending block available
+	pendingTxCount int
 }
 
 func (b *testBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) {
@@ -122,9 +123,17 @@ func (b *testBackend) StateAt(root common.Hash) (*state.StateDB, error) {
 	return b.chain.StateAt(root)
 }
 
+func (b *testBackend) Stats() (int, int) {
+	return b.pendingTxCount, 0
+}
+
+func (b *testBackend) StatsWithMinBaseFee(minBaseFee *big.Int) (int, int) {
+	return b.pendingTxCount, 0
+}
+
 // newTestBackend creates a test backend. OBS: don't forget to invoke tearDown
 // after use, otherwise the blockchain instance will mem-leak via goroutines.
-func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBackend {
+func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool, pendingTxCount int) *testBackend {
 	var (
 		key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		addr   = crypto.PubkeyToAddress(key.PublicKey)
@@ -189,7 +198,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke
 	chain.InsertChain(blocks)
 	chain.SetFinalized(chain.GetBlockByNumber(25).Header())
 	chain.SetSafe(chain.GetBlockByNumber(25).Header())
-	return &testBackend{chain: chain, pending: pending}
+	return &testBackend{chain: chain, pending: pending, pendingTxCount: pendingTxCount}
 }
 
 func (b *testBackend) CurrentHeader() *types.Header {
@@ -217,7 +226,67 @@ func TestSuggestTipCap(t *testing.T) {
 		{big.NewInt(33), big.NewInt(params.GWei * int64(30))}, // Fork point in the future
 	}
 	for _, c := range cases {
-		backend := newTestBackend(t, c.fork, false)
+		backend := newTestBackend(t, c.fork, false, 0)
+		oracle := NewOracle(backend, config)
+
+		// The gas price sampled is: 32G, 31G, 30G, 29G, 28G, 27G
+		got, err := oracle.SuggestTipCap(context.Background())
+		if err != nil {
+			t.Fatalf("Failed to retrieve recommended gas price: %v", err)
+		}
+		if got.Cmp(c.expect) != 0 {
+			t.Fatalf("Gas price mismatch, want %d, got %d", c.expect, got)
+		}
+	}
+}
+
+func TestSuggestTipCapCongestedThreshold(t *testing.T) {
+	expectedDefaultBasePricePreCurie := big.NewInt(2000)
+	expectedDefaultBasePricePostCurie := big.NewInt(2)
+
+	config := Config{
+		Blocks:             3,
+		Percentile:         60,
+		Default:            big.NewInt(params.GWei),
+		CongestedThreshold: 50,
+		DefaultBasePrice:   expectedDefaultBasePricePreCurie,
+	}
+	var cases = []struct {
+		fork      *big.Int // London fork number
+		pendingTx int      // Number of pending transactions in the mempool
+		expect    *big.Int // Expected gasprice suggestion
+	}{
+		{nil, 0, expectedDefaultBasePricePreCurie},      // No congestion - default base price
+		{nil, 49, expectedDefaultBasePricePreCurie},     // No congestion - default base price
+		{nil, 50, big.NewInt(params.GWei * int64(30))},  // Congestion - normal behavior
+		{nil, 100, big.NewInt(params.GWei * int64(30))}, // Congestion - normal behavior
+
+		// Fork point in genesis
+		{big.NewInt(0), 0, expectedDefaultBasePricePostCurie},     // No congestion - default base price
+		{big.NewInt(0), 49, expectedDefaultBasePricePostCurie},    // No congestion - default base price
+		{big.NewInt(0), 50, big.NewInt(params.GWei * int64(30))},  // Congestion - normal behavior
+		{big.NewInt(0), 100, big.NewInt(params.GWei * int64(30))}, // Congestion - normal behavior
+
+		// Fork point in first block
+		{big.NewInt(1), 0, expectedDefaultBasePricePostCurie},     // No congestion - default base price
+		{big.NewInt(1), 49, expectedDefaultBasePricePostCurie},    // No congestion - default base price
+		{big.NewInt(1), 50, big.NewInt(params.GWei * int64(30))},  // Congestion - normal behavior
+		{big.NewInt(1), 100, big.NewInt(params.GWei * int64(30))}, // Congestion - normal behavior
+
+		// Fork point in last block
+		{big.NewInt(32), 0, expectedDefaultBasePricePostCurie},     // No congestion - default base price
+		{big.NewInt(32), 49, expectedDefaultBasePricePostCurie},    // No congestion - default base price
+		{big.NewInt(32), 50, big.NewInt(params.GWei * int64(30))},  // Congestion - normal behavior
+		{big.NewInt(32), 100, big.NewInt(params.GWei * int64(30))}, // Congestion - normal behavior
+
+		// Fork point in the future
+		{big.NewInt(33), 0, expectedDefaultBasePricePreCurie},      // No congestion - default base price
+		{big.NewInt(33), 49, expectedDefaultBasePricePreCurie},     // No congestion - default base price
+		{big.NewInt(33), 50, big.NewInt(params.GWei * int64(30))},  // Congestion - normal behavior
+		{big.NewInt(33), 100, big.NewInt(params.GWei * int64(30))}, // Congestion - normal behavior
+	}
+	for _, c := range cases {
+		backend := newTestBackend(t, c.fork, false, c.pendingTx)
 		oracle := NewOracle(backend, config)
 
 		// The gas price sampled is: 32G, 31G, 30G, 29G, 28G, 27G

les/api_backend.go

@@ -225,6 +225,10 @@ func (b *LesApiBackend) Stats() (pending int, queued int) {
 	return b.eth.txPool.Stats(), 0
 }
 
+func (b *LesApiBackend) StatsWithMinBaseFee(minBaseFee *big.Int) (pending int, queued int) {
+	return b.eth.txPool.StatsWithMinBaseFee(minBaseFee), 0
+}
+
 func (b *LesApiBackend) TxPoolContent() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) {
 	return b.eth.txPool.Content()
 }

light/txpool.go

@@ -351,6 +351,20 @@ func (pool *TxPool) Stats() (pending int) {
 	return
 }
 
+// StatsWithMinBaseFee returns the number of currently pending (locally created) transactions and ignores the base fee.
+func (pool *TxPool) StatsWithMinBaseFee(minBaseFee *big.Int) (pending int) {
+	pool.mu.RLock()
+	defer pool.mu.RUnlock()
+
+	for _, tx := range pool.pending {
+		if _, err := tx.EffectiveGasTip(minBaseFee); err == nil {
+			pending++
+		}
+	}
+
+	return pending
+}
+
 // validateTx checks whether a transaction is valid according to the consensus rules.
 func (pool *TxPool) validateTx(ctx context.Context, tx *types.Transaction) error {
 	// Validate sender