Merge branch 'ethereum:master' into master

.mailmap

@@ -56,7 +56,6 @@ Diederik Loerakker <proto@protolambda.com>
 Dimitry Khokhlov <winsvega@mail.ru>
 
 Domino Valdano <dominoplural@gmail.com>
-Domino Valdano <dominoplural@gmail.com> <jeff@okcupid.com>
 
 Edgar Aroutiounian <edgar.factorial@gmail.com>
 

accounts/abi/type.go

@@ -179,9 +179,6 @@ func NewType(t string, internalType string, components []ArgumentMarshaling) (ty
 				return Type{}, errors.New("abi: purely anonymous or underscored field is not supported")
 			}
 			fieldName := ResolveNameConflict(name, func(s string) bool { return used[s] })
-			if err != nil {
-				return Type{}, err
-			}
 			used[fieldName] = true
 			if !isValidFieldName(fieldName) {
 				return Type{}, fmt.Errorf("field %d has invalid name", idx)

accounts/accounts.go

@@ -195,7 +195,7 @@ func TextHash(data []byte) []byte {
 //
 // This gives context to the signed message and prevents signing of transactions.
 func TextAndHash(data []byte) ([]byte, string) {
-	msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), string(data))
+	msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data)
 	hasher := sha3.NewLegacyKeccak256()
 	hasher.Write([]byte(msg))
 	return hasher.Sum(nil), msg

accounts/keystore/account_cache_test.go

@@ -86,7 +86,7 @@ func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error {
 func TestWatchNewFile(t *testing.T) {
 	t.Parallel()
 
-	dir, ks := tmpKeyStore(t, false)
+	dir, ks := tmpKeyStore(t)
 
 	// Ensure the watcher is started before adding any files.
 	ks.Accounts()

accounts/keystore/keystore.go

@@ -87,15 +87,6 @@ func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore {
 	return ks
 }
 
-// NewPlaintextKeyStore creates a keystore for the given directory.
-// Deprecated: Use NewKeyStore.
-func NewPlaintextKeyStore(keydir string) *KeyStore {
-	keydir, _ = filepath.Abs(keydir)
-	ks := &KeyStore{storage: &keyStorePlain{keydir}}
-	ks.init(keydir)
-	return ks
-}
-
 func (ks *KeyStore) init(keydir string) {
 	// Lock the mutex since the account cache might call back with events
 	ks.mu.Lock()

accounts/keystore/keystore_test.go

@@ -37,7 +37,7 @@ var testSigData = make([]byte, 32)
 
 func TestKeyStore(t *testing.T) {
 	t.Parallel()
-	dir, ks := tmpKeyStore(t, true)
+	dir, ks := tmpKeyStore(t)
 
 	a, err := ks.NewAccount("foo")
 	if err != nil {
@@ -72,7 +72,7 @@ func TestKeyStore(t *testing.T) {
 
 func TestSign(t *testing.T) {
 	t.Parallel()
-	_, ks := tmpKeyStore(t, true)
+	_, ks := tmpKeyStore(t)
 
 	pass := "" // not used but required by API
 	a1, err := ks.NewAccount(pass)
@@ -89,7 +89,7 @@ func TestSign(t *testing.T) {
 
 func TestSignWithPassphrase(t *testing.T) {
 	t.Parallel()
-	_, ks := tmpKeyStore(t, true)
+	_, ks := tmpKeyStore(t)
 
 	pass := "passwd"
 	acc, err := ks.NewAccount(pass)
@@ -117,7 +117,7 @@ func TestSignWithPassphrase(t *testing.T) {
 
 func TestTimedUnlock(t *testing.T) {
 	t.Parallel()
-	_, ks := tmpKeyStore(t, true)
+	_, ks := tmpKeyStore(t)
 
 	pass := "foo"
 	a1, err := ks.NewAccount(pass)
@@ -152,7 +152,7 @@ func TestTimedUnlock(t *testing.T) {
 
 func TestOverrideUnlock(t *testing.T) {
 	t.Parallel()
-	_, ks := tmpKeyStore(t, false)
+	_, ks := tmpKeyStore(t)
 
 	pass := "foo"
 	a1, err := ks.NewAccount(pass)
@@ -193,7 +193,7 @@ func TestOverrideUnlock(t *testing.T) {
 // This test should fail under -race if signing races the expiration goroutine.
 func TestSignRace(t *testing.T) {
 	t.Parallel()
-	_, ks := tmpKeyStore(t, false)
+	_, ks := tmpKeyStore(t)
 
 	// Create a test account.
 	a1, err := ks.NewAccount("")
@@ -238,7 +238,7 @@ func waitForKsUpdating(t *testing.T, ks *KeyStore, wantStatus bool, maxTime time
 func TestWalletNotifierLifecycle(t *testing.T) {
 	t.Parallel()
 	// Create a temporary keystore to test with
-	_, ks := tmpKeyStore(t, false)
+	_, ks := tmpKeyStore(t)
 
 	// Ensure that the notification updater is not running yet
 	time.Sleep(250 * time.Millisecond)
@@ -284,7 +284,7 @@ type walletEvent struct {
 // or deleted from the keystore.
 func TestWalletNotifications(t *testing.T) {
 	t.Parallel()
-	_, ks := tmpKeyStore(t, false)
+	_, ks := tmpKeyStore(t)
 
 	// Subscribe to the wallet feed and collect events.
 	var (
@@ -346,7 +346,7 @@ func TestWalletNotifications(t *testing.T) {
 // TestImportExport tests the import functionality of a keystore.
 func TestImportECDSA(t *testing.T) {
 	t.Parallel()
-	_, ks := tmpKeyStore(t, true)
+	_, ks := tmpKeyStore(t)
 	key, err := crypto.GenerateKey()
 	if err != nil {
 		t.Fatalf("failed to generate key: %v", key)
@@ -365,7 +365,7 @@ func TestImportECDSA(t *testing.T) {
 // TestImportECDSA tests the import and export functionality of a keystore.
 func TestImportExport(t *testing.T) {
 	t.Parallel()
-	_, ks := tmpKeyStore(t, true)
+	_, ks := tmpKeyStore(t)
 	acc, err := ks.NewAccount("old")
 	if err != nil {
 		t.Fatalf("failed to create account: %v", acc)
@@ -374,7 +374,7 @@ func TestImportExport(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to export account: %v", acc)
 	}
-	_, ks2 := tmpKeyStore(t, true)
+	_, ks2 := tmpKeyStore(t)
 	if _, err = ks2.Import(json, "old", "old"); err == nil {
 		t.Errorf("importing with invalid password succeeded")
 	}
@@ -394,7 +394,7 @@ func TestImportExport(t *testing.T) {
 // This test should fail under -race if importing races.
 func TestImportRace(t *testing.T) {
 	t.Parallel()
-	_, ks := tmpKeyStore(t, true)
+	_, ks := tmpKeyStore(t)
 	acc, err := ks.NewAccount("old")
 	if err != nil {
 		t.Fatalf("failed to create account: %v", acc)
@@ -403,7 +403,7 @@ func TestImportRace(t *testing.T) {
 	if err != nil {
 		t.Fatalf("failed to export account: %v", acc)
 	}
-	_, ks2 := tmpKeyStore(t, true)
+	_, ks2 := tmpKeyStore(t)
 	var atom atomic.Uint32
 	var wg sync.WaitGroup
 	wg.Add(2)
@@ -457,11 +457,7 @@ func checkEvents(t *testing.T, want []walletEvent, have []walletEvent) {
 	}
 }
 
-func tmpKeyStore(t *testing.T, encrypted bool) (string, *KeyStore) {
+func tmpKeyStore(t *testing.T) (string, *KeyStore) {
 	d := t.TempDir()
-	newKs := NewPlaintextKeyStore
+	return d, NewKeyStore(d, veryLightScryptN, veryLightScryptP)
-	if encrypted {
-		newKs = func(kd string) *KeyStore { return NewKeyStore(kd, veryLightScryptN, veryLightScryptP) }
-	}
-	return d, newKs(d)
 }

accounts/usbwallet/hub.go

@@ -26,7 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/karalabe/usb"
+	"github.com/karalabe/hid"
 )
 
 // LedgerScheme is the protocol scheme prefixing account and wallet URLs.
@@ -109,7 +109,7 @@ func NewTrezorHubWithWebUSB() (*Hub, error) {
 
 // newHub creates a new hardware wallet manager for generic USB devices.
 func newHub(scheme string, vendorID uint16, productIDs []uint16, usageID uint16, endpointID int, makeDriver func(log.Logger) driver) (*Hub, error) {
-	if !usb.Supported() {
+	if !hid.Supported() {
 		return nil, errors.New("unsupported platform")
 	}
 	hub := &Hub{
@@ -155,7 +155,7 @@ func (hub *Hub) refreshWallets() {
 		return
 	}
 	// Retrieve the current list of USB wallet devices
-	var devices []usb.DeviceInfo
+	var devices []hid.DeviceInfo
 
 	if runtime.GOOS == "linux" {
 		// hidapi on Linux opens the device during enumeration to retrieve some infos,
@@ -170,7 +170,7 @@ func (hub *Hub) refreshWallets() {
 			return
 		}
 	}
-	infos, err := usb.Enumerate(hub.vendorID, 0)
+	infos, err := hid.Enumerate(hub.vendorID, 0)
 	if err != nil {
 		failcount := hub.enumFails.Add(1)
 		if runtime.GOOS == "linux" {

accounts/usbwallet/wallet.go

@@ -31,7 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/karalabe/usb"
+	"github.com/karalabe/hid"
 )
 
 // Maximum time between wallet health checks to detect USB unplugs.
@@ -79,8 +79,8 @@ type wallet struct {
 	driver driver        // Hardware implementation of the low level device operations
 	url    *accounts.URL // Textual URL uniquely identifying this wallet
 
-	info   usb.DeviceInfo // Known USB device infos about the wallet
+	info   hid.DeviceInfo // Known USB device infos about the wallet
-	device usb.Device     // USB device advertising itself as a hardware wallet
+	device hid.Device     // USB device advertising itself as a hardware wallet
 
 	accounts []accounts.Account                         // List of derive accounts pinned on the hardware wallet
 	paths    map[common.Address]accounts.DerivationPath // Known derivation paths for signing operations

beacon/blsync/block_sync.go

@@ -0,0 +1,203 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package blsync
+
+import (
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/beacon/engine"
+	"github.com/ethereum/go-ethereum/beacon/light/request"
+	"github.com/ethereum/go-ethereum/beacon/light/sync"
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/lru"
+	ctypes "github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/holiman/uint256"
+	"github.com/protolambda/zrnt/eth2/beacon/capella"
+	"github.com/protolambda/zrnt/eth2/configs"
+	"github.com/protolambda/ztyp/tree"
+)
+
+// beaconBlockSync implements request.Module; it fetches the beacon blocks belonging
+// to the validated and prefetch heads.
+type beaconBlockSync struct {
+	recentBlocks *lru.Cache[common.Hash, *capella.BeaconBlock]
+	locked       map[common.Hash]request.ServerAndID
+	serverHeads  map[request.Server]common.Hash
+	headTracker  headTracker
+
+	lastHeadInfo  types.HeadInfo
+	chainHeadFeed *event.Feed
+}
+
+type headTracker interface {
+	PrefetchHead() types.HeadInfo
+	ValidatedHead() (types.SignedHeader, bool)
+	ValidatedFinality() (types.FinalityUpdate, bool)
+}
+
+// newBeaconBlockSync returns a new beaconBlockSync.
+func newBeaconBlockSync(headTracker headTracker, chainHeadFeed *event.Feed) *beaconBlockSync {
+	return &beaconBlockSync{
+		headTracker:   headTracker,
+		chainHeadFeed: chainHeadFeed,
+		recentBlocks:  lru.NewCache[common.Hash, *capella.BeaconBlock](10),
+		locked:        make(map[common.Hash]request.ServerAndID),
+		serverHeads:   make(map[request.Server]common.Hash),
+	}
+}
+
+// Process implements request.Module.
+func (s *beaconBlockSync) Process(requester request.Requester, events []request.Event) {
+	for _, event := range events {
+		switch event.Type {
+		case request.EvResponse, request.EvFail, request.EvTimeout:
+			sid, req, resp := event.RequestInfo()
+			blockRoot := common.Hash(req.(sync.ReqBeaconBlock))
+			if resp != nil {
+				s.recentBlocks.Add(blockRoot, resp.(*capella.BeaconBlock))
+			}
+			if s.locked[blockRoot] == sid {
+				delete(s.locked, blockRoot)
+			}
+		case sync.EvNewHead:
+			s.serverHeads[event.Server] = event.Data.(types.HeadInfo).BlockRoot
+		case request.EvUnregistered:
+			delete(s.serverHeads, event.Server)
+		}
+	}
+	s.updateEventFeed()
+	// request validated head block if unavailable and not yet requested
+	if vh, ok := s.headTracker.ValidatedHead(); ok {
+		s.tryRequestBlock(requester, vh.Header.Hash(), false)
+	}
+	// request prefetch head if the given server has announced it
+	if prefetchHead := s.headTracker.PrefetchHead().BlockRoot; prefetchHead != (common.Hash{}) {
+		s.tryRequestBlock(requester, prefetchHead, true)
+	}
+}
+
+func (s *beaconBlockSync) tryRequestBlock(requester request.Requester, blockRoot common.Hash, needSameHead bool) {
+	if _, ok := s.recentBlocks.Get(blockRoot); ok {
+		return
+	}
+	if _, ok := s.locked[blockRoot]; ok {
+		return
+	}
+	for _, server := range requester.CanSendTo() {
+		if needSameHead && (s.serverHeads[server] != blockRoot) {
+			continue
+		}
+		id := requester.Send(server, sync.ReqBeaconBlock(blockRoot))
+		s.locked[blockRoot] = request.ServerAndID{Server: server, ID: id}
+		return
+	}
+}
+
+func blockHeadInfo(block *capella.BeaconBlock) types.HeadInfo {
+	if block == nil {
+		return types.HeadInfo{}
+	}
+	return types.HeadInfo{Slot: uint64(block.Slot), BlockRoot: beaconBlockHash(block)}
+}
+
+// beaconBlockHash calculates the hash of a beacon block.
+func beaconBlockHash(beaconBlock *capella.BeaconBlock) common.Hash {
+	return common.Hash(beaconBlock.HashTreeRoot(configs.Mainnet, tree.GetHashFn()))
+}
+
+// getExecBlock extracts the execution block from the beacon block's payload.
+func getExecBlock(beaconBlock *capella.BeaconBlock) (*ctypes.Block, error) {
+	payload := &beaconBlock.Body.ExecutionPayload
+	txs := make([]*ctypes.Transaction, len(payload.Transactions))
+	for i, opaqueTx := range payload.Transactions {
+		var tx ctypes.Transaction
+		if err := tx.UnmarshalBinary(opaqueTx); err != nil {
+			return nil, fmt.Errorf("failed to parse tx %d: %v", i, err)
+		}
+		txs[i] = &tx
+	}
+	withdrawals := make([]*ctypes.Withdrawal, len(payload.Withdrawals))
+	for i, w := range payload.Withdrawals {
+		withdrawals[i] = &ctypes.Withdrawal{
+			Index:     uint64(w.Index),
+			Validator: uint64(w.ValidatorIndex),
+			Address:   common.Address(w.Address),
+			Amount:    uint64(w.Amount),
+		}
+	}
+	wroot := ctypes.DeriveSha(ctypes.Withdrawals(withdrawals), trie.NewStackTrie(nil))
+	execHeader := &ctypes.Header{
+		ParentHash:      common.Hash(payload.ParentHash),
+		UncleHash:       ctypes.EmptyUncleHash,
+		Coinbase:        common.Address(payload.FeeRecipient),
+		Root:            common.Hash(payload.StateRoot),
+		TxHash:          ctypes.DeriveSha(ctypes.Transactions(txs), trie.NewStackTrie(nil)),
+		ReceiptHash:     common.Hash(payload.ReceiptsRoot),
+		Bloom:           ctypes.Bloom(payload.LogsBloom),
+		Difficulty:      common.Big0,
+		Number:          new(big.Int).SetUint64(uint64(payload.BlockNumber)),
+		GasLimit:        uint64(payload.GasLimit),
+		GasUsed:         uint64(payload.GasUsed),
+		Time:            uint64(payload.Timestamp),
+		Extra:           []byte(payload.ExtraData),
+		MixDigest:       common.Hash(payload.PrevRandao), // reused in merge
+		Nonce:           ctypes.BlockNonce{},             // zero
+		BaseFee:         (*uint256.Int)(&payload.BaseFeePerGas).ToBig(),
+		WithdrawalsHash: &wroot,
+	}
+	execBlock := ctypes.NewBlockWithHeader(execHeader).WithBody(txs, nil).WithWithdrawals(withdrawals)
+	if execBlockHash := execBlock.Hash(); execBlockHash != common.Hash(payload.BlockHash) {
+		return execBlock, fmt.Errorf("Sanity check failed, payload hash does not match (expected %x, got %x)", common.Hash(payload.BlockHash), execBlockHash)
+	}
+	return execBlock, nil
+}
+
+func (s *beaconBlockSync) updateEventFeed() {
+	head, ok := s.headTracker.ValidatedHead()
+	if !ok {
+		return
+	}
+	finality, ok := s.headTracker.ValidatedFinality() //TODO fetch directly if subscription does not deliver
+	if !ok || head.Header.Epoch() != finality.Attested.Header.Epoch() {
+		return
+	}
+	validatedHead := head.Header.Hash()
+	headBlock, ok := s.recentBlocks.Get(validatedHead)
+	if !ok {
+		return
+	}
+	headInfo := blockHeadInfo(headBlock)
+	if headInfo == s.lastHeadInfo {
+		return
+	}
+	s.lastHeadInfo = headInfo
+	// new head block and finality info available; extract executable data and send event to feed
+	execBlock, err := getExecBlock(headBlock)
+	if err != nil {
+		log.Error("Error extracting execution block from validated beacon block", "error", err)
+		return
+	}
+	s.chainHeadFeed.Send(types.ChainHeadEvent{
+		HeadBlock: engine.BlockToExecutableData(execBlock, nil, nil).ExecutionPayload,
+		Finalized: common.Hash(finality.Finalized.PayloadHeader.BlockHash),
+	})
+}

beacon/blsync/block_sync_test.go

@@ -0,0 +1,160 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package blsync
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/beacon/light/request"
+	"github.com/ethereum/go-ethereum/beacon/light/sync"
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/protolambda/zrnt/eth2/beacon/capella"
+	"github.com/protolambda/zrnt/eth2/configs"
+	"github.com/protolambda/ztyp/tree"
+)
+
+var (
+	testServer1 = "testServer1"
+	testServer2 = "testServer2"
+
+	testBlock1 = &capella.BeaconBlock{
+		Slot: 123,
+		Body: capella.BeaconBlockBody{
+			ExecutionPayload: capella.ExecutionPayload{BlockNumber: 456},
+		},
+	}
+	testBlock2 = &capella.BeaconBlock{
+		Slot: 124,
+		Body: capella.BeaconBlockBody{
+			ExecutionPayload: capella.ExecutionPayload{BlockNumber: 457},
+		},
+	}
+)
+
+func init() {
+	eb1, _ := getExecBlock(testBlock1)
+	testBlock1.Body.ExecutionPayload.BlockHash = tree.Root(eb1.Hash())
+	eb2, _ := getExecBlock(testBlock2)
+	testBlock2.Body.ExecutionPayload.BlockHash = tree.Root(eb2.Hash())
+}
+
+func TestBlockSync(t *testing.T) {
+	ht := &testHeadTracker{}
+	eventFeed := new(event.Feed)
+	blockSync := newBeaconBlockSync(ht, eventFeed)
+	headCh := make(chan types.ChainHeadEvent, 16)
+	eventFeed.Subscribe(headCh)
+	ts := sync.NewTestScheduler(t, blockSync)
+	ts.AddServer(testServer1, 1)
+	ts.AddServer(testServer2, 1)
+
+	expHeadBlock := func(tci int, expHead *capella.BeaconBlock) {
+		var expNumber, headNumber uint64
+		if expHead != nil {
+			expNumber = uint64(expHead.Body.ExecutionPayload.BlockNumber)
+		}
+		select {
+		case event := <-headCh:
+			headNumber = event.HeadBlock.Number
+		default:
+		}
+		if headNumber != expNumber {
+			t.Errorf("Wrong head block in test case #%d (expected block number %d, got %d)", tci, expNumber, headNumber)
+		}
+	}
+
+	// no block requests expected until head tracker knows about a head
+	ts.Run(1)
+	expHeadBlock(1, nil)
+
+	// set block 1 as prefetch head, announced by server 2
+	head1 := blockHeadInfo(testBlock1)
+	ht.prefetch = head1
+	ts.ServerEvent(sync.EvNewHead, testServer2, head1)
+	// expect request to server 2 which has announced the head
+	ts.Run(2, testServer2, sync.ReqBeaconBlock(head1.BlockRoot))
+
+	// valid response
+	ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testBlock1)
+	ts.AddAllowance(testServer2, 1)
+	ts.Run(3)
+	// head block still not expected as the fetched block is not the validated head yet
+	expHeadBlock(3, nil)
+
+	// set as validated head, expect no further requests but block 1 set as head block
+	ht.validated.Header = blockHeader(testBlock1)
+	ts.Run(4)
+	expHeadBlock(4, testBlock1)
+
+	// set block 2 as prefetch head, announced by server 1
+	head2 := blockHeadInfo(testBlock2)
+	ht.prefetch = head2
+	ts.ServerEvent(sync.EvNewHead, testServer1, head2)
+	// expect request to server 1
+	ts.Run(5, testServer1, sync.ReqBeaconBlock(head2.BlockRoot))
+
+	// req2 fails, no further requests expected because server 2 has not announced it
+	ts.RequestEvent(request.EvFail, ts.Request(5, 1), nil)
+	ts.Run(6)
+
+	// set as validated head before retrieving block; now it's assumed to be available from server 2 too
+	ht.validated.Header = blockHeader(testBlock2)
+	// expect req2 retry to server 2
+	ts.Run(7, testServer2, sync.ReqBeaconBlock(head2.BlockRoot))
+	// now head block should be unavailable again
+	expHeadBlock(4, nil)
+
+	// valid response, now head block should be block 2 immediately as it is already validated
+	ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testBlock2)
+	ts.Run(8)
+	expHeadBlock(5, testBlock2)
+}
+
+func blockHeader(block *capella.BeaconBlock) types.Header {
+	return types.Header{
+		Slot:          uint64(block.Slot),
+		ProposerIndex: uint64(block.ProposerIndex),
+		ParentRoot:    common.Hash(block.ParentRoot),
+		StateRoot:     common.Hash(block.StateRoot),
+		BodyRoot:      common.Hash(block.Body.HashTreeRoot(configs.Mainnet, tree.GetHashFn())),
+	}
+}
+
+type testHeadTracker struct {
+	prefetch  types.HeadInfo
+	validated types.SignedHeader
+}
+
+func (h *testHeadTracker) PrefetchHead() types.HeadInfo {
+	return h.prefetch
+}
+
+func (h *testHeadTracker) ValidatedHead() (types.SignedHeader, bool) {
+	return h.validated, h.validated.Header != (types.Header{})
+}
+
+// TODO add test case for finality
+func (h *testHeadTracker) ValidatedFinality() (types.FinalityUpdate, bool) {
+	return types.FinalityUpdate{
+		Attested:      types.HeaderWithExecProof{Header: h.validated.Header},
+		Finalized:     types.HeaderWithExecProof{PayloadHeader: &capella.ExecutionPayloadHeader{}},
+		Signature:     h.validated.Signature,
+		SignatureSlot: h.validated.SignatureSlot,
+	}, h.validated.Header != (types.Header{})
+}

beacon/blsync/client.go

@@ -0,0 +1,103 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package blsync
+
+import (
+	"strings"
+
+	"github.com/ethereum/go-ethereum/beacon/light"
+	"github.com/ethereum/go-ethereum/beacon/light/api"
+	"github.com/ethereum/go-ethereum/beacon/light/request"
+	"github.com/ethereum/go-ethereum/beacon/light/sync"
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common/mclock"
+	"github.com/ethereum/go-ethereum/ethdb/memorydb"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/urfave/cli/v2"
+)
+
+type Client struct {
+	scheduler     *request.Scheduler
+	chainHeadFeed *event.Feed
+	urls          []string
+	customHeader  map[string]string
+}
+
+func NewClient(ctx *cli.Context) *Client {
+	if !ctx.IsSet(utils.BeaconApiFlag.Name) {
+		utils.Fatalf("Beacon node light client API URL not specified")
+	}
+	var (
+		chainConfig  = makeChainConfig(ctx)
+		customHeader = make(map[string]string)
+	)
+	for _, s := range ctx.StringSlice(utils.BeaconApiHeaderFlag.Name) {
+		kv := strings.Split(s, ":")
+		if len(kv) != 2 {
+			utils.Fatalf("Invalid custom API header entry: %s", s)
+		}
+		customHeader[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
+	}
+	// create data structures
+	var (
+		db             = memorydb.New()
+		threshold      = ctx.Int(utils.BeaconThresholdFlag.Name)
+		committeeChain = light.NewCommitteeChain(db, chainConfig.ChainConfig, threshold, !ctx.Bool(utils.BeaconNoFilterFlag.Name))
+		headTracker    = light.NewHeadTracker(committeeChain, threshold)
+	)
+	headSync := sync.NewHeadSync(headTracker, committeeChain)
+
+	// set up scheduler and sync modules
+	chainHeadFeed := new(event.Feed)
+	scheduler := request.NewScheduler()
+	checkpointInit := sync.NewCheckpointInit(committeeChain, chainConfig.Checkpoint)
+	forwardSync := sync.NewForwardUpdateSync(committeeChain)
+	beaconBlockSync := newBeaconBlockSync(headTracker, chainHeadFeed)
+	scheduler.RegisterTarget(headTracker)
+	scheduler.RegisterTarget(committeeChain)
+	scheduler.RegisterModule(checkpointInit, "checkpointInit")
+	scheduler.RegisterModule(forwardSync, "forwardSync")
+	scheduler.RegisterModule(headSync, "headSync")
+	scheduler.RegisterModule(beaconBlockSync, "beaconBlockSync")
+
+	return &Client{
+		scheduler:     scheduler,
+		urls:          ctx.StringSlice(utils.BeaconApiFlag.Name),
+		customHeader:  customHeader,
+		chainHeadFeed: chainHeadFeed,
+	}
+}
+
+// SubscribeChainHeadEvent allows callers to subscribe a provided channel to new
+// head updates.
+func (c *Client) SubscribeChainHeadEvent(ch chan<- types.ChainHeadEvent) event.Subscription {
+	return c.chainHeadFeed.Subscribe(ch)
+}
+
+func (c *Client) Start() {
+	c.scheduler.Start()
+	// register server(s)
+	for _, url := range c.urls {
+		beaconApi := api.NewBeaconLightApi(url, c.customHeader)
+		c.scheduler.RegisterServer(request.NewServer(api.NewApiServer(beaconApi), &mclock.System{}))
+	}
+}
+
+func (c *Client) Stop() {
+	c.scheduler.Stop()
+}

beacon/blsync/config.go

@@ -0,0 +1,113 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package blsync
+
+import (
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/urfave/cli/v2"
+)
+
+// lightClientConfig contains beacon light client configuration
+type lightClientConfig struct {
+	*types.ChainConfig
+	Checkpoint common.Hash
+}
+
+var (
+	MainnetConfig = lightClientConfig{
+		ChainConfig: (&types.ChainConfig{
+			GenesisValidatorsRoot: common.HexToHash("0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95"),
+			GenesisTime:           1606824023,
+		}).
+			AddFork("GENESIS", 0, []byte{0, 0, 0, 0}).
+			AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}).
+			AddFork("BELLATRIX", 144896, []byte{2, 0, 0, 0}).
+			AddFork("CAPELLA", 194048, []byte{3, 0, 0, 0}),
+		Checkpoint: common.HexToHash("0x388be41594ec7d6a6894f18c73f3469f07e2c19a803de4755d335817ed8e2e5a"),
+	}
+
+	SepoliaConfig = lightClientConfig{
+		ChainConfig: (&types.ChainConfig{
+			GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"),
+			GenesisTime:           1655733600,
+		}).
+			AddFork("GENESIS", 0, []byte{144, 0, 0, 105}).
+			AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}).
+			AddFork("BELLATRIX", 100, []byte{144, 0, 0, 113}).
+			AddFork("CAPELLA", 56832, []byte{144, 0, 0, 114}),
+		Checkpoint: common.HexToHash("0x1005a6d9175e96bfbce4d35b80f468e9bff0b674e1e861d16e09e10005a58e81"),
+	}
+
+	GoerliConfig = lightClientConfig{
+		ChainConfig: (&types.ChainConfig{
+			GenesisValidatorsRoot: common.HexToHash("0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb"),
+			GenesisTime:           1614588812,
+		}).
+			AddFork("GENESIS", 0, []byte{0, 0, 16, 32}).
+			AddFork("ALTAIR", 36660, []byte{1, 0, 16, 32}).
+			AddFork("BELLATRIX", 112260, []byte{2, 0, 16, 32}).
+			AddFork("CAPELLA", 162304, []byte{3, 0, 16, 32}),
+		Checkpoint: common.HexToHash("0x53a0f4f0a378e2c4ae0a9ee97407eb69d0d737d8d8cd0a5fb1093f42f7b81c49"),
+	}
+)
+
+func makeChainConfig(ctx *cli.Context) lightClientConfig {
+	utils.CheckExclusive(ctx, utils.MainnetFlag, utils.GoerliFlag, utils.SepoliaFlag)
+	customConfig := ctx.IsSet(utils.BeaconConfigFlag.Name) || ctx.IsSet(utils.BeaconGenesisRootFlag.Name) || ctx.IsSet(utils.BeaconGenesisTimeFlag.Name)
+	var config lightClientConfig
+	switch {
+	case ctx.Bool(utils.MainnetFlag.Name):
+		config = MainnetConfig
+	case ctx.Bool(utils.SepoliaFlag.Name):
+		config = SepoliaConfig
+	case ctx.Bool(utils.GoerliFlag.Name):
+		config = GoerliConfig
+	default:
+		if !customConfig {
+			config = MainnetConfig
+		}
+	}
+	if customConfig && config.Forks != nil {
+		utils.Fatalf("Cannot use custom beacon chain config flags in combination with pre-defined network config")
+	}
+	if ctx.IsSet(utils.BeaconGenesisRootFlag.Name) {
+		if c, err := hexutil.Decode(ctx.String(utils.BeaconGenesisRootFlag.Name)); err == nil && len(c) <= 32 {
+			copy(config.GenesisValidatorsRoot[:len(c)], c)
+		} else {
+			utils.Fatalf("Invalid hex string", "beacon.genesis.gvroot", ctx.String(utils.BeaconGenesisRootFlag.Name), "error", err)
+		}
+	}
+	if ctx.IsSet(utils.BeaconGenesisTimeFlag.Name) {
+		config.GenesisTime = ctx.Uint64(utils.BeaconGenesisTimeFlag.Name)
+	}
+	if ctx.IsSet(utils.BeaconConfigFlag.Name) {
+		if err := config.ChainConfig.LoadForks(ctx.String(utils.BeaconConfigFlag.Name)); err != nil {
+			utils.Fatalf("Could not load beacon chain config file", "file name", ctx.String(utils.BeaconConfigFlag.Name), "error", err)
+		}
+	}
+	if ctx.IsSet(utils.BeaconCheckpointFlag.Name) {
+		if c, err := hexutil.Decode(ctx.String(utils.BeaconCheckpointFlag.Name)); err == nil && len(c) <= 32 {
+			copy(config.Checkpoint[:len(c)], c)
+		} else {
+			utils.Fatalf("Invalid hex string", "beacon.checkpoint", ctx.String(utils.BeaconCheckpointFlag.Name), "error", err)
+		}
+	}
+	return config
+}

beacon/light/api/api_server.go

@@ -0,0 +1,103 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"reflect"
+
+	"github.com/ethereum/go-ethereum/beacon/light/request"
+	"github.com/ethereum/go-ethereum/beacon/light/sync"
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// ApiServer is a wrapper around BeaconLightApi that implements request.requestServer.
+type ApiServer struct {
+	api           *BeaconLightApi
+	eventCallback func(event request.Event)
+	unsubscribe   func()
+}
+
+// NewApiServer creates a new ApiServer.
+func NewApiServer(api *BeaconLightApi) *ApiServer {
+	return &ApiServer{api: api}
+}
+
+// Subscribe implements request.requestServer.
+func (s *ApiServer) Subscribe(eventCallback func(event request.Event)) {
+	s.eventCallback = eventCallback
+	listener := HeadEventListener{
+		OnNewHead: func(slot uint64, blockRoot common.Hash) {
+			log.Debug("New head received", "slot", slot, "blockRoot", blockRoot)
+			eventCallback(request.Event{Type: sync.EvNewHead, Data: types.HeadInfo{Slot: slot, BlockRoot: blockRoot}})
+		},
+		OnSignedHead: func(head types.SignedHeader) {
+			log.Debug("New signed head received", "slot", head.Header.Slot, "blockRoot", head.Header.Hash(), "signerCount", head.Signature.SignerCount())
+			eventCallback(request.Event{Type: sync.EvNewSignedHead, Data: head})
+		},
+		OnFinality: func(head types.FinalityUpdate) {
+			log.Debug("New finality update received", "slot", head.Attested.Slot, "blockRoot", head.Attested.Hash(), "signerCount", head.Signature.SignerCount())
+			eventCallback(request.Event{Type: sync.EvNewFinalityUpdate, Data: head})
+		},
+		OnError: func(err error) {
+			log.Warn("Head event stream error", "err", err)
+		},
+	}
+	s.unsubscribe = s.api.StartHeadListener(listener)
+}
+
+// SendRequest implements request.requestServer.
+func (s *ApiServer) SendRequest(id request.ID, req request.Request) {
+	go func() {
+		var resp request.Response
+		var err error
+		switch data := req.(type) {
+		case sync.ReqUpdates:
+			log.Debug("Beacon API: requesting light client update", "reqid", id, "period", data.FirstPeriod, "count", data.Count)
+			var r sync.RespUpdates
+			r.Updates, r.Committees, err = s.api.GetBestUpdatesAndCommittees(data.FirstPeriod, data.Count)
+			resp = r
+		case sync.ReqHeader:
+			log.Debug("Beacon API: requesting header", "reqid", id, "hash", common.Hash(data))
+			resp, err = s.api.GetHeader(common.Hash(data))
+		case sync.ReqCheckpointData:
+			log.Debug("Beacon API: requesting checkpoint data", "reqid", id, "hash", common.Hash(data))
+			resp, err = s.api.GetCheckpointData(common.Hash(data))
+		case sync.ReqBeaconBlock:
+			log.Debug("Beacon API: requesting block", "reqid", id, "hash", common.Hash(data))
+			resp, err = s.api.GetBeaconBlock(common.Hash(data))
+		default:
+		}
+
+		if err != nil {
+			log.Warn("Beacon API request failed", "type", reflect.TypeOf(req), "reqid", id, "err", err)
+			s.eventCallback(request.Event{Type: request.EvFail, Data: request.RequestResponse{ID: id, Request: req}})
+		} else {
+			s.eventCallback(request.Event{Type: request.EvResponse, Data: request.RequestResponse{ID: id, Request: req, Response: resp}})
+		}
+	}()
+}
+
+// Unsubscribe implements request.requestServer.
+// Note: Unsubscribe should not be called concurrently with Subscribe.
+func (s *ApiServer) Unsubscribe() {
+	if s.unsubscribe != nil {
+		s.unsubscribe()
+		s.unsubscribe = nil
+	}
+}

beacon/light/api/light_api.go

@@ -0,0 +1,496 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more detaiapi.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package api
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"github.com/donovanhide/eventsource"
+	"github.com/ethereum/go-ethereum/beacon/merkle"
+	"github.com/ethereum/go-ethereum/beacon/params"
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/protolambda/zrnt/eth2/beacon/capella"
+	"github.com/protolambda/zrnt/eth2/configs"
+	"github.com/protolambda/ztyp/tree"
+)
+
+var (
+	ErrNotFound = errors.New("404 Not Found")
+	ErrInternal = errors.New("500 Internal Server Error")
+)
+
+type CommitteeUpdate struct {
+	Version           string
+	Update            types.LightClientUpdate
+	NextSyncCommittee types.SerializedSyncCommittee
+}
+
+// See data structure definition here:
+// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientupdate
+type committeeUpdateJson struct {
+	Version string              `json:"version"`
+	Data    committeeUpdateData `json:"data"`
+}
+
+type committeeUpdateData struct {
+	Header                  jsonBeaconHeader              `json:"attested_header"`
+	NextSyncCommittee       types.SerializedSyncCommittee `json:"next_sync_committee"`
+	NextSyncCommitteeBranch merkle.Values                 `json:"next_sync_committee_branch"`
+	FinalizedHeader         *jsonBeaconHeader             `json:"finalized_header,omitempty"`
+	FinalityBranch          merkle.Values                 `json:"finality_branch,omitempty"`
+	SyncAggregate           types.SyncAggregate           `json:"sync_aggregate"`
+	SignatureSlot           common.Decimal                `json:"signature_slot"`
+}
+
+type jsonBeaconHeader struct {
+	Beacon types.Header `json:"beacon"`
+}
+
+type jsonHeaderWithExecProof struct {
+	Beacon          types.Header                    `json:"beacon"`
+	Execution       *capella.ExecutionPayloadHeader `json:"execution"`
+	ExecutionBranch merkle.Values                   `json:"execution_branch"`
+}
+
+// UnmarshalJSON unmarshals from JSON.
+func (u *CommitteeUpdate) UnmarshalJSON(input []byte) error {
+	var dec committeeUpdateJson
+	if err := json.Unmarshal(input, &dec); err != nil {
+		return err
+	}
+	u.Version = dec.Version
+	u.NextSyncCommittee = dec.Data.NextSyncCommittee
+	u.Update = types.LightClientUpdate{
+		AttestedHeader: types.SignedHeader{
+			Header:        dec.Data.Header.Beacon,
+			Signature:     dec.Data.SyncAggregate,
+			SignatureSlot: uint64(dec.Data.SignatureSlot),
+		},
+		NextSyncCommitteeRoot:   u.NextSyncCommittee.Root(),
+		NextSyncCommitteeBranch: dec.Data.NextSyncCommitteeBranch,
+		FinalityBranch:          dec.Data.FinalityBranch,
+	}
+	if dec.Data.FinalizedHeader != nil {
+		u.Update.FinalizedHeader = &dec.Data.FinalizedHeader.Beacon
+	}
+	return nil
+}
+
+// fetcher is an interface useful for debug-harnessing the http api.
+type fetcher interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+// BeaconLightApi requests light client information from a beacon node REST API.
+// Note: all required API endpoints are currently only implemented by Lodestar.
+type BeaconLightApi struct {
+	url           string
+	client        fetcher
+	customHeaders map[string]string
+}
+
+func NewBeaconLightApi(url string, customHeaders map[string]string) *BeaconLightApi {
+	return &BeaconLightApi{
+		url: url,
+		client: &http.Client{
+			Timeout: time.Second * 10,
+		},
+		customHeaders: customHeaders,
+	}
+}
+
+func (api *BeaconLightApi) httpGet(path string) ([]byte, error) {
+	req, err := http.NewRequest("GET", api.url+path, nil)
+	if err != nil {
+		return nil, err
+	}
+	for k, v := range api.customHeaders {
+		req.Header.Set(k, v)
+	}
+	resp, err := api.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	switch resp.StatusCode {
+	case 200:
+		return io.ReadAll(resp.Body)
+	case 404:
+		return nil, ErrNotFound
+	case 500:
+		return nil, ErrInternal
+	default:
+		return nil, fmt.Errorf("unexpected error from API endpoint \"%s\": status code %d", path, resp.StatusCode)
+	}
+}
+
+func (api *BeaconLightApi) httpGetf(format string, params ...any) ([]byte, error) {
+	return api.httpGet(fmt.Sprintf(format, params...))
+}
+
+// GetBestUpdateAndCommittee fetches and validates LightClientUpdate for given
+// period and full serialized committee for the next period (committee root hash
+// equals update.NextSyncCommitteeRoot).
+// Note that the results are validated but the update signature should be verified
+// by the caller as its validity depends on the update chain.
+func (api *BeaconLightApi) GetBestUpdatesAndCommittees(firstPeriod, count uint64) ([]*types.LightClientUpdate, []*types.SerializedSyncCommittee, error) {
+	resp, err := api.httpGetf("/eth/v1/beacon/light_client/updates?start_period=%d&count=%d", firstPeriod, count)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var data []CommitteeUpdate
+	if err := json.Unmarshal(resp, &data); err != nil {
+		return nil, nil, err
+	}
+	if len(data) != int(count) {
+		return nil, nil, errors.New("invalid number of committee updates")
+	}
+	updates := make([]*types.LightClientUpdate, int(count))
+	committees := make([]*types.SerializedSyncCommittee, int(count))
+	for i, d := range data {
+		if d.Update.AttestedHeader.Header.SyncPeriod() != firstPeriod+uint64(i) {
+			return nil, nil, errors.New("wrong committee update header period")
+		}
+		if err := d.Update.Validate(); err != nil {
+			return nil, nil, err
+		}
+		if d.NextSyncCommittee.Root() != d.Update.NextSyncCommitteeRoot {
+			return nil, nil, errors.New("wrong sync committee root")
+		}
+		updates[i], committees[i] = new(types.LightClientUpdate), new(types.SerializedSyncCommittee)
+		*updates[i], *committees[i] = d.Update, d.NextSyncCommittee
+	}
+	return updates, committees, nil
+}
+
+// GetOptimisticHeadUpdate fetches a signed header based on the latest available
+// optimistic update. Note that the signature should be verified by the caller
+// as its validity depends on the update chain.
+//
+// See data structure definition here:
+// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientoptimisticupdate
+func (api *BeaconLightApi) GetOptimisticHeadUpdate() (types.SignedHeader, error) {
+	resp, err := api.httpGet("/eth/v1/beacon/light_client/optimistic_update")
+	if err != nil {
+		return types.SignedHeader{}, err
+	}
+	return decodeOptimisticHeadUpdate(resp)
+}
+
+func decodeOptimisticHeadUpdate(enc []byte) (types.SignedHeader, error) {
+	var data struct {
+		Data struct {
+			Header        jsonBeaconHeader    `json:"attested_header"`
+			Aggregate     types.SyncAggregate `json:"sync_aggregate"`
+			SignatureSlot common.Decimal      `json:"signature_slot"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(enc, &data); err != nil {
+		return types.SignedHeader{}, err
+	}
+	if data.Data.Header.Beacon.StateRoot == (common.Hash{}) {
+		// workaround for different event encoding format in Lodestar
+		if err := json.Unmarshal(enc, &data.Data); err != nil {
+			return types.SignedHeader{}, err
+		}
+	}
+
+	if len(data.Data.Aggregate.Signers) != params.SyncCommitteeBitmaskSize {
+		return types.SignedHeader{}, errors.New("invalid sync_committee_bits length")
+	}
+	if len(data.Data.Aggregate.Signature) != params.BLSSignatureSize {
+		return types.SignedHeader{}, errors.New("invalid sync_committee_signature length")
+	}
+	return types.SignedHeader{
+		Header:        data.Data.Header.Beacon,
+		Signature:     data.Data.Aggregate,
+		SignatureSlot: uint64(data.Data.SignatureSlot),
+	}, nil
+}
+
+// GetFinalityUpdate fetches the latest available finality update.
+//
+// See data structure definition here:
+// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientfinalityupdate
+func (api *BeaconLightApi) GetFinalityUpdate() (types.FinalityUpdate, error) {
+	resp, err := api.httpGet("/eth/v1/beacon/light_client/finality_update")
+	if err != nil {
+		return types.FinalityUpdate{}, err
+	}
+	return decodeFinalityUpdate(resp)
+}
+
+func decodeFinalityUpdate(enc []byte) (types.FinalityUpdate, error) {
+	var data struct {
+		Data struct {
+			Attested       jsonHeaderWithExecProof `json:"attested_header"`
+			Finalized      jsonHeaderWithExecProof `json:"finalized_header"`
+			FinalityBranch merkle.Values           `json:"finality_branch"`
+			Aggregate      types.SyncAggregate     `json:"sync_aggregate"`
+			SignatureSlot  common.Decimal          `json:"signature_slot"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(enc, &data); err != nil {
+		return types.FinalityUpdate{}, err
+	}
+
+	if len(data.Data.Aggregate.Signers) != params.SyncCommitteeBitmaskSize {
+		return types.FinalityUpdate{}, errors.New("invalid sync_committee_bits length")
+	}
+	if len(data.Data.Aggregate.Signature) != params.BLSSignatureSize {
+		return types.FinalityUpdate{}, errors.New("invalid sync_committee_signature length")
+	}
+	return types.FinalityUpdate{
+		Attested: types.HeaderWithExecProof{
+			Header:        data.Data.Attested.Beacon,
+			PayloadHeader: data.Data.Attested.Execution,
+			PayloadBranch: data.Data.Attested.ExecutionBranch,
+		},
+		Finalized: types.HeaderWithExecProof{
+			Header:        data.Data.Finalized.Beacon,
+			PayloadHeader: data.Data.Finalized.Execution,
+			PayloadBranch: data.Data.Finalized.ExecutionBranch,
+		},
+		FinalityBranch: data.Data.FinalityBranch,
+		Signature:      data.Data.Aggregate,
+		SignatureSlot:  uint64(data.Data.SignatureSlot),
+	}, nil
+}
+
+// GetHead fetches and validates the beacon header with the given blockRoot.
+// If blockRoot is null hash then the latest head header is fetched.
+func (api *BeaconLightApi) GetHeader(blockRoot common.Hash) (types.Header, error) {
+	var blockId string
+	if blockRoot == (common.Hash{}) {
+		blockId = "head"
+	} else {
+		blockId = blockRoot.Hex()
+	}
+	resp, err := api.httpGetf("/eth/v1/beacon/headers/%s", blockId)
+	if err != nil {
+		return types.Header{}, err
+	}
+
+	var data struct {
+		Data struct {
+			Root      common.Hash `json:"root"`
+			Canonical bool        `json:"canonical"`
+			Header    struct {
+				Message   types.Header  `json:"message"`
+				Signature hexutil.Bytes `json:"signature"`
+			} `json:"header"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(resp, &data); err != nil {
+		return types.Header{}, err
+	}
+	header := data.Data.Header.Message
+	if blockRoot == (common.Hash{}) {
+		blockRoot = data.Data.Root
+	}
+	if header.Hash() != blockRoot {
+		return types.Header{}, errors.New("retrieved beacon header root does not match")
+	}
+	return header, nil
+}
+
+// GetCheckpointData fetches and validates bootstrap data belonging to the given checkpoint.
+func (api *BeaconLightApi) GetCheckpointData(checkpointHash common.Hash) (*types.BootstrapData, error) {
+	resp, err := api.httpGetf("/eth/v1/beacon/light_client/bootstrap/0x%x", checkpointHash[:])
+	if err != nil {
+		return nil, err
+	}
+
+	// See data structure definition here:
+	// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientbootstrap
+	type bootstrapData struct {
+		Data struct {
+			Header          jsonBeaconHeader               `json:"header"`
+			Committee       *types.SerializedSyncCommittee `json:"current_sync_committee"`
+			CommitteeBranch merkle.Values                  `json:"current_sync_committee_branch"`
+		} `json:"data"`
+	}
+
+	var data bootstrapData
+	if err := json.Unmarshal(resp, &data); err != nil {
+		return nil, err
+	}
+	if data.Data.Committee == nil {
+		return nil, errors.New("sync committee is missing")
+	}
+	header := data.Data.Header.Beacon
+	if header.Hash() != checkpointHash {
+		return nil, fmt.Errorf("invalid checkpoint block header, have %v want %v", header.Hash(), checkpointHash)
+	}
+	checkpoint := &types.BootstrapData{
+		Header:          header,
+		CommitteeBranch: data.Data.CommitteeBranch,
+		CommitteeRoot:   data.Data.Committee.Root(),
+		Committee:       data.Data.Committee,
+	}
+	if err := checkpoint.Validate(); err != nil {
+		return nil, fmt.Errorf("invalid checkpoint: %w", err)
+	}
+	if checkpoint.Header.Hash() != checkpointHash {
+		return nil, errors.New("wrong checkpoint hash")
+	}
+	return checkpoint, nil
+}
+
+func (api *BeaconLightApi) GetBeaconBlock(blockRoot common.Hash) (*capella.BeaconBlock, error) {
+	resp, err := api.httpGetf("/eth/v2/beacon/blocks/0x%x", blockRoot)
+	if err != nil {
+		return nil, err
+	}
+
+	var beaconBlockMessage struct {
+		Data struct {
+			Message capella.BeaconBlock `json:"message"`
+		} `json:"data"`
+	}
+	if err := json.Unmarshal(resp, &beaconBlockMessage); err != nil {
+		return nil, fmt.Errorf("invalid block json data: %v", err)
+	}
+	beaconBlock := new(capella.BeaconBlock)
+	*beaconBlock = beaconBlockMessage.Data.Message
+	root := common.Hash(beaconBlock.HashTreeRoot(configs.Mainnet, tree.GetHashFn()))
+	if root != blockRoot {
+		return nil, fmt.Errorf("Beacon block root hash mismatch (expected: %x, got: %x)", blockRoot, root)
+	}
+	return beaconBlock, nil
+}
+
+func decodeHeadEvent(enc []byte) (uint64, common.Hash, error) {
+	var data struct {
+		Slot  common.Decimal `json:"slot"`
+		Block common.Hash    `json:"block"`
+	}
+	if err := json.Unmarshal(enc, &data); err != nil {
+		return 0, common.Hash{}, err
+	}
+	return uint64(data.Slot), data.Block, nil
+}
+
+type HeadEventListener struct {
+	OnNewHead    func(slot uint64, blockRoot common.Hash)
+	OnSignedHead func(head types.SignedHeader)
+	OnFinality   func(head types.FinalityUpdate)
+	OnError      func(err error)
+}
+
+// StartHeadListener creates an event subscription for heads and signed (optimistic)
+// head updates and calls the specified callback functions when they are received.
+// The callbacks are also called for the current head and optimistic head at startup.
+// They are never called concurrently.
+func (api *BeaconLightApi) StartHeadListener(listener HeadEventListener) func() {
+	closeCh := make(chan struct{})   // initiate closing the stream
+	closedCh := make(chan struct{})  // stream closed (or failed to create)
+	stoppedCh := make(chan struct{}) // sync loop stopped
+	streamCh := make(chan *eventsource.Stream, 1)
+	go func() {
+		defer close(closedCh)
+		// when connected to a Lodestar node the subscription blocks until the
+		// first actual event arrives; therefore we create the subscription in
+		// a separate goroutine while letting the main goroutine sync up to the
+		// current head
+		req, err := http.NewRequest("GET", api.url+
+			"/eth/v1/events?topics=head&topics=light_client_optimistic_update&topics=light_client_finality_update", nil)
+		if err != nil {
+			listener.OnError(fmt.Errorf("error creating event subscription request: %v", err))
+			return
+		}
+		for k, v := range api.customHeaders {
+			req.Header.Set(k, v)
+		}
+		stream, err := eventsource.SubscribeWithRequest("", req)
+		if err != nil {
+			listener.OnError(fmt.Errorf("error creating event subscription: %v", err))
+			close(streamCh)
+			return
+		}
+		streamCh <- stream
+		<-closeCh
+		stream.Close()
+	}()
+
+	go func() {
+		defer close(stoppedCh)
+
+		if head, err := api.GetHeader(common.Hash{}); err == nil {
+			listener.OnNewHead(head.Slot, head.Hash())
+		}
+		if signedHead, err := api.GetOptimisticHeadUpdate(); err == nil {
+			listener.OnSignedHead(signedHead)
+		}
+		if finalityUpdate, err := api.GetFinalityUpdate(); err == nil {
+			listener.OnFinality(finalityUpdate)
+		}
+		stream := <-streamCh
+		if stream == nil {
+			return
+		}
+
+		for {
+			select {
+			case event, ok := <-stream.Events:
+				if !ok {
+					break
+				}
+				switch event.Event() {
+				case "head":
+					if slot, blockRoot, err := decodeHeadEvent([]byte(event.Data())); err == nil {
+						listener.OnNewHead(slot, blockRoot)
+					} else {
+						listener.OnError(fmt.Errorf("error decoding head event: %v", err))
+					}
+				case "light_client_optimistic_update":
+					if signedHead, err := decodeOptimisticHeadUpdate([]byte(event.Data())); err == nil {
+						listener.OnSignedHead(signedHead)
+					} else {
+						listener.OnError(fmt.Errorf("error decoding optimistic update event: %v", err))
+					}
+				case "light_client_finality_update":
+					if finalityUpdate, err := decodeFinalityUpdate([]byte(event.Data())); err == nil {
+						listener.OnFinality(finalityUpdate)
+					} else {
+						listener.OnError(fmt.Errorf("error decoding finality update event: %v", err))
+					}
+				default:
+					listener.OnError(fmt.Errorf("unexpected event: %s", event.Event()))
+				}
+			case err, ok := <-stream.Errors:
+				if !ok {
+					break
+				}
+				listener.OnError(err)
+			}
+		}
+	}()
+	return func() {
+		close(closeCh)
+		<-closedCh
+		<-stoppedCh
+	}
+}

beacon/light/committee_chain.go

@@ -70,6 +70,7 @@ type CommitteeChain struct {
 	committees          *canonicalStore[*types.SerializedSyncCommittee]
 	fixedCommitteeRoots *canonicalStore[common.Hash]
 	committeeCache      *lru.Cache[uint64, syncCommittee] // cache deserialized committees
+	changeCounter       uint64
 
 	clock       mclock.Clock         // monotonic clock (simulated clock in tests)
 	unixNano    func() int64         // system clock (simulated clock in tests)
@@ -86,6 +87,11 @@ func NewCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signer
 	return newCommitteeChain(db, config, signerThreshold, enforceTime, blsVerifier{}, &mclock.System{}, func() int64 { return time.Now().UnixNano() })
 }
 
+// NewTestCommitteeChain creates a new CommitteeChain for testing.
+func NewTestCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool, clock *mclock.Simulated) *CommitteeChain {
+	return newCommitteeChain(db, config, signerThreshold, enforceTime, dummyVerifier{}, clock, func() int64 { return int64(clock.Now()) })
+}
+
 // newCommitteeChain creates a new CommitteeChain with the option of replacing the
 // clock source and signature verification for testing purposes.
 func newCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool, sigVerifier committeeSigVerifier, clock mclock.Clock, unixNano func() int64) *CommitteeChain {
@@ -181,20 +187,20 @@ func (s *CommitteeChain) Reset() {
 	if err := s.rollback(0); err != nil {
 		log.Error("Error writing batch into chain database", "error", err)
 	}
+	s.changeCounter++
 }
 
-// CheckpointInit initializes a CommitteeChain based on the checkpoint.
+// CheckpointInit initializes a CommitteeChain based on a checkpoint.
 // Note: if the chain is already initialized and the committees proven by the
 // checkpoint do match the existing chain then the chain is retained and the
 // new checkpoint becomes fixed.
-func (s *CommitteeChain) CheckpointInit(bootstrap *types.BootstrapData) error {
+func (s *CommitteeChain) CheckpointInit(bootstrap types.BootstrapData) error {
 	s.chainmu.Lock()
 	defer s.chainmu.Unlock()
 
 	if err := bootstrap.Validate(); err != nil {
 		return err
 	}
-
 	period := bootstrap.Header.SyncPeriod()
 	if err := s.deleteFixedCommitteeRootsFrom(period + 2); err != nil {
 		s.Reset()
@@ -215,6 +221,7 @@ func (s *CommitteeChain) CheckpointInit(bootstrap *types.BootstrapData) error {
 		s.Reset()
 		return err
 	}
+	s.changeCounter++
 	return nil
 }
 
@@ -367,6 +374,7 @@ func (s *CommitteeChain) InsertUpdate(update *types.LightClientUpdate, nextCommi
 			return ErrWrongCommitteeRoot
 		}
 	}
+	s.changeCounter++
 	if reorg {
 		if err := s.rollback(period + 1); err != nil {
 			return err
@@ -405,6 +413,13 @@ func (s *CommitteeChain) NextSyncPeriod() (uint64, bool) {
 	return s.committees.periods.End - 1, true
 }
 
+func (s *CommitteeChain) ChangeCounter() uint64 {
+	s.chainmu.RLock()
+	defer s.chainmu.RUnlock()
+
+	return s.changeCounter
+}
+
 // rollback removes all committees and fixed roots from the given period and updates
 // starting from the previous period.
 func (s *CommitteeChain) rollback(period uint64) error {
@@ -452,12 +467,12 @@ func (s *CommitteeChain) getSyncCommittee(period uint64) (syncCommittee, error)
 	if sc, ok := s.committees.get(s.db, period); ok {
 		c, err := s.sigVerifier.deserializeSyncCommittee(sc)
 		if err != nil {
-			return nil, fmt.Errorf("Sync committee #%d deserialization error: %v", period, err)
+			return nil, fmt.Errorf("sync committee #%d deserialization error: %v", period, err)
 		}
 		s.committeeCache.Add(period, c)
 		return c, nil
 	}
-	return nil, fmt.Errorf("Missing serialized sync committee #%d", period)
+	return nil, fmt.Errorf("missing serialized sync committee #%d", period)
 }
 
 // VerifySignedHeader returns true if the given signed header has a valid signature

beacon/light/committee_chain_test.go

@@ -241,12 +241,12 @@ func newCommitteeChainTest(t *testing.T, config types.ChainConfig, signerThresho
 		signerThreshold: signerThreshold,
 		enforceTime:     enforceTime,
 	}
-	c.chain = newCommitteeChain(c.db, &config, signerThreshold, enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) })
+	c.chain = NewTestCommitteeChain(c.db, &config, signerThreshold, enforceTime, c.clock)
 	return c
 }
 
 func (c *committeeChainTest) reloadChain() {
-	c.chain = newCommitteeChain(c.db, &c.config, c.signerThreshold, c.enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) })
+	c.chain = NewTestCommitteeChain(c.db, &c.config, c.signerThreshold, c.enforceTime, c.clock)
 }
 
 func (c *committeeChainTest) setClockPeriod(period float64) {

beacon/light/head_tracker.go

@@ -0,0 +1,150 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package light
+
+import (
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// HeadTracker keeps track of the latest validated head and the "prefetch" head
+// which is the (not necessarily validated) head announced by the majority of
+// servers.
+type HeadTracker struct {
+	lock              sync.RWMutex
+	committeeChain    *CommitteeChain
+	minSignerCount    int
+	signedHead        types.SignedHeader
+	hasSignedHead     bool
+	finalityUpdate    types.FinalityUpdate
+	hasFinalityUpdate bool
+	prefetchHead      types.HeadInfo
+	changeCounter     uint64
+}
+
+// NewHeadTracker creates a new HeadTracker.
+func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int) *HeadTracker {
+	return &HeadTracker{
+		committeeChain: committeeChain,
+		minSignerCount: minSignerCount,
+	}
+}
+
+// ValidatedHead returns the latest validated head.
+func (h *HeadTracker) ValidatedHead() (types.SignedHeader, bool) {
+	h.lock.RLock()
+	defer h.lock.RUnlock()
+
+	return h.signedHead, h.hasSignedHead
+}
+
+// ValidatedHead returns the latest validated head.
+func (h *HeadTracker) ValidatedFinality() (types.FinalityUpdate, bool) {
+	h.lock.RLock()
+	defer h.lock.RUnlock()
+
+	return h.finalityUpdate, h.hasFinalityUpdate
+}
+
+// Validate validates the given signed head. If the head is successfully validated
+// and it is better than the old validated head (higher slot or same slot and more
+// signers) then ValidatedHead is updated. The boolean return flag signals if
+// ValidatedHead has been changed.
+func (h *HeadTracker) ValidateHead(head types.SignedHeader) (bool, error) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	replace, err := h.validate(head, h.signedHead)
+	if replace {
+		h.signedHead, h.hasSignedHead = head, true
+		h.changeCounter++
+	}
+	return replace, err
+}
+
+func (h *HeadTracker) ValidateFinality(update types.FinalityUpdate) (bool, error) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	replace, err := h.validate(update.SignedHeader(), h.finalityUpdate.SignedHeader())
+	if replace {
+		h.finalityUpdate, h.hasFinalityUpdate = update, true
+		h.changeCounter++
+	}
+	return replace, err
+}
+
+func (h *HeadTracker) validate(head, oldHead types.SignedHeader) (bool, error) {
+	signerCount := head.Signature.SignerCount()
+	if signerCount < h.minSignerCount {
+		return false, errors.New("low signer count")
+	}
+	if head.Header.Slot < oldHead.Header.Slot || (head.Header.Slot == oldHead.Header.Slot && signerCount <= oldHead.Signature.SignerCount()) {
+		return false, nil
+	}
+	sigOk, age, err := h.committeeChain.VerifySignedHeader(head)
+	if err != nil {
+		return false, err
+	}
+	if age < 0 {
+		log.Warn("Future signed head received", "age", age)
+	}
+	if age > time.Minute*2 {
+		log.Warn("Old signed head received", "age", age)
+	}
+	if !sigOk {
+		return false, errors.New("invalid header signature")
+	}
+	return true, nil
+}
+
+// PrefetchHead returns the latest known prefetch head's head info.
+// This head can be used to start fetching related data hoping that it will be
+// validated soon.
+// Note that the prefetch head cannot be validated cryptographically so it should
+// only be used as a performance optimization hint.
+func (h *HeadTracker) PrefetchHead() types.HeadInfo {
+	h.lock.RLock()
+	defer h.lock.RUnlock()
+
+	return h.prefetchHead
+}
+
+// SetPrefetchHead sets the prefetch head info.
+// Note that HeadTracker does not verify the prefetch head, just acts as a thread
+// safe bulletin board.
+func (h *HeadTracker) SetPrefetchHead(head types.HeadInfo) {
+	h.lock.Lock()
+	defer h.lock.Unlock()
+
+	if head == h.prefetchHead {
+		return
+	}
+	h.prefetchHead = head
+	h.changeCounter++
+}
+
+func (h *HeadTracker) ChangeCounter() uint64 {
+	h.lock.RLock()
+	defer h.lock.RUnlock()
+
+	return h.changeCounter
+}

beacon/light/request/scheduler.go

@@ -0,0 +1,401 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package request
+
+import (
+	"sync"
+
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// Module represents a mechanism which is typically responsible for downloading
+// and updating a passive data structure. It does not directly interact with the
+// servers. It can start requests using the Requester interface, maintain its
+// internal state by receiving and processing Events and update its target data
+// structure based on the obtained data.
+// It is the Scheduler's responsibility to feed events to the modules, call
+// Process as long as there might be something to process and then generate request
+// candidates using MakeRequest and start the best possible requests.
+// Modules are called by Scheduler whenever a global trigger is fired. All events
+// fire the trigger. Changing a target data structure also triggers a next
+// processing round as it could make further actions possible either by the same
+// or another Module.
+type Module interface {
+	// Process is a non-blocking function responsible for starting requests,
+	// processing events and updating the target data structures(s) and the
+	// internal state of the module. Module state typically consists of information
+	// about pending requests and registered servers.
+	// Process is always called after an event is received or after a target data
+	// structure has been changed.
+	//
+	// Note: Process functions of different modules are never called concurrently;
+	// they are called by Scheduler in the same order of priority as they were
+	// registered in.
+	Process(Requester, []Event)
+}
+
+// Requester allows Modules to obtain the list of momentarily available servers,
+// start new requests and report server failure when a response has been proven
+// to be invalid in the processing phase.
+// Note that all Requester functions should be safe to call from Module.Process.
+type Requester interface {
+	CanSendTo() []Server
+	Send(Server, Request) ID
+	Fail(Server, string)
+}
+
+// Scheduler is a modular network data retrieval framework that coordinates multiple
+// servers and retrieval mechanisms (modules). It implements a trigger mechanism
+// that calls the Process function of registered modules whenever either the state
+// of existing data structures or events coming from registered servers could
+// allow new operations.
+type Scheduler struct {
+	lock    sync.Mutex
+	modules []Module // first has highest priority
+	names   map[Module]string
+	servers map[server]struct{}
+	targets map[targetData]uint64
+
+	requesterLock sync.RWMutex
+	serverOrder   []server
+	pending       map[ServerAndID]pendingRequest
+
+	// eventLock guards access to the events list. Note that eventLock can be
+	// locked either while lock is locked or unlocked but lock cannot be locked
+	// while eventLock is locked.
+	eventLock sync.Mutex
+	events    []Event
+	stopCh    chan chan struct{}
+
+	triggerCh chan struct{} // restarts waiting sync loop
+	// if trigger has already been fired then send to testWaitCh blocks until
+	// the triggered processing round is finished
+	testWaitCh chan struct{}
+}
+
+type (
+	// Server identifies a server without allowing any direct interaction.
+	// Note: server interface is used by Scheduler and Tracker but not used by
+	// the modules that do not interact with them directly.
+	// In order to make module testing easier, Server interface is used in
+	// events and modules.
+	Server      any
+	Request     any
+	Response    any
+	ID          uint64
+	ServerAndID struct {
+		Server Server
+		ID     ID
+	}
+)
+
+// targetData represents a registered target data structure that increases its
+// ChangeCounter whenever it has been changed.
+type targetData interface {
+	ChangeCounter() uint64
+}
+
+// pendingRequest keeps track of sent and not yet finalized requests and their
+// sender modules.
+type pendingRequest struct {
+	request Request
+	module  Module
+}
+
+// NewScheduler creates a new Scheduler.
+func NewScheduler() *Scheduler {
+	s := &Scheduler{
+		servers: make(map[server]struct{}),
+		names:   make(map[Module]string),
+		pending: make(map[ServerAndID]pendingRequest),
+		targets: make(map[targetData]uint64),
+		stopCh:  make(chan chan struct{}),
+		// Note: testWaitCh should not have capacity in order to ensure
+		// that after a trigger happens testWaitCh will block until the resulting
+		// processing round has been finished
+		triggerCh:  make(chan struct{}, 1),
+		testWaitCh: make(chan struct{}),
+	}
+	return s
+}
+
+// RegisterTarget registers a target data structure, ensuring that any changes
+// made to it trigger a new round of Module.Process calls, giving a chance to
+// modules to react to the changes.
+func (s *Scheduler) RegisterTarget(t targetData) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.targets[t] = 0
+}
+
+// RegisterModule registers a module. Should be called before starting the scheduler.
+// In each processing round the order of module processing depends on the order of
+// registration.
+func (s *Scheduler) RegisterModule(m Module, name string) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.modules = append(s.modules, m)
+	s.names[m] = name
+}
+
+// RegisterServer registers a new server.
+func (s *Scheduler) RegisterServer(server server) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.addEvent(Event{Type: EvRegistered, Server: server})
+	server.subscribe(func(event Event) {
+		event.Server = server
+		s.addEvent(event)
+	})
+}
+
+// UnregisterServer removes a registered server.
+func (s *Scheduler) UnregisterServer(server server) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	server.unsubscribe()
+	s.addEvent(Event{Type: EvUnregistered, Server: server})
+}
+
+// Start starts the scheduler. It should be called after registering all modules
+// and before registering any servers.
+func (s *Scheduler) Start() {
+	go s.syncLoop()
+}
+
+// Stop stops the scheduler.
+func (s *Scheduler) Stop() {
+	stop := make(chan struct{})
+	s.stopCh <- stop
+	<-stop
+	s.lock.Lock()
+	for server := range s.servers {
+		server.unsubscribe()
+	}
+	s.servers = nil
+	s.lock.Unlock()
+}
+
+// syncLoop is the main event loop responsible for event/data processing and
+// sending new requests.
+// A round of processing starts whenever the global trigger is fired. Triggers
+// fired during a processing round ensure that there is going to be a next round.
+func (s *Scheduler) syncLoop() {
+	for {
+		s.lock.Lock()
+		s.processRound()
+		s.lock.Unlock()
+	loop:
+		for {
+			select {
+			case stop := <-s.stopCh:
+				close(stop)
+				return
+			case <-s.triggerCh:
+				break loop
+			case <-s.testWaitCh:
+			}
+		}
+	}
+}
+
+// targetChanged returns true if a registered target data structure has been
+// changed since the last call to this function.
+func (s *Scheduler) targetChanged() (changed bool) {
+	for target, counter := range s.targets {
+		if newCounter := target.ChangeCounter(); newCounter != counter {
+			s.targets[target] = newCounter
+			changed = true
+		}
+	}
+	return
+}
+
+// processRound runs an entire processing round. It calls the Process functions
+// of all modules, passing all relevant events and repeating Process calls as
+// long as any changes have been made to the registered target data structures.
+// Once all events have been processed and a stable state has been achieved,
+// requests are generated and sent if necessary and possible.
+func (s *Scheduler) processRound() {
+	for {
+		log.Trace("Processing modules")
+		filteredEvents := s.filterEvents()
+		for _, module := range s.modules {
+			log.Trace("Processing module", "name", s.names[module], "events", len(filteredEvents[module]))
+			module.Process(requester{s, module}, filteredEvents[module])
+		}
+		if !s.targetChanged() {
+			break
+		}
+	}
+}
+
+// Trigger starts a new processing round. If fired during processing, it ensures
+// another full round of processing all modules.
+func (s *Scheduler) Trigger() {
+	select {
+	case s.triggerCh <- struct{}{}:
+	default:
+	}
+}
+
+// addEvent adds an event to be processed in the next round. Note that it can be
+// called regardless of the state of the lock mutex, making it safe for use in
+// the server event callback.
+func (s *Scheduler) addEvent(event Event) {
+	s.eventLock.Lock()
+	s.events = append(s.events, event)
+	s.eventLock.Unlock()
+	s.Trigger()
+}
+
+// filterEvent sorts each Event either as a request event or a server event,
+// depending on its type. Request events are also sorted in a map based on the
+// module that originally initiated the request. It also ensures that no events
+// related to a server are returned before EvRegistered or after EvUnregistered.
+// In case of an EvUnregistered server event it also closes all pending requests
+// to the given server by adding a failed request event (EvFail), ensuring that
+// all requests get finalized and thereby allowing the module logic to be safe
+// and simple.
+func (s *Scheduler) filterEvents() map[Module][]Event {
+	s.eventLock.Lock()
+	events := s.events
+	s.events = nil
+	s.eventLock.Unlock()
+
+	s.requesterLock.Lock()
+	defer s.requesterLock.Unlock()
+
+	filteredEvents := make(map[Module][]Event)
+	for _, event := range events {
+		server := event.Server.(server)
+		if _, ok := s.servers[server]; !ok && event.Type != EvRegistered {
+			continue // before EvRegister or after EvUnregister, discard
+		}
+
+		if event.IsRequestEvent() {
+			sid, _, _ := event.RequestInfo()
+			pending, ok := s.pending[sid]
+			if !ok {
+				continue // request already closed, ignore further events
+			}
+			if event.Type == EvResponse || event.Type == EvFail {
+				delete(s.pending, sid) // final event, close pending request
+			}
+			filteredEvents[pending.module] = append(filteredEvents[pending.module], event)
+		} else {
+			switch event.Type {
+			case EvRegistered:
+				s.servers[server] = struct{}{}
+				s.serverOrder = append(s.serverOrder, nil)
+				copy(s.serverOrder[1:], s.serverOrder[:len(s.serverOrder)-1])
+				s.serverOrder[0] = server
+			case EvUnregistered:
+				s.closePending(event.Server, filteredEvents)
+				delete(s.servers, server)
+				for i, srv := range s.serverOrder {
+					if srv == server {
+						copy(s.serverOrder[i:len(s.serverOrder)-1], s.serverOrder[i+1:])
+						s.serverOrder = s.serverOrder[:len(s.serverOrder)-1]
+						break
+					}
+				}
+			}
+			for _, module := range s.modules {
+				filteredEvents[module] = append(filteredEvents[module], event)
+			}
+		}
+	}
+	return filteredEvents
+}
+
+// closePending closes all pending requests to the given server and adds an EvFail
+// event to properly finalize them
+func (s *Scheduler) closePending(server Server, filteredEvents map[Module][]Event) {
+	for sid, pending := range s.pending {
+		if sid.Server == server {
+			filteredEvents[pending.module] = append(filteredEvents[pending.module], Event{
+				Type:   EvFail,
+				Server: server,
+				Data: RequestResponse{
+					ID:      sid.ID,
+					Request: pending.request,
+				},
+			})
+			delete(s.pending, sid)
+		}
+	}
+}
+
+// requester implements Requester. Note that while requester basically wraps
+// Scheduler (with the added information of the currently processed Module), all
+// functions are safe to call from Module.Process which is running while
+// the Scheduler.lock mutex is held.
+type requester struct {
+	*Scheduler
+	module Module
+}
+
+// CanSendTo returns the list of currently available servers. It also returns
+// them in an order of least to most recently used, ensuring a round-robin usage
+// of suitable servers if the module always chooses the first suitable one.
+func (s requester) CanSendTo() []Server {
+	s.requesterLock.RLock()
+	defer s.requesterLock.RUnlock()
+
+	list := make([]Server, 0, len(s.serverOrder))
+	for _, server := range s.serverOrder {
+		if server.canRequestNow() {
+			list = append(list, server)
+		}
+	}
+	return list
+}
+
+// Send sends a request and adds an entry to Scheduler.pending map, ensuring that
+// related request events will be delivered to the sender Module.
+func (s requester) Send(srv Server, req Request) ID {
+	s.requesterLock.Lock()
+	defer s.requesterLock.Unlock()
+
+	server := srv.(server)
+	id := server.sendRequest(req)
+	sid := ServerAndID{Server: srv, ID: id}
+	s.pending[sid] = pendingRequest{request: req, module: s.module}
+	for i, ss := range s.serverOrder {
+		if ss == server {
+			copy(s.serverOrder[i:len(s.serverOrder)-1], s.serverOrder[i+1:])
+			s.serverOrder[len(s.serverOrder)-1] = server
+			return id
+		}
+	}
+	log.Error("Target server not found in ordered list of registered servers")
+	return id
+}
+
+// Fail should be called when a server delivers invalid or useless information.
+// Calling Fail disables the given server for a period that is initially short
+// but is exponentially growing if it happens frequently. This results in a
+// somewhat fault tolerant operation that avoids hammering servers with requests
+// that they cannot serve but still gives them a chance periodically.
+func (s requester) Fail(srv Server, desc string) {
+	srv.(server).fail(desc)
+}

beacon/light/request/scheduler_test.go

@@ -0,0 +1,122 @@
+package request
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestEventFilter(t *testing.T) {
+	s := NewScheduler()
+	module1 := &testModule{name: "module1"}
+	module2 := &testModule{name: "module2"}
+	s.RegisterModule(module1, "module1")
+	s.RegisterModule(module2, "module2")
+	s.Start()
+	// startup process round without events
+	s.testWaitCh <- struct{}{}
+	module1.expProcess(t, nil)
+	module2.expProcess(t, nil)
+	srv := &testServer{}
+	// register server; both modules should receive server event
+	s.RegisterServer(srv)
+	s.testWaitCh <- struct{}{}
+	module1.expProcess(t, []Event{
+		{Type: EvRegistered, Server: srv},
+	})
+	module2.expProcess(t, []Event{
+		{Type: EvRegistered, Server: srv},
+	})
+	// let module1 send a request
+	srv.canRequest = 1
+	module1.sendReq = testRequest
+	s.Trigger()
+	// in first triggered round module1 sends the request, no events yet
+	s.testWaitCh <- struct{}{}
+	module1.expProcess(t, nil)
+	module2.expProcess(t, nil)
+	// server emits EvTimeout; only module1 should receive it
+	srv.eventCb(Event{Type: EvTimeout, Data: RequestResponse{ID: 1, Request: testRequest}})
+	s.testWaitCh <- struct{}{}
+	module1.expProcess(t, []Event{
+		{Type: EvTimeout, Server: srv, Data: RequestResponse{ID: 1, Request: testRequest}},
+	})
+	module2.expProcess(t, nil)
+	// unregister server; both modules should receive server event
+	s.UnregisterServer(srv)
+	s.testWaitCh <- struct{}{}
+	module1.expProcess(t, []Event{
+		// module1 should also receive EvFail on its pending request
+		{Type: EvFail, Server: srv, Data: RequestResponse{ID: 1, Request: testRequest}},
+		{Type: EvUnregistered, Server: srv},
+	})
+	module2.expProcess(t, []Event{
+		{Type: EvUnregistered, Server: srv},
+	})
+	// response after server unregistered; should be discarded
+	srv.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}})
+	s.testWaitCh <- struct{}{}
+	module1.expProcess(t, nil)
+	module2.expProcess(t, nil)
+	// no more process rounds expected; shut down
+	s.testWaitCh <- struct{}{}
+	module1.expNoMoreProcess(t)
+	module2.expNoMoreProcess(t)
+	s.Stop()
+}
+
+type testServer struct {
+	eventCb    func(Event)
+	lastID     ID
+	canRequest int
+}
+
+func (s *testServer) subscribe(eventCb func(Event)) {
+	s.eventCb = eventCb
+}
+
+func (s *testServer) canRequestNow() bool {
+	return s.canRequest > 0
+}
+
+func (s *testServer) sendRequest(req Request) ID {
+	s.canRequest--
+	s.lastID++
+	return s.lastID
+}
+
+func (s *testServer) fail(string)  {}
+func (s *testServer) unsubscribe() {}
+
+type testModule struct {
+	name      string
+	processed [][]Event
+	sendReq   Request
+}
+
+func (m *testModule) Process(requester Requester, events []Event) {
+	m.processed = append(m.processed, events)
+	if m.sendReq != nil {
+		if cs := requester.CanSendTo(); len(cs) > 0 {
+			requester.Send(cs[0], m.sendReq)
+		}
+	}
+}
+
+func (m *testModule) expProcess(t *testing.T, expEvents []Event) {
+	if len(m.processed) == 0 {
+		t.Errorf("Missing call to %s.Process", m.name)
+		return
+	}
+	events := m.processed[0]
+	m.processed = m.processed[1:]
+	if !reflect.DeepEqual(events, expEvents) {
+		t.Errorf("Call to %s.Process with wrong events (expected %v, got %v)", m.name, expEvents, events)
+	}
+}
+
+func (m *testModule) expNoMoreProcess(t *testing.T) {
+	for len(m.processed) > 0 {
+		t.Errorf("Unexpected call to %s.Process with events %v", m.name, m.processed[0])
+		m.processed = m.processed[1:]
+	}
+}

beacon/light/request/server.go

@@ -0,0 +1,439 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package request
+
+import (
+	"math"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/mclock"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+var (
+	// request events
+	EvResponse = &EventType{Name: "response", requestEvent: true} // data: RequestResponse; sent by requestServer
+	EvFail     = &EventType{Name: "fail", requestEvent: true}     // data: RequestResponse; sent by requestServer
+	EvTimeout  = &EventType{Name: "timeout", requestEvent: true}  // data: RequestResponse; sent by serverWithTimeout
+	// server events
+	EvRegistered      = &EventType{Name: "registered"}      // data: nil; sent by Scheduler
+	EvUnregistered    = &EventType{Name: "unregistered"}    // data: nil; sent by Scheduler
+	EvCanRequestAgain = &EventType{Name: "canRequestAgain"} // data: nil; sent by serverWithLimits
+)
+
+const (
+	softRequestTimeout = time.Second      // allow resending request to a different server but do not cancel yet
+	hardRequestTimeout = time.Second * 10 // cancel request
+)
+
+const (
+	// serverWithLimits parameters
+	parallelAdjustUp     = 0.1                    // adjust parallelLimit up in case of success under full load
+	parallelAdjustDown   = 1                      // adjust parallelLimit down in case of timeout/failure
+	minParallelLimit     = 1                      // parallelLimit lower bound
+	defaultParallelLimit = 3                      // parallelLimit initial value
+	minFailureDelay      = time.Millisecond * 100 // minimum disable time in case of request failure
+	maxFailureDelay      = time.Minute            // maximum disable time in case of request failure
+	maxServerEventBuffer = 5                      // server event allowance buffer limit
+	maxServerEventRate   = time.Second            // server event allowance buffer recharge rate
+)
+
+// requestServer can send requests in a non-blocking way and feed back events
+// through the event callback. After each request it should send back either
+// EvResponse or EvFail. Additionally, it may also send application-defined
+// events that the Modules can interpret.
+type requestServer interface {
+	Subscribe(eventCallback func(Event))
+	SendRequest(ID, Request)
+	Unsubscribe()
+}
+
+// server is implemented by a requestServer wrapped into serverWithTimeout and
+// serverWithLimits and is used by Scheduler.
+// In addition to requestServer functionality, server can also handle timeouts,
+// limit the number of parallel in-flight requests and temporarily disable
+// new requests based on timeouts and response failures.
+type server interface {
+	subscribe(eventCallback func(Event))
+	canRequestNow() bool
+	sendRequest(Request) ID
+	fail(string)
+	unsubscribe()
+}
+
+// NewServer wraps a requestServer and returns a server
+func NewServer(rs requestServer, clock mclock.Clock) server {
+	s := &serverWithLimits{}
+	s.parent = rs
+	s.serverWithTimeout.init(clock)
+	s.init()
+	return s
+}
+
+// EventType identifies an event type, either related to a request or the server
+// in general. Server events can also be externally defined.
+type EventType struct {
+	Name         string
+	requestEvent bool // all request events are pre-defined in request package
+}
+
+// Event describes an event where the type of Data depends on Type.
+// Server field is not required when sent through the event callback; it is filled
+// out when processed by the Scheduler. Note that the Scheduler can also create
+// and send events (EvRegistered, EvUnregistered) directly.
+type Event struct {
+	Type   *EventType
+	Server Server // filled by Scheduler
+	Data   any
+}
+
+// IsRequestEvent returns true if the event is a request event
+func (e *Event) IsRequestEvent() bool {
+	return e.Type.requestEvent
+}
+
+// RequestInfo assumes that the event is a request event and returns its contents
+// in a convenient form.
+func (e *Event) RequestInfo() (ServerAndID, Request, Response) {
+	data := e.Data.(RequestResponse)
+	return ServerAndID{Server: e.Server, ID: data.ID}, data.Request, data.Response
+}
+
+// RequestResponse is the Data type of request events.
+type RequestResponse struct {
+	ID       ID
+	Request  Request
+	Response Response
+}
+
+// serverWithTimeout wraps a requestServer and introduces timeouts.
+// The request's lifecycle is concluded if EvResponse or EvFail emitted by the
+// parent requestServer. If this does not happen until softRequestTimeout then
+// EvTimeout is emitted, after which the final EvResponse or EvFail is still
+// guaranteed to follow.
+// If the parent fails to send this final event for hardRequestTimeout then
+// serverWithTimeout emits EvFail and discards any further events from the
+// parent related to the given request.
+type serverWithTimeout struct {
+	parent       requestServer
+	lock         sync.Mutex
+	clock        mclock.Clock
+	childEventCb func(event Event)
+	timeouts     map[ID]mclock.Timer
+	lastID       ID
+}
+
+// init initializes serverWithTimeout
+func (s *serverWithTimeout) init(clock mclock.Clock) {
+	s.clock = clock
+	s.timeouts = make(map[ID]mclock.Timer)
+}
+
+// subscribe subscribes to events which include parent (requestServer) events
+// plus EvTimeout.
+func (s *serverWithTimeout) subscribe(eventCallback func(event Event)) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.childEventCb = eventCallback
+	s.parent.Subscribe(s.eventCallback)
+}
+
+// sendRequest generated a new request ID, emits EvRequest, sets up the timeout
+// timer, then sends the request through the parent (requestServer).
+func (s *serverWithTimeout) sendRequest(request Request) (reqId ID) {
+	s.lock.Lock()
+	s.lastID++
+	id := s.lastID
+	s.startTimeout(RequestResponse{ID: id, Request: request})
+	s.lock.Unlock()
+	s.parent.SendRequest(id, request)
+	return id
+}
+
+// eventCallback is called by parent (requestServer) event subscription.
+func (s *serverWithTimeout) eventCallback(event Event) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	switch event.Type {
+	case EvResponse, EvFail:
+		id := event.Data.(RequestResponse).ID
+		if timer, ok := s.timeouts[id]; ok {
+			// Note: if stopping the timer is unsuccessful then the resulting AfterFunc
+			// call will just do nothing
+			timer.Stop()
+			delete(s.timeouts, id)
+			s.childEventCb(event)
+		}
+	default:
+		s.childEventCb(event)
+	}
+}
+
+// startTimeout starts a timeout timer for the given request.
+func (s *serverWithTimeout) startTimeout(reqData RequestResponse) {
+	id := reqData.ID
+	s.timeouts[id] = s.clock.AfterFunc(softRequestTimeout, func() {
+		s.lock.Lock()
+		if _, ok := s.timeouts[id]; !ok {
+			s.lock.Unlock()
+			return
+		}
+		s.timeouts[id] = s.clock.AfterFunc(hardRequestTimeout-softRequestTimeout, func() {
+			s.lock.Lock()
+			if _, ok := s.timeouts[id]; !ok {
+				s.lock.Unlock()
+				return
+			}
+			delete(s.timeouts, id)
+			childEventCb := s.childEventCb
+			s.lock.Unlock()
+			childEventCb(Event{Type: EvFail, Data: reqData})
+		})
+		childEventCb := s.childEventCb
+		s.lock.Unlock()
+		childEventCb(Event{Type: EvTimeout, Data: reqData})
+	})
+}
+
+// stop stops all goroutines associated with the server.
+func (s *serverWithTimeout) unsubscribe() {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	for _, timer := range s.timeouts {
+		if timer != nil {
+			timer.Stop()
+		}
+	}
+	s.childEventCb = nil
+	s.parent.Unsubscribe()
+}
+
+// serverWithLimits wraps serverWithTimeout and implements server. It limits the
+// number of parallel in-flight requests and prevents sending new requests when a
+// pending one has already timed out. Server events are also rate limited.
+// It also implements a failure delay mechanism that adds an exponentially growing
+// delay each time a request fails (wrong answer or hard timeout). This makes the
+// syncing mechanism less brittle as temporary failures of the server might happen
+// sometimes, but still avoids hammering a non-functional server with requests.
+type serverWithLimits struct {
+	serverWithTimeout
+	lock                       sync.Mutex
+	childEventCb               func(event Event)
+	softTimeouts               map[ID]struct{}
+	pendingCount, timeoutCount int
+	parallelLimit              float32
+	sendEvent                  bool
+	delayTimer                 mclock.Timer
+	delayCounter               int
+	failureDelayEnd            mclock.AbsTime
+	failureDelay               float64
+	serverEventBuffer          int
+	eventBufferUpdated         mclock.AbsTime
+}
+
+// init initializes serverWithLimits
+func (s *serverWithLimits) init() {
+	s.softTimeouts = make(map[ID]struct{})
+	s.parallelLimit = defaultParallelLimit
+	s.serverEventBuffer = maxServerEventBuffer
+}
+
+// subscribe subscribes to events which include parent (serverWithTimeout) events
+// plus EvCanRequstAgain.
+func (s *serverWithLimits) subscribe(eventCallback func(event Event)) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.childEventCb = eventCallback
+	s.serverWithTimeout.subscribe(s.eventCallback)
+}
+
+// eventCallback is called by parent (serverWithTimeout) event subscription.
+func (s *serverWithLimits) eventCallback(event Event) {
+	s.lock.Lock()
+	var sendCanRequestAgain bool
+	passEvent := true
+	switch event.Type {
+	case EvTimeout:
+		id := event.Data.(RequestResponse).ID
+		s.softTimeouts[id] = struct{}{}
+		s.timeoutCount++
+		s.parallelLimit -= parallelAdjustDown
+		if s.parallelLimit < minParallelLimit {
+			s.parallelLimit = minParallelLimit
+		}
+		log.Debug("Server timeout", "count", s.timeoutCount, "parallelLimit", s.parallelLimit)
+	case EvResponse, EvFail:
+		id := event.Data.(RequestResponse).ID
+		if _, ok := s.softTimeouts[id]; ok {
+			delete(s.softTimeouts, id)
+			s.timeoutCount--
+			log.Debug("Server timeout finalized", "count", s.timeoutCount, "parallelLimit", s.parallelLimit)
+		}
+		if event.Type == EvResponse && s.pendingCount >= int(s.parallelLimit) {
+			s.parallelLimit += parallelAdjustUp
+		}
+		s.pendingCount--
+		if s.canRequest() {
+			sendCanRequestAgain = s.sendEvent
+			s.sendEvent = false
+		}
+		if event.Type == EvFail {
+			s.failLocked("failed request")
+		}
+	default:
+		// server event; check rate limit
+		if s.serverEventBuffer < maxServerEventBuffer {
+			now := s.clock.Now()
+			sinceUpdate := time.Duration(now - s.eventBufferUpdated)
+			if sinceUpdate >= maxServerEventRate*time.Duration(maxServerEventBuffer-s.serverEventBuffer) {
+				s.serverEventBuffer = maxServerEventBuffer
+				s.eventBufferUpdated = now
+			} else {
+				addBuffer := int(sinceUpdate / maxServerEventRate)
+				s.serverEventBuffer += addBuffer
+				s.eventBufferUpdated += mclock.AbsTime(maxServerEventRate * time.Duration(addBuffer))
+			}
+		}
+		if s.serverEventBuffer > 0 {
+			s.serverEventBuffer--
+		} else {
+			passEvent = false
+		}
+	}
+	childEventCb := s.childEventCb
+	s.lock.Unlock()
+	if passEvent {
+		childEventCb(event)
+	}
+	if sendCanRequestAgain {
+		childEventCb(Event{Type: EvCanRequestAgain})
+	}
+}
+
+// sendRequest sends a request through the parent (serverWithTimeout).
+func (s *serverWithLimits) sendRequest(request Request) (reqId ID) {
+	s.lock.Lock()
+	s.pendingCount++
+	s.lock.Unlock()
+	return s.serverWithTimeout.sendRequest(request)
+}
+
+// stop stops all goroutines associated with the server.
+func (s *serverWithLimits) unsubscribe() {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if s.delayTimer != nil {
+		s.delayTimer.Stop()
+		s.delayTimer = nil
+	}
+	s.childEventCb = nil
+	s.serverWithTimeout.unsubscribe()
+}
+
+// canRequest checks whether a new request can be started.
+func (s *serverWithLimits) canRequest() bool {
+	if s.delayTimer != nil || s.pendingCount >= int(s.parallelLimit) || s.timeoutCount > 0 {
+		return false
+	}
+	if s.parallelLimit < minParallelLimit {
+		s.parallelLimit = minParallelLimit
+	}
+	return true
+}
+
+// canRequestNow checks whether a new request can be started, according to the
+// current in-flight request count and parallelLimit, and also the failure delay
+// timer.
+// If it returns false then it is guaranteed that an EvCanRequestAgain will be
+// sent whenever the server becomes available for requesting again.
+func (s *serverWithLimits) canRequestNow() bool {
+	var sendCanRequestAgain bool
+	s.lock.Lock()
+	canRequest := s.canRequest()
+	if canRequest {
+		sendCanRequestAgain = s.sendEvent
+		s.sendEvent = false
+	}
+	childEventCb := s.childEventCb
+	s.lock.Unlock()
+	if sendCanRequestAgain {
+		childEventCb(Event{Type: EvCanRequestAgain})
+	}
+	return canRequest
+}
+
+// delay sets the delay timer to the given duration, disabling new requests for
+// the given period.
+func (s *serverWithLimits) delay(delay time.Duration) {
+	if s.delayTimer != nil {
+		// Note: if stopping the timer is unsuccessful then the resulting AfterFunc
+		// call will just do nothing
+		s.delayTimer.Stop()
+		s.delayTimer = nil
+	}
+
+	s.delayCounter++
+	delayCounter := s.delayCounter
+	log.Debug("Server delay started", "length", delay)
+	s.delayTimer = s.clock.AfterFunc(delay, func() {
+		log.Debug("Server delay ended", "length", delay)
+		var sendCanRequestAgain bool
+		s.lock.Lock()
+		if s.delayTimer != nil && s.delayCounter == delayCounter { // do nothing if there is a new timer now
+			s.delayTimer = nil
+			if s.canRequest() {
+				sendCanRequestAgain = s.sendEvent
+				s.sendEvent = false
+			}
+		}
+		childEventCb := s.childEventCb
+		s.lock.Unlock()
+		if sendCanRequestAgain {
+			childEventCb(Event{Type: EvCanRequestAgain})
+		}
+	})
+}
+
+// fail reports that a response from the server was found invalid by the processing
+// Module, disabling new requests for a dynamically adjused time period.
+func (s *serverWithLimits) fail(desc string) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	s.failLocked(desc)
+}
+
+// failLocked calculates the dynamic failure delay and applies it.
+func (s *serverWithLimits) failLocked(desc string) {
+	log.Debug("Server error", "description", desc)
+	s.failureDelay *= 2
+	now := s.clock.Now()
+	if now > s.failureDelayEnd {
+		s.failureDelay *= math.Pow(2, -float64(now-s.failureDelayEnd)/float64(maxFailureDelay))
+	}
+	if s.failureDelay < float64(minFailureDelay) {
+		s.failureDelay = float64(minFailureDelay)
+	}
+	s.failureDelayEnd = now + mclock.AbsTime(s.failureDelay)
+	s.delay(time.Duration(s.failureDelay))
+}

beacon/light/request/server_test.go

@@ -0,0 +1,158 @@
+package request
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common/mclock"
+)
+
+const (
+	testRequest  = "Life, the Universe, and Everything"
+	testResponse = 42
+)
+
+var testEventType = &EventType{Name: "testEvent"}
+
+func TestServerEvents(t *testing.T) {
+	rs := &testRequestServer{}
+	clock := &mclock.Simulated{}
+	srv := NewServer(rs, clock)
+	var lastEventType *EventType
+	srv.subscribe(func(event Event) { lastEventType = event.Type })
+	evTypeName := func(evType *EventType) string {
+		if evType == nil {
+			return "none"
+		}
+		return evType.Name
+	}
+	expEvent := func(expType *EventType) {
+		if lastEventType != expType {
+			t.Errorf("Wrong event type (expected %s, got %s)", evTypeName(expType), evTypeName(lastEventType))
+		}
+		lastEventType = nil
+	}
+	// user events should simply be passed through
+	rs.eventCb(Event{Type: testEventType})
+	expEvent(testEventType)
+	// send request, soft timeout, then valid response
+	srv.sendRequest(testRequest)
+	clock.WaitForTimers(1)
+	clock.Run(softRequestTimeout)
+	expEvent(EvTimeout)
+	rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}})
+	expEvent(EvResponse)
+	// send request, hard timeout (response after hard timeout should be ignored)
+	srv.sendRequest(testRequest)
+	clock.WaitForTimers(1)
+	clock.Run(softRequestTimeout)
+	expEvent(EvTimeout)
+	clock.WaitForTimers(1)
+	clock.Run(hardRequestTimeout)
+	expEvent(EvFail)
+	rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}})
+	expEvent(nil)
+}
+
+func TestServerParallel(t *testing.T) {
+	rs := &testRequestServer{}
+	srv := NewServer(rs, &mclock.Simulated{})
+	srv.subscribe(func(event Event) {})
+
+	expSend := func(expSent int) {
+		var sent int
+		for sent <= expSent {
+			if !srv.canRequestNow() {
+				break
+			}
+			sent++
+			srv.sendRequest(testRequest)
+		}
+		if sent != expSent {
+			t.Errorf("Wrong number of parallel requests accepted (expected %d, got %d)", expSent, sent)
+		}
+	}
+	// max out parallel allowance
+	expSend(defaultParallelLimit)
+	// 1 answered, should accept 1 more
+	rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}})
+	expSend(1)
+	// 2 answered, should accept 2 more
+	rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 2, Request: testRequest, Response: testResponse}})
+	rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 3, Request: testRequest, Response: testResponse}})
+	expSend(2)
+	// failed request, should decrease allowance and not accept more
+	rs.eventCb(Event{Type: EvFail, Data: RequestResponse{ID: 4, Request: testRequest}})
+	expSend(0)
+	srv.unsubscribe()
+}
+
+func TestServerFail(t *testing.T) {
+	rs := &testRequestServer{}
+	clock := &mclock.Simulated{}
+	srv := NewServer(rs, clock)
+	srv.subscribe(func(event Event) {})
+	expCanRequest := func(expCanRequest bool) {
+		if canRequest := srv.canRequestNow(); canRequest != expCanRequest {
+			t.Errorf("Wrong result for canRequestNow (expected %v, got %v)", expCanRequest, canRequest)
+		}
+	}
+	// timed out request
+	expCanRequest(true)
+	srv.sendRequest(testRequest)
+	clock.WaitForTimers(1)
+	expCanRequest(true)
+	clock.Run(softRequestTimeout)
+	expCanRequest(false) // cannot request when there is a timed out request
+	rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}})
+	expCanRequest(true)
+	// explicit server.Fail
+	srv.fail("")
+	clock.WaitForTimers(1)
+	expCanRequest(false) // cannot request for a while after a failure
+	clock.Run(minFailureDelay)
+	expCanRequest(true)
+	// request returned with EvFail
+	srv.sendRequest(testRequest)
+	rs.eventCb(Event{Type: EvFail, Data: RequestResponse{ID: 2, Request: testRequest}})
+	clock.WaitForTimers(1)
+	expCanRequest(false) // EvFail should also start failure delay
+	clock.Run(minFailureDelay)
+	expCanRequest(false) // second failure delay is longer, should still be disabled
+	clock.Run(minFailureDelay)
+	expCanRequest(true)
+	srv.unsubscribe()
+}
+
+func TestServerEventRateLimit(t *testing.T) {
+	rs := &testRequestServer{}
+	clock := &mclock.Simulated{}
+	srv := NewServer(rs, clock)
+	var eventCount int
+	srv.subscribe(func(event Event) {
+		if !event.IsRequestEvent() {
+			eventCount++
+		}
+	})
+	expEvents := func(send, expAllowed int) {
+		eventCount = 0
+		for sent := 0; sent < send; sent++ {
+			rs.eventCb(Event{Type: testEventType})
+		}
+		if eventCount != expAllowed {
+			t.Errorf("Wrong number of server events passing rate limitation (sent %d, expected %d, got %d)", send, expAllowed, eventCount)
+		}
+	}
+	expEvents(maxServerEventBuffer+5, maxServerEventBuffer)
+	clock.Run(maxServerEventRate)
+	expEvents(5, 1)
+	clock.Run(maxServerEventRate * maxServerEventBuffer * 2)
+	expEvents(maxServerEventBuffer+5, maxServerEventBuffer)
+}
+
+type testRequestServer struct {
+	eventCb func(Event)
+}
+
+func (rs *testRequestServer) Subscribe(eventCb func(Event)) { rs.eventCb = eventCb }
+func (rs *testRequestServer) SendRequest(ID, Request)       {}
+func (rs *testRequestServer) Unsubscribe()                  {}

beacon/light/sync/head_sync.go

@@ -0,0 +1,176 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package sync
+
+import (
+	"github.com/ethereum/go-ethereum/beacon/light/request"
+	"github.com/ethereum/go-ethereum/beacon/types"
+)
+
+type headTracker interface {
+	ValidateHead(head types.SignedHeader) (bool, error)
+	ValidateFinality(head types.FinalityUpdate) (bool, error)
+	SetPrefetchHead(head types.HeadInfo)
+}
+
+// HeadSync implements request.Module; it updates the validated and prefetch
+// heads of HeadTracker based on the EvHead and EvSignedHead events coming from
+// registered servers.
+// It can also postpone the validation of the latest announced signed head
+// until the committee chain is synced up to at least the required period.
+type HeadSync struct {
+	headTracker         headTracker
+	chain               committeeChain
+	nextSyncPeriod      uint64
+	chainInit           bool
+	unvalidatedHeads    map[request.Server]types.SignedHeader
+	unvalidatedFinality map[request.Server]types.FinalityUpdate
+	serverHeads         map[request.Server]types.HeadInfo
+	headServerCount     map[types.HeadInfo]headServerCount
+	headCounter         uint64
+	prefetchHead        types.HeadInfo
+}
+
+// headServerCount is associated with most recently seen head infos; it counts
+// the number of servers currently having the given head info as their announced
+// head and a counter signaling how recent that head is.
+// This data is used for selecting the prefetch head.
+type headServerCount struct {
+	serverCount int
+	headCounter uint64
+}
+
+// NewHeadSync creates a new HeadSync.
+func NewHeadSync(headTracker headTracker, chain committeeChain) *HeadSync {
+	s := &HeadSync{
+		headTracker:         headTracker,
+		chain:               chain,
+		unvalidatedHeads:    make(map[request.Server]types.SignedHeader),
+		unvalidatedFinality: make(map[request.Server]types.FinalityUpdate),
+		serverHeads:         make(map[request.Server]types.HeadInfo),
+		headServerCount:     make(map[types.HeadInfo]headServerCount),
+	}
+	return s
+}
+
+// Process implements request.Module.
+func (s *HeadSync) Process(requester request.Requester, events []request.Event) {
+	for _, event := range events {
+		switch event.Type {
+		case EvNewHead:
+			s.setServerHead(event.Server, event.Data.(types.HeadInfo))
+		case EvNewSignedHead:
+			s.newSignedHead(event.Server, event.Data.(types.SignedHeader))
+		case EvNewFinalityUpdate:
+			s.newFinalityUpdate(event.Server, event.Data.(types.FinalityUpdate))
+		case request.EvUnregistered:
+			s.setServerHead(event.Server, types.HeadInfo{})
+			delete(s.serverHeads, event.Server)
+			delete(s.unvalidatedHeads, event.Server)
+		}
+	}
+
+	nextPeriod, chainInit := s.chain.NextSyncPeriod()
+	if nextPeriod != s.nextSyncPeriod || chainInit != s.chainInit {
+		s.nextSyncPeriod, s.chainInit = nextPeriod, chainInit
+		s.processUnvalidated()
+	}
+}
+
+// newSignedHead handles received signed head; either validates it if the chain
+// is properly synced or stores it for further validation.
+func (s *HeadSync) newSignedHead(server request.Server, signedHead types.SignedHeader) {
+	if !s.chainInit || types.SyncPeriod(signedHead.SignatureSlot) > s.nextSyncPeriod {
+		s.unvalidatedHeads[server] = signedHead
+		return
+	}
+	s.headTracker.ValidateHead(signedHead)
+}
+
+// newSignedHead handles received signed head; either validates it if the chain
+// is properly synced or stores it for further validation.
+func (s *HeadSync) newFinalityUpdate(server request.Server, finalityUpdate types.FinalityUpdate) {
+	if !s.chainInit || types.SyncPeriod(finalityUpdate.SignatureSlot) > s.nextSyncPeriod {
+		s.unvalidatedFinality[server] = finalityUpdate
+		return
+	}
+	s.headTracker.ValidateFinality(finalityUpdate)
+}
+
+// processUnvalidatedHeads iterates the list of unvalidated heads and validates
+// those which can be validated.
+func (s *HeadSync) processUnvalidated() {
+	if !s.chainInit {
+		return
+	}
+	for server, signedHead := range s.unvalidatedHeads {
+		if types.SyncPeriod(signedHead.SignatureSlot) <= s.nextSyncPeriod {
+			s.headTracker.ValidateHead(signedHead)
+			delete(s.unvalidatedHeads, server)
+		}
+	}
+	for server, finalityUpdate := range s.unvalidatedFinality {
+		if types.SyncPeriod(finalityUpdate.SignatureSlot) <= s.nextSyncPeriod {
+			s.headTracker.ValidateFinality(finalityUpdate)
+			delete(s.unvalidatedFinality, server)
+		}
+	}
+}
+
+// setServerHead processes non-validated server head announcements and updates
+// the prefetch head if necessary.
+func (s *HeadSync) setServerHead(server request.Server, head types.HeadInfo) bool {
+	if oldHead, ok := s.serverHeads[server]; ok {
+		if head == oldHead {
+			return false
+		}
+		h := s.headServerCount[oldHead]
+		if h.serverCount--; h.serverCount > 0 {
+			s.headServerCount[oldHead] = h
+		} else {
+			delete(s.headServerCount, oldHead)
+		}
+	}
+	if head != (types.HeadInfo{}) {
+		h, ok := s.headServerCount[head]
+		if !ok {
+			s.headCounter++
+			h.headCounter = s.headCounter
+		}
+		h.serverCount++
+		s.headServerCount[head] = h
+		s.serverHeads[server] = head
+	} else {
+		delete(s.serverHeads, server)
+	}
+	var (
+		bestHead     types.HeadInfo
+		bestHeadInfo headServerCount
+	)
+	for head, headServerCount := range s.headServerCount {
+		if headServerCount.serverCount > bestHeadInfo.serverCount ||
+			(headServerCount.serverCount == bestHeadInfo.serverCount && headServerCount.headCounter > bestHeadInfo.headCounter) {
+			bestHead, bestHeadInfo = head, headServerCount
+		}
+	}
+	if bestHead == s.prefetchHead {
+		return false
+	}
+	s.prefetchHead = bestHead
+	s.headTracker.SetPrefetchHead(bestHead)
+	return true
+}

beacon/light/sync/head_sync_test.go

@@ -0,0 +1,151 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package sync
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+var (
+	testServer1 = "testServer1"
+	testServer2 = "testServer2"
+	testServer3 = "testServer3"
+	testServer4 = "testServer4"
+
+	testHead0 = types.HeadInfo{}
+	testHead1 = types.HeadInfo{Slot: 123, BlockRoot: common.Hash{1}}
+	testHead2 = types.HeadInfo{Slot: 124, BlockRoot: common.Hash{2}}
+	testHead3 = types.HeadInfo{Slot: 124, BlockRoot: common.Hash{3}}
+	testHead4 = types.HeadInfo{Slot: 125, BlockRoot: common.Hash{4}}
+
+	testSHead1 = types.SignedHeader{SignatureSlot: 0x0124, Header: types.Header{Slot: 0x0123, StateRoot: common.Hash{1}}}
+	testSHead2 = types.SignedHeader{SignatureSlot: 0x2010, Header: types.Header{Slot: 0x200e, StateRoot: common.Hash{2}}}
+	// testSHead3 is at the end of period 1 but signed in period 2
+	testSHead3 = types.SignedHeader{SignatureSlot: 0x4000, Header: types.Header{Slot: 0x3fff, StateRoot: common.Hash{3}}}
+	testSHead4 = types.SignedHeader{SignatureSlot: 0x6444, Header: types.Header{Slot: 0x6443, StateRoot: common.Hash{4}}}
+)
+
+func TestValidatedHead(t *testing.T) {
+	chain := &TestCommitteeChain{}
+	ht := &TestHeadTracker{}
+	headSync := NewHeadSync(ht, chain)
+	ts := NewTestScheduler(t, headSync)
+
+	ht.ExpValidated(t, 0, nil)
+
+	ts.AddServer(testServer1, 1)
+	ts.ServerEvent(EvNewSignedHead, testServer1, testSHead1)
+	ts.Run(1)
+	// announced head should be queued because of uninitialized chain
+	ht.ExpValidated(t, 1, nil)
+
+	chain.SetNextSyncPeriod(0) // initialize chain
+	ts.Run(2)
+	// expect previously queued head to be validated
+	ht.ExpValidated(t, 2, []types.SignedHeader{testSHead1})
+
+	chain.SetNextSyncPeriod(1)
+	ts.ServerEvent(EvNewSignedHead, testServer1, testSHead2)
+	ts.AddServer(testServer2, 1)
+	ts.ServerEvent(EvNewSignedHead, testServer2, testSHead2)
+	ts.Run(3)
+	// expect both head announcements to be validated instantly
+	ht.ExpValidated(t, 3, []types.SignedHeader{testSHead2, testSHead2})
+
+	ts.ServerEvent(EvNewSignedHead, testServer1, testSHead3)
+	ts.AddServer(testServer3, 1)
+	ts.ServerEvent(EvNewSignedHead, testServer3, testSHead4)
+	ts.Run(4)
+	// future period annonced heads should be queued
+	ht.ExpValidated(t, 4, nil)
+
+	chain.SetNextSyncPeriod(2)
+	ts.Run(5)
+	// testSHead3 can be validated now but not testSHead4
+	ht.ExpValidated(t, 5, []types.SignedHeader{testSHead3})
+
+	// server 3 disconnected without proving period 3, its announced head should be dropped
+	ts.RemoveServer(testServer3)
+	ts.Run(6)
+	ht.ExpValidated(t, 6, nil)
+
+	chain.SetNextSyncPeriod(3)
+	ts.Run(7)
+	// testSHead4 could be validated now but it's not queued by any registered server
+	ht.ExpValidated(t, 7, nil)
+
+	ts.ServerEvent(EvNewSignedHead, testServer2, testSHead4)
+	ts.Run(8)
+	// now testSHead4 should be validated
+	ht.ExpValidated(t, 8, []types.SignedHeader{testSHead4})
+}
+
+func TestPrefetchHead(t *testing.T) {
+	chain := &TestCommitteeChain{}
+	ht := &TestHeadTracker{}
+	headSync := NewHeadSync(ht, chain)
+	ts := NewTestScheduler(t, headSync)
+
+	ht.ExpPrefetch(t, 0, testHead0) // no servers registered
+
+	ts.AddServer(testServer1, 1)
+	ts.ServerEvent(EvNewHead, testServer1, testHead1)
+	ts.Run(1)
+	ht.ExpPrefetch(t, 1, testHead1) // s1: h1
+
+	ts.AddServer(testServer2, 1)
+	ts.ServerEvent(EvNewHead, testServer2, testHead2)
+	ts.Run(2)
+	ht.ExpPrefetch(t, 2, testHead2) // s1: h1, s2: h2
+
+	ts.ServerEvent(EvNewHead, testServer1, testHead2)
+	ts.Run(3)
+	ht.ExpPrefetch(t, 3, testHead2) // s1: h2, s2: h2
+
+	ts.AddServer(testServer3, 1)
+	ts.ServerEvent(EvNewHead, testServer3, testHead3)
+	ts.Run(4)
+	ht.ExpPrefetch(t, 4, testHead2) // s1: h2, s2: h2, s3: h3
+
+	ts.AddServer(testServer4, 1)
+	ts.ServerEvent(EvNewHead, testServer4, testHead4)
+	ts.Run(5)
+	ht.ExpPrefetch(t, 5, testHead2) // s1: h2, s2: h2, s3: h3, s4: h4
+
+	ts.ServerEvent(EvNewHead, testServer2, testHead3)
+	ts.Run(6)
+	ht.ExpPrefetch(t, 6, testHead3) // s1: h2, s2: h3, s3: h3, s4: h4
+
+	ts.RemoveServer(testServer3)
+	ts.Run(7)
+	ht.ExpPrefetch(t, 7, testHead4) // s1: h2, s2: h3, s4: h4
+
+	ts.RemoveServer(testServer1)
+	ts.Run(8)
+	ht.ExpPrefetch(t, 8, testHead4) // s2: h3, s4: h4
+
+	ts.RemoveServer(testServer4)
+	ts.Run(9)
+	ht.ExpPrefetch(t, 9, testHead3) // s2: h3
+
+	ts.RemoveServer(testServer2)
+	ts.Run(10)
+	ht.ExpPrefetch(t, 10, testHead0) // no servers registered
+}

beacon/light/sync/test_helpers.go

@@ -0,0 +1,254 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package sync
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/beacon/light"
+	"github.com/ethereum/go-ethereum/beacon/light/request"
+	"github.com/ethereum/go-ethereum/beacon/types"
+)
+
+type requestWithID struct {
+	sid     request.ServerAndID
+	request request.Request
+}
+
+type TestScheduler struct {
+	t         *testing.T
+	module    request.Module
+	events    []request.Event
+	servers   []request.Server
+	allowance map[request.Server]int
+	sent      map[int][]requestWithID
+	testIndex int
+	expFail   map[request.Server]int // expected Server.Fail calls during next Run
+	lastId    request.ID
+}
+
+func NewTestScheduler(t *testing.T, module request.Module) *TestScheduler {
+	return &TestScheduler{
+		t:         t,
+		module:    module,
+		allowance: make(map[request.Server]int),
+		expFail:   make(map[request.Server]int),
+		sent:      make(map[int][]requestWithID),
+	}
+}
+
+func (ts *TestScheduler) Run(testIndex int, exp ...any) {
+	expReqs := make([]requestWithID, len(exp)/2)
+	id := ts.lastId
+	for i := range expReqs {
+		id++
+		expReqs[i] = requestWithID{
+			sid:     request.ServerAndID{Server: exp[i*2].(request.Server), ID: id},
+			request: exp[i*2+1].(request.Request),
+		}
+	}
+	if len(expReqs) == 0 {
+		expReqs = nil
+	}
+
+	ts.testIndex = testIndex
+	ts.module.Process(ts, ts.events)
+	ts.events = nil
+
+	for server, count := range ts.expFail {
+		delete(ts.expFail, server)
+		if count == 0 {
+			continue
+		}
+		ts.t.Errorf("Missing %d Server.Fail(s) from server %s in test case #%d", count, server.(string), testIndex)
+	}
+
+	if !reflect.DeepEqual(ts.sent[testIndex], expReqs) {
+		ts.t.Errorf("Wrong sent requests in test case #%d (expected %v, got %v)", testIndex, expReqs, ts.sent[testIndex])
+	}
+}
+
+func (ts *TestScheduler) CanSendTo() (cs []request.Server) {
+	for _, server := range ts.servers {
+		if ts.allowance[server] > 0 {
+			cs = append(cs, server)
+		}
+	}
+	return
+}
+
+func (ts *TestScheduler) Send(server request.Server, req request.Request) request.ID {
+	ts.lastId++
+	ts.sent[ts.testIndex] = append(ts.sent[ts.testIndex], requestWithID{
+		sid:     request.ServerAndID{Server: server, ID: ts.lastId},
+		request: req,
+	})
+	ts.allowance[server]--
+	return ts.lastId
+}
+
+func (ts *TestScheduler) Fail(server request.Server, desc string) {
+	if ts.expFail[server] == 0 {
+		ts.t.Errorf("Unexpected Fail from server %s in test case #%d: %s", server.(string), ts.testIndex, desc)
+		return
+	}
+	ts.expFail[server]--
+}
+
+func (ts *TestScheduler) Request(testIndex, reqIndex int) requestWithID {
+	if len(ts.sent[testIndex]) < reqIndex {
+		ts.t.Errorf("Missing request from test case %d index %d", testIndex, reqIndex)
+		return requestWithID{}
+	}
+	return ts.sent[testIndex][reqIndex-1]
+}
+
+func (ts *TestScheduler) ServerEvent(evType *request.EventType, server request.Server, data any) {
+	ts.events = append(ts.events, request.Event{
+		Type:   evType,
+		Server: server,
+		Data:   data,
+	})
+}
+
+func (ts *TestScheduler) RequestEvent(evType *request.EventType, req requestWithID, resp request.Response) {
+	if req.request == nil {
+		return
+	}
+	ts.events = append(ts.events, request.Event{
+		Type:   evType,
+		Server: req.sid.Server,
+		Data: request.RequestResponse{
+			ID:       req.sid.ID,
+			Request:  req.request,
+			Response: resp,
+		},
+	})
+}
+
+func (ts *TestScheduler) AddServer(server request.Server, allowance int) {
+	ts.servers = append(ts.servers, server)
+	ts.allowance[server] = allowance
+	ts.ServerEvent(request.EvRegistered, server, nil)
+}
+
+func (ts *TestScheduler) RemoveServer(server request.Server) {
+	ts.servers = append(ts.servers, server)
+	for i, s := range ts.servers {
+		if s == server {
+			copy(ts.servers[i:len(ts.servers)-1], ts.servers[i+1:])
+			ts.servers = ts.servers[:len(ts.servers)-1]
+			break
+		}
+	}
+	delete(ts.allowance, server)
+	ts.ServerEvent(request.EvUnregistered, server, nil)
+}
+
+func (ts *TestScheduler) AddAllowance(server request.Server, allowance int) {
+	ts.allowance[server] += allowance
+}
+
+func (ts *TestScheduler) ExpFail(server request.Server) {
+	ts.expFail[server]++
+}
+
+type TestCommitteeChain struct {
+	fsp, nsp uint64
+	init     bool
+}
+
+func (t *TestCommitteeChain) CheckpointInit(bootstrap types.BootstrapData) error {
+	t.fsp, t.nsp, t.init = bootstrap.Header.SyncPeriod(), bootstrap.Header.SyncPeriod()+2, true
+	return nil
+}
+
+func (t *TestCommitteeChain) InsertUpdate(update *types.LightClientUpdate, nextCommittee *types.SerializedSyncCommittee) error {
+	period := update.AttestedHeader.Header.SyncPeriod()
+	if period < t.fsp || period > t.nsp || !t.init {
+		return light.ErrInvalidPeriod
+	}
+	if period == t.nsp {
+		t.nsp++
+	}
+	return nil
+}
+
+func (t *TestCommitteeChain) NextSyncPeriod() (uint64, bool) {
+	return t.nsp, t.init
+}
+
+func (tc *TestCommitteeChain) ExpInit(t *testing.T, ExpInit bool) {
+	if tc.init != ExpInit {
+		t.Errorf("Incorrect init flag (expected %v, got %v)", ExpInit, tc.init)
+	}
+}
+
+func (t *TestCommitteeChain) SetNextSyncPeriod(nsp uint64) {
+	t.init, t.nsp = true, nsp
+}
+
+func (tc *TestCommitteeChain) ExpNextSyncPeriod(t *testing.T, expNsp uint64) {
+	tc.ExpInit(t, true)
+	if tc.nsp != expNsp {
+		t.Errorf("Incorrect NextSyncPeriod (expected %d, got %d)", expNsp, tc.nsp)
+	}
+}
+
+type TestHeadTracker struct {
+	phead     types.HeadInfo
+	validated []types.SignedHeader
+}
+
+func (ht *TestHeadTracker) ValidateHead(head types.SignedHeader) (bool, error) {
+	ht.validated = append(ht.validated, head)
+	return true, nil
+}
+
+// TODO add test case for finality
+func (ht *TestHeadTracker) ValidateFinality(head types.FinalityUpdate) (bool, error) {
+	return true, nil
+}
+
+func (ht *TestHeadTracker) ExpValidated(t *testing.T, tci int, expHeads []types.SignedHeader) {
+	for i, expHead := range expHeads {
+		if i >= len(ht.validated) {
+			t.Errorf("Missing validated head in test case #%d index #%d (expected {slot %d blockRoot %x}, got none)", tci, i, expHead.Header.Slot, expHead.Header.Hash())
+			continue
+		}
+		if ht.validated[i] != expHead {
+			vhead := ht.validated[i].Header
+			t.Errorf("Wrong validated head in test case #%d index #%d (expected {slot %d blockRoot %x}, got {slot %d blockRoot %x})", tci, i, expHead.Header.Slot, expHead.Header.Hash(), vhead.Slot, vhead.Hash())
+		}
+	}
+	for i := len(expHeads); i < len(ht.validated); i++ {
+		vhead := ht.validated[i].Header
+		t.Errorf("Unexpected validated head in test case #%d index #%d (expected none, got {slot %d blockRoot %x})", tci, i, vhead.Slot, vhead.Hash())
+	}
+	ht.validated = nil
+}
+
+func (ht *TestHeadTracker) SetPrefetchHead(head types.HeadInfo) {
+	ht.phead = head
+}
+
+func (ht *TestHeadTracker) ExpPrefetch(t *testing.T, tci int, exp types.HeadInfo) {
+	if ht.phead != exp {
+		t.Errorf("Wrong prefetch head in test case #%d (expected {slot %d blockRoot %x}, got {slot %d blockRoot %x})", tci, exp.Slot, exp.BlockRoot, ht.phead.Slot, ht.phead.BlockRoot)
+	}
+}

beacon/light/sync/types.go

@@ -0,0 +1,42 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package sync
+
+import (
+	"github.com/ethereum/go-ethereum/beacon/light/request"
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+var (
+	EvNewHead           = &request.EventType{Name: "newHead"}           // data: types.HeadInfo
+	EvNewSignedHead     = &request.EventType{Name: "newSignedHead"}     // data: types.SignedHeader
+	EvNewFinalityUpdate = &request.EventType{Name: "newFinalityUpdate"} // data: types.FinalityUpdate
+)
+
+type (
+	ReqUpdates struct {
+		FirstPeriod, Count uint64
+	}
+	RespUpdates struct {
+		Updates    []*types.LightClientUpdate
+		Committees []*types.SerializedSyncCommittee
+	}
+	ReqHeader         common.Hash
+	ReqCheckpointData common.Hash
+	ReqBeaconBlock    common.Hash
+)

beacon/light/sync/update_sync.go

@@ -0,0 +1,299 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package sync
+
+import (
+	"sort"
+
+	"github.com/ethereum/go-ethereum/beacon/light"
+	"github.com/ethereum/go-ethereum/beacon/light/request"
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+const maxUpdateRequest = 8 // maximum number of updates requested in a single request
+
+type committeeChain interface {
+	CheckpointInit(bootstrap types.BootstrapData) error
+	InsertUpdate(update *types.LightClientUpdate, nextCommittee *types.SerializedSyncCommittee) error
+	NextSyncPeriod() (uint64, bool)
+}
+
+// CheckpointInit implements request.Module; it fetches the light client bootstrap
+// data belonging to the given checkpoint hash and initializes the committee chain
+// if successful.
+type CheckpointInit struct {
+	chain          committeeChain
+	checkpointHash common.Hash
+	locked         request.ServerAndID
+	initialized    bool
+}
+
+// NewCheckpointInit creates a new CheckpointInit.
+func NewCheckpointInit(chain committeeChain, checkpointHash common.Hash) *CheckpointInit {
+	return &CheckpointInit{
+		chain:          chain,
+		checkpointHash: checkpointHash,
+	}
+}
+
+// Process implements request.Module.
+func (s *CheckpointInit) Process(requester request.Requester, events []request.Event) {
+	for _, event := range events {
+		if !event.IsRequestEvent() {
+			continue
+		}
+		sid, req, resp := event.RequestInfo()
+		if s.locked == sid {
+			s.locked = request.ServerAndID{}
+		}
+		if resp != nil {
+			if checkpoint := resp.(*types.BootstrapData); checkpoint.Header.Hash() == common.Hash(req.(ReqCheckpointData)) {
+				s.chain.CheckpointInit(*checkpoint)
+				s.initialized = true
+				return
+			}
+
+			requester.Fail(event.Server, "invalid checkpoint data")
+		}
+	}
+	// start a request if possible
+	if s.initialized || s.locked != (request.ServerAndID{}) {
+		return
+	}
+	cs := requester.CanSendTo()
+	if len(cs) == 0 {
+		return
+	}
+	server := cs[0]
+	id := requester.Send(server, ReqCheckpointData(s.checkpointHash))
+	s.locked = request.ServerAndID{Server: server, ID: id}
+}
+
+// ForwardUpdateSync implements request.Module; it fetches updates between the
+// committee chain head and each server's announced head. Updates are fetched
+// in batches and multiple batches can also be requested in parallel.
+// Out of order responses are also handled; if a batch of updates cannot be added
+// to the chain immediately because of a gap then the future updates are
+// remembered until they can be processed.
+type ForwardUpdateSync struct {
+	chain          committeeChain
+	rangeLock      rangeLock
+	lockedIDs      map[request.ServerAndID]struct{}
+	processQueue   []updateResponse
+	nextSyncPeriod map[request.Server]uint64
+}
+
+// NewForwardUpdateSync creates a new ForwardUpdateSync.
+func NewForwardUpdateSync(chain committeeChain) *ForwardUpdateSync {
+	return &ForwardUpdateSync{
+		chain:          chain,
+		rangeLock:      make(rangeLock),
+		lockedIDs:      make(map[request.ServerAndID]struct{}),
+		nextSyncPeriod: make(map[request.Server]uint64),
+	}
+}
+
+// rangeLock allows locking sections of an integer space, preventing the syncing
+// mechanism from making requests again for sections where a not timed out request
+// is already pending or where already fetched and unprocessed data is available.
+type rangeLock map[uint64]int
+
+// lock locks or unlocks the given section, depending on the sign of the add parameter.
+func (r rangeLock) lock(first, count uint64, add int) {
+	for i := first; i < first+count; i++ {
+		if v := r[i] + add; v > 0 {
+			r[i] = v
+		} else {
+			delete(r, i)
+		}
+	}
+}
+
+// firstUnlocked returns the first unlocked section starting at or after start
+// and not longer than maxCount.
+func (r rangeLock) firstUnlocked(start, maxCount uint64) (first, count uint64) {
+	first = start
+	for {
+		if _, ok := r[first]; !ok {
+			break
+		}
+		first++
+	}
+	for {
+		count++
+		if count == maxCount {
+			break
+		}
+		if _, ok := r[first+count]; ok {
+			break
+		}
+	}
+	return
+}
+
+// lockRange locks the range belonging to the given update request, unless the
+// same request has already been locked
+func (s *ForwardUpdateSync) lockRange(sid request.ServerAndID, req ReqUpdates) {
+	if _, ok := s.lockedIDs[sid]; ok {
+		return
+	}
+	s.lockedIDs[sid] = struct{}{}
+	s.rangeLock.lock(req.FirstPeriod, req.Count, 1)
+}
+
+// unlockRange unlocks the range belonging to the given update request, unless
+// same request has already been unlocked
+func (s *ForwardUpdateSync) unlockRange(sid request.ServerAndID, req ReqUpdates) {
+	if _, ok := s.lockedIDs[sid]; !ok {
+		return
+	}
+	delete(s.lockedIDs, sid)
+	s.rangeLock.lock(req.FirstPeriod, req.Count, -1)
+}
+
+// verifyRange returns true if the number of updates and the individual update
+// periods in the response match the requested section.
+func (s *ForwardUpdateSync) verifyRange(request ReqUpdates, response RespUpdates) bool {
+	if uint64(len(response.Updates)) != request.Count || uint64(len(response.Committees)) != request.Count {
+		return false
+	}
+	for i, update := range response.Updates {
+		if update.AttestedHeader.Header.SyncPeriod() != request.FirstPeriod+uint64(i) {
+			return false
+		}
+	}
+	return true
+}
+
+// updateResponse is a response that has passed initial verification and has been
+// queued for processing. Note that an update response cannot be processed until
+// the previous updates have also been added to the chain.
+type updateResponse struct {
+	sid      request.ServerAndID
+	request  ReqUpdates
+	response RespUpdates
+}
+
+// updateResponseList implements sort.Sort and sorts update request/response events by FirstPeriod.
+type updateResponseList []updateResponse
+
+func (u updateResponseList) Len() int      { return len(u) }
+func (u updateResponseList) Swap(i, j int) { u[i], u[j] = u[j], u[i] }
+func (u updateResponseList) Less(i, j int) bool {
+	return u[i].request.FirstPeriod < u[j].request.FirstPeriod
+}
+
+// Process implements request.Module.
+func (s *ForwardUpdateSync) Process(requester request.Requester, events []request.Event) {
+	for _, event := range events {
+		switch event.Type {
+		case request.EvResponse, request.EvFail, request.EvTimeout:
+			sid, rq, rs := event.RequestInfo()
+			req := rq.(ReqUpdates)
+			var queued bool
+			if event.Type == request.EvResponse {
+				resp := rs.(RespUpdates)
+				if s.verifyRange(req, resp) {
+					// there is a response with a valid format; put it in the process queue
+					s.processQueue = append(s.processQueue, updateResponse{sid: sid, request: req, response: resp})
+					s.lockRange(sid, req)
+					queued = true
+				} else {
+					requester.Fail(event.Server, "invalid update range")
+				}
+			}
+			if !queued {
+				s.unlockRange(sid, req)
+			}
+		case EvNewSignedHead:
+			signedHead := event.Data.(types.SignedHeader)
+			s.nextSyncPeriod[event.Server] = types.SyncPeriod(signedHead.SignatureSlot + 256)
+		case request.EvUnregistered:
+			delete(s.nextSyncPeriod, event.Server)
+		}
+	}
+
+	// try processing ordered list of available responses
+	sort.Sort(updateResponseList(s.processQueue))
+	for s.processQueue != nil {
+		u := s.processQueue[0]
+		if !s.processResponse(requester, u) {
+			break
+		}
+		s.unlockRange(u.sid, u.request)
+		s.processQueue = s.processQueue[1:]
+		if len(s.processQueue) == 0 {
+			s.processQueue = nil
+		}
+	}
+
+	// start new requests if possible
+	startPeriod, chainInit := s.chain.NextSyncPeriod()
+	if !chainInit {
+		return
+	}
+	for {
+		firstPeriod, maxCount := s.rangeLock.firstUnlocked(startPeriod, maxUpdateRequest)
+		var (
+			sendTo    request.Server
+			bestCount uint64
+		)
+		for _, server := range requester.CanSendTo() {
+			nextPeriod := s.nextSyncPeriod[server]
+			if nextPeriod <= firstPeriod {
+				continue
+			}
+			count := maxCount
+			if nextPeriod < firstPeriod+maxCount {
+				count = nextPeriod - firstPeriod
+			}
+			if count > bestCount {
+				sendTo, bestCount = server, count
+			}
+		}
+		if sendTo == nil {
+			return
+		}
+		req := ReqUpdates{FirstPeriod: firstPeriod, Count: bestCount}
+		id := requester.Send(sendTo, req)
+		s.lockRange(request.ServerAndID{Server: sendTo, ID: id}, req)
+	}
+}
+
+// processResponse adds the fetched updates and committees to the committee chain.
+// Returns true in case of full or partial success.
+func (s *ForwardUpdateSync) processResponse(requester request.Requester, u updateResponse) (success bool) {
+	for i, update := range u.response.Updates {
+		if err := s.chain.InsertUpdate(update, u.response.Committees[i]); err != nil {
+			if err == light.ErrInvalidPeriod {
+				// there is a gap in the update periods; stop processing without
+				// failing and try again next time
+				return
+			}
+			if err == light.ErrInvalidUpdate || err == light.ErrWrongCommitteeRoot || err == light.ErrCannotReorg {
+				requester.Fail(u.sid.Server, "invalid update received")
+			} else {
+				log.Error("Unexpected InsertUpdate error", "error", err)
+			}
+			return
+		}
+		success = true
+	}
+	return
+}

beacon/light/sync/update_sync_test.go

@@ -0,0 +1,219 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package sync
+
+import (
+	"testing"
+
+	"github.com/ethereum/go-ethereum/beacon/light/request"
+	"github.com/ethereum/go-ethereum/beacon/types"
+)
+
+func TestCheckpointInit(t *testing.T) {
+	chain := &TestCommitteeChain{}
+	checkpoint := &types.BootstrapData{Header: types.Header{Slot: 0x2000*4 + 0x1000}} // period 4
+	checkpointHash := checkpoint.Header.Hash()
+	chkInit := NewCheckpointInit(chain, checkpointHash)
+	ts := NewTestScheduler(t, chkInit)
+	// add 2 servers
+	ts.AddServer(testServer1, 1)
+	ts.AddServer(testServer2, 1)
+
+	// expect bootstrap request to server 1
+	ts.Run(1, testServer1, ReqCheckpointData(checkpointHash))
+
+	// server 1 times out; expect request to server 2
+	ts.RequestEvent(request.EvTimeout, ts.Request(1, 1), nil)
+	ts.Run(2, testServer2, ReqCheckpointData(checkpointHash))
+
+	// invalid response from server 2; expect init state to still be false
+	ts.RequestEvent(request.EvResponse, ts.Request(2, 1), &types.BootstrapData{Header: types.Header{Slot: 123456}})
+	ts.ExpFail(testServer2)
+	ts.Run(3)
+	chain.ExpInit(t, false)
+
+	// server 1 fails (hard timeout)
+	ts.RequestEvent(request.EvFail, ts.Request(1, 1), nil)
+	ts.Run(4)
+	chain.ExpInit(t, false)
+
+	// server 3 is registered; expect bootstrap request to server 3
+	ts.AddServer(testServer3, 1)
+	ts.Run(5, testServer3, ReqCheckpointData(checkpointHash))
+
+	// valid response from server 3; expect chain to be initialized
+	ts.RequestEvent(request.EvResponse, ts.Request(5, 1), checkpoint)
+	ts.Run(6)
+	chain.ExpInit(t, true)
+}
+
+func TestUpdateSyncParallel(t *testing.T) {
+	chain := &TestCommitteeChain{}
+	chain.SetNextSyncPeriod(0)
+	updateSync := NewForwardUpdateSync(chain)
+	ts := NewTestScheduler(t, updateSync)
+	// add 2 servers, head at period 100; allow 3-3 parallel requests for each
+	ts.AddServer(testServer1, 3)
+	ts.ServerEvent(EvNewSignedHead, testServer1, types.SignedHeader{SignatureSlot: 0x2000*100 + 0x1000})
+	ts.AddServer(testServer2, 3)
+	ts.ServerEvent(EvNewSignedHead, testServer2, types.SignedHeader{SignatureSlot: 0x2000*100 + 0x1000})
+
+	// expect 6 requests to be sent
+	ts.Run(1,
+		testServer1, ReqUpdates{FirstPeriod: 0, Count: 8},
+		testServer1, ReqUpdates{FirstPeriod: 8, Count: 8},
+		testServer1, ReqUpdates{FirstPeriod: 16, Count: 8},
+		testServer2, ReqUpdates{FirstPeriod: 24, Count: 8},
+		testServer2, ReqUpdates{FirstPeriod: 32, Count: 8},
+		testServer2, ReqUpdates{FirstPeriod: 40, Count: 8})
+
+	// valid response to request 1; expect 8 periods synced and a new request started
+	ts.RequestEvent(request.EvResponse, ts.Request(1, 1), testRespUpdate(ts.Request(1, 1)))
+	ts.AddAllowance(testServer1, 1)
+	ts.Run(2, testServer1, ReqUpdates{FirstPeriod: 48, Count: 8})
+	chain.ExpNextSyncPeriod(t, 8)
+
+	// valid response to requests 4 and 5
+	ts.RequestEvent(request.EvResponse, ts.Request(1, 4), testRespUpdate(ts.Request(1, 4)))
+	ts.RequestEvent(request.EvResponse, ts.Request(1, 5), testRespUpdate(ts.Request(1, 5)))
+	ts.AddAllowance(testServer2, 2)
+	// expect 2 more requests but no sync progress (responses 4 and 5 cannot be added before 2 and 3)
+	ts.Run(3,
+		testServer2, ReqUpdates{FirstPeriod: 56, Count: 8},
+		testServer2, ReqUpdates{FirstPeriod: 64, Count: 8})
+	chain.ExpNextSyncPeriod(t, 8)
+
+	// soft timeout for requests 2 and 3 (server 1 is overloaded)
+	ts.RequestEvent(request.EvTimeout, ts.Request(1, 2), nil)
+	ts.RequestEvent(request.EvTimeout, ts.Request(1, 3), nil)
+	// no allowance, no more requests
+	ts.Run(4)
+
+	// valid response to requests 6 and 8 and 9
+	ts.RequestEvent(request.EvResponse, ts.Request(1, 6), testRespUpdate(ts.Request(1, 6)))
+	ts.RequestEvent(request.EvResponse, ts.Request(3, 1), testRespUpdate(ts.Request(3, 1)))
+	ts.RequestEvent(request.EvResponse, ts.Request(3, 2), testRespUpdate(ts.Request(3, 2)))
+	ts.AddAllowance(testServer2, 3)
+	// server 2 can now resend requests 2 and 3 (timed out by server 1) and also send a new one
+	ts.Run(5,
+		testServer2, ReqUpdates{FirstPeriod: 8, Count: 8},
+		testServer2, ReqUpdates{FirstPeriod: 16, Count: 8},
+		testServer2, ReqUpdates{FirstPeriod: 72, Count: 8})
+
+	// server 1 finally answers timed out request 2
+	ts.RequestEvent(request.EvResponse, ts.Request(1, 2), testRespUpdate(ts.Request(1, 2)))
+	ts.AddAllowance(testServer1, 1)
+	// expect sync progress and one new request
+	ts.Run(6, testServer1, ReqUpdates{FirstPeriod: 80, Count: 8})
+	chain.ExpNextSyncPeriod(t, 16)
+
+	// server 2 answers requests 11 and 12 (resends of requests 2 and 3)
+	ts.RequestEvent(request.EvResponse, ts.Request(5, 1), testRespUpdate(ts.Request(5, 1)))
+	ts.RequestEvent(request.EvResponse, ts.Request(5, 2), testRespUpdate(ts.Request(5, 2)))
+	ts.AddAllowance(testServer2, 2)
+	ts.Run(7,
+		testServer2, ReqUpdates{FirstPeriod: 88, Count: 8},
+		testServer2, ReqUpdates{FirstPeriod: 96, Count: 4})
+	// finally the gap is filled, update can process responses up to req6
+	chain.ExpNextSyncPeriod(t, 48)
+
+	// all remaining requests are answered
+	ts.RequestEvent(request.EvResponse, ts.Request(1, 3), testRespUpdate(ts.Request(1, 3)))
+	ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testRespUpdate(ts.Request(2, 1)))
+	ts.RequestEvent(request.EvResponse, ts.Request(5, 3), testRespUpdate(ts.Request(5, 3)))
+	ts.RequestEvent(request.EvResponse, ts.Request(6, 1), testRespUpdate(ts.Request(6, 1)))
+	ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testRespUpdate(ts.Request(7, 1)))
+	ts.RequestEvent(request.EvResponse, ts.Request(7, 2), testRespUpdate(ts.Request(7, 2)))
+	ts.Run(8)
+	// expect chain to be fully synced
+	chain.ExpNextSyncPeriod(t, 100)
+}
+
+func TestUpdateSyncDifferentHeads(t *testing.T) {
+	chain := &TestCommitteeChain{}
+	chain.SetNextSyncPeriod(10)
+	updateSync := NewForwardUpdateSync(chain)
+	ts := NewTestScheduler(t, updateSync)
+	// add 3 servers with different announced head periods
+	ts.AddServer(testServer1, 1)
+	ts.ServerEvent(EvNewSignedHead, testServer1, types.SignedHeader{SignatureSlot: 0x2000*15 + 0x1000})
+	ts.AddServer(testServer2, 1)
+	ts.ServerEvent(EvNewSignedHead, testServer2, types.SignedHeader{SignatureSlot: 0x2000*16 + 0x1000})
+	ts.AddServer(testServer3, 1)
+	ts.ServerEvent(EvNewSignedHead, testServer3, types.SignedHeader{SignatureSlot: 0x2000*17 + 0x1000})
+
+	// expect request to the best announced head
+	ts.Run(1, testServer3, ReqUpdates{FirstPeriod: 10, Count: 7})
+
+	// request times out, expect request to the next best head
+	ts.RequestEvent(request.EvTimeout, ts.Request(1, 1), nil)
+	ts.Run(2, testServer2, ReqUpdates{FirstPeriod: 10, Count: 6})
+
+	// request times out, expect request to the last available server
+	ts.RequestEvent(request.EvTimeout, ts.Request(2, 1), nil)
+	ts.Run(3, testServer1, ReqUpdates{FirstPeriod: 10, Count: 5})
+
+	// valid response to request 3, expect chain synced to period 15
+	ts.RequestEvent(request.EvResponse, ts.Request(3, 1), testRespUpdate(ts.Request(3, 1)))
+	ts.AddAllowance(testServer1, 1)
+	ts.Run(4)
+	chain.ExpNextSyncPeriod(t, 15)
+
+	// invalid response to request 1, server can only deliver updates up to period 15 despite announced head
+	truncated := ts.Request(1, 1)
+	truncated.request = ReqUpdates{FirstPeriod: 10, Count: 5}
+	ts.RequestEvent(request.EvResponse, ts.Request(1, 1), testRespUpdate(truncated))
+	ts.ExpFail(testServer3)
+	ts.Run(5)
+	// expect no progress of chain head
+	chain.ExpNextSyncPeriod(t, 15)
+
+	// valid response to request 2, expect chain synced to period 16
+	ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testRespUpdate(ts.Request(2, 1)))
+	ts.AddAllowance(testServer2, 1)
+	ts.Run(6)
+	chain.ExpNextSyncPeriod(t, 16)
+
+	// a new server is registered with announced head period 17
+	ts.AddServer(testServer4, 1)
+	ts.ServerEvent(EvNewSignedHead, testServer4, types.SignedHeader{SignatureSlot: 0x2000*17 + 0x1000})
+	// expect request to sync one more period
+	ts.Run(7, testServer4, ReqUpdates{FirstPeriod: 16, Count: 1})
+
+	// valid response, expect chain synced to period 17
+	ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testRespUpdate(ts.Request(7, 1)))
+	ts.AddAllowance(testServer4, 1)
+	ts.Run(8)
+	chain.ExpNextSyncPeriod(t, 17)
+}
+
+func testRespUpdate(request requestWithID) request.Response {
+	var resp RespUpdates
+	if request.request == nil {
+		return resp
+	}
+	req := request.request.(ReqUpdates)
+	resp.Updates = make([]*types.LightClientUpdate, int(req.Count))
+	resp.Committees = make([]*types.SerializedSyncCommittee, int(req.Count))
+	period := req.FirstPeriod
+	for i := range resp.Updates {
+		resp.Updates[i] = &types.LightClientUpdate{AttestedHeader: types.SignedHeader{Header: types.Header{Slot: 0x2000*period + 0x1000}}}
+		resp.Committees[i] = new(types.SerializedSyncCommittee)
+		period++
+	}
+	return resp
+}

beacon/params/params.go

@@ -41,4 +41,6 @@ const (
 	StateIndexNextSyncCommittee = 55
 	StateIndexExecPayload       = 56
 	StateIndexExecHead          = 908
+
+	BodyIndexExecPayload = 25
 )

beacon/types/light_sync.go

@@ -20,11 +20,20 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/beacon/engine"
 	"github.com/ethereum/go-ethereum/beacon/merkle"
 	"github.com/ethereum/go-ethereum/beacon/params"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/protolambda/zrnt/eth2/beacon/capella"
+	"github.com/protolambda/ztyp/tree"
 )
 
+// HeadInfo represents an unvalidated new head announcement.
+type HeadInfo struct {
+	Slot      uint64
+	BlockRoot common.Hash
+}
+
 // BootstrapData contains a sync committee where light sync can be started,
 // together with a proof through a beacon header and corresponding state.
 // Note: BootstrapData is fetched from a server based on a known checkpoint hash.
@@ -134,3 +143,50 @@ func (u UpdateScore) BetterThan(w UpdateScore) bool {
 	}
 	return u.SignerCount > w.SignerCount
 }
+
+type HeaderWithExecProof struct {
+	Header
+	PayloadHeader *capella.ExecutionPayloadHeader
+	PayloadBranch merkle.Values
+}
+
+func (h *HeaderWithExecProof) Validate() error {
+	payloadRoot := merkle.Value(h.PayloadHeader.HashTreeRoot(tree.GetHashFn()))
+	return merkle.VerifyProof(h.BodyRoot, params.BodyIndexExecPayload, h.PayloadBranch, payloadRoot)
+}
+
+type FinalityUpdate struct {
+	Attested, Finalized HeaderWithExecProof
+	FinalityBranch      merkle.Values
+	// Sync committee BLS signature aggregate
+	Signature SyncAggregate
+	// Slot in which the signature has been created (newer than Header.Slot,
+	// determines the signing sync committee)
+	SignatureSlot uint64
+}
+
+func (u *FinalityUpdate) SignedHeader() SignedHeader {
+	return SignedHeader{
+		Header:        u.Attested.Header,
+		Signature:     u.Signature,
+		SignatureSlot: u.SignatureSlot,
+	}
+}
+
+func (u *FinalityUpdate) Validate() error {
+	if err := u.Attested.Validate(); err != nil {
+		return err
+	}
+	if err := u.Finalized.Validate(); err != nil {
+		return err
+	}
+	return merkle.VerifyProof(u.Attested.StateRoot, params.StateIndexFinalBlock, u.FinalityBranch, merkle.Value(u.Finalized.Hash()))
+}
+
+// ChainHeadEvent returns an authenticated execution payload associated with the
+// latest accepted head of the beacon chain, along with the hash of the latest
+// finalized execution block.
+type ChainHeadEvent struct {
+	HeadBlock *engine.ExecutableData
+	Finalized common.Hash
+}

cmd/blsync/engine_api.go

@@ -0,0 +1,69 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"time"
+
+	"github.com/ethereum/go-ethereum/beacon/engine"
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+func updateEngineApi(client *rpc.Client, headCh chan types.ChainHeadEvent) {
+	for event := range headCh {
+		if client == nil { // dry run, no engine API specified
+			log.Info("New execution block retrieved", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "finalized block hash", event.Finalized)
+		} else {
+			if status, err := callNewPayloadV2(client, event.HeadBlock); err == nil {
+				log.Info("Successful NewPayload", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "status", status)
+			} else {
+				log.Error("Failed NewPayload", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "error", err)
+			}
+			if status, err := callForkchoiceUpdatedV1(client, event.HeadBlock.BlockHash, event.Finalized); err == nil {
+				log.Info("Successful ForkchoiceUpdated", "head", event.HeadBlock.BlockHash, "status", status)
+			} else {
+				log.Error("Failed ForkchoiceUpdated", "head", event.HeadBlock.BlockHash, "error", err)
+			}
+		}
+	}
+}
+
+func callNewPayloadV2(client *rpc.Client, execData *engine.ExecutableData) (string, error) {
+	var resp engine.PayloadStatusV1
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	err := client.CallContext(ctx, &resp, "engine_newPayloadV2", execData)
+	cancel()
+	return resp.Status, err
+}
+
+func callForkchoiceUpdatedV1(client *rpc.Client, headHash, finalizedHash common.Hash) (string, error) {
+	var resp engine.ForkChoiceResponse
+	update := engine.ForkchoiceStateV1{
+		HeadBlockHash:      headHash,
+		SafeBlockHash:      finalizedHash,
+		FinalizedBlockHash: finalizedHash,
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	err := client.CallContext(ctx, &resp, "engine_forkchoiceUpdatedV1", update, nil)
+	cancel()
+	return resp.PayloadStatus.Status, err
+}

cmd/blsync/main.go

@@ -0,0 +1,125 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/ethereum/go-ethereum/beacon/blsync"
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/cmd/utils"
+	"github.com/ethereum/go-ethereum/internal/flags"
+	"github.com/ethereum/go-ethereum/log"
+	"github.com/ethereum/go-ethereum/node"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/mattn/go-colorable"
+	"github.com/mattn/go-isatty"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	verbosityFlag = &cli.IntFlag{
+		Name:     "verbosity",
+		Usage:    "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail",
+		Value:    3,
+		Category: flags.LoggingCategory,
+	}
+	vmoduleFlag = &cli.StringFlag{
+		Name:     "vmodule",
+		Usage:    "Per-module verbosity: comma-separated list of <pattern>=<level> (e.g. eth/*=5,p2p=4)",
+		Value:    "",
+		Hidden:   true,
+		Category: flags.LoggingCategory,
+	}
+)
+
+func main() {
+	app := flags.NewApp("beacon light syncer tool")
+	app.Flags = []cli.Flag{
+		utils.BeaconApiFlag,
+		utils.BeaconApiHeaderFlag,
+		utils.BeaconThresholdFlag,
+		utils.BeaconNoFilterFlag,
+		utils.BeaconConfigFlag,
+		utils.BeaconGenesisRootFlag,
+		utils.BeaconGenesisTimeFlag,
+		utils.BeaconCheckpointFlag,
+		//TODO datadir for optional permanent database
+		utils.MainnetFlag,
+		utils.SepoliaFlag,
+		utils.GoerliFlag,
+		utils.BlsyncApiFlag,
+		utils.BlsyncJWTSecretFlag,
+		verbosityFlag,
+		vmoduleFlag,
+	}
+	app.Action = sync
+
+	if err := app.Run(os.Args); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
+}
+
+func sync(ctx *cli.Context) error {
+	usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb"
+	output := io.Writer(os.Stderr)
+	if usecolor {
+		output = colorable.NewColorable(os.Stderr)
+	}
+	verbosity := log.FromLegacyLevel(ctx.Int(verbosityFlag.Name))
+	log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(output, verbosity, usecolor)))
+
+	headCh := make(chan types.ChainHeadEvent, 16)
+	client := blsync.NewClient(ctx)
+	sub := client.SubscribeChainHeadEvent(headCh)
+	go updateEngineApi(makeRPCClient(ctx), headCh)
+	client.Start()
+	// run until stopped
+	<-ctx.Done()
+	client.Stop()
+	sub.Unsubscribe()
+	close(headCh)
+	return nil
+}
+
+func makeRPCClient(ctx *cli.Context) *rpc.Client {
+	if !ctx.IsSet(utils.BlsyncApiFlag.Name) {
+		log.Warn("No engine API target specified, performing a dry run")
+		return nil
+	}
+	if !ctx.IsSet(utils.BlsyncJWTSecretFlag.Name) {
+		utils.Fatalf("JWT secret parameter missing") //TODO use default if datadir is specified
+	}
+
+	engineApiUrl, jwtFileName := ctx.String(utils.BlsyncApiFlag.Name), ctx.String(utils.BlsyncJWTSecretFlag.Name)
+	var jwtSecret [32]byte
+	if jwt, err := node.ObtainJWTSecret(jwtFileName); err == nil {
+		copy(jwtSecret[:], jwt)
+	} else {
+		utils.Fatalf("Error loading or generating JWT secret: %v", err)
+	}
+	auth := node.NewJWTAuth(jwtSecret)
+	cl, err := rpc.DialOptions(context.Background(), engineApiUrl, rpc.WithHTTPAuth(auth))
+	if err != nil {
+		utils.Fatalf("Could not create RPC client: %v", err)
+	}
+	return cl
+}

cmd/clef/README.md

@@ -916,7 +916,7 @@ There are a couple of implementation for a UI. We'll try to keep this list up to
 
 | Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters|
 | ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- |
-| QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: |  :+1: (partially)|
+| QtSigner| https://github.com/holiman/qtsigner/ | Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: |  :+1: (partially)|
-| GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: |  :x: |
+| GtkSigner| https://github.com/holiman/gtksigner | Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: |  :x: |
-| Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: |  :x: |
+| Frame | https://github.com/floating/frame/commits/go-signer | Electron-based| :x:| :x:| :x:| :x:| ?| :x: |  :x: |
-| Clef UI| https://github.com/ethereum/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: |  :+1: (approve tx only)|
+| Clef UI| https://github.com/ethereum/clef-ui | Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: |  :+1: (approve tx only)|

cmd/devp2p/internal/ethtest/snap.go

@@ -648,7 +648,7 @@ The server should reject the request.`,
 					0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8}},
 			},
 			nBytes:    5000,
-			expHashes: []common.Hash{common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")},
+			expHashes: []common.Hash{types.EmptyCodeHash},
 		},
 
 		{

cmd/devp2p/main.go

@@ -66,9 +66,15 @@ func commandHasFlag(ctx *cli.Context, flag cli.Flag) bool {
 	for _, name := range names {
 		set[name] = struct{}{}
 	}
-	for _, fn := range ctx.FlagNames() {
+	for _, ctx := range ctx.Lineage() {
-		if _, ok := set[fn]; ok {
+		if ctx.Command != nil {
-			return true
+			for _, f := range ctx.Command.Flags {
+				for _, name := range f.Names() {
+					if _, ok := set[name]; ok {
+						return true
+					}
+				}
+			}
 		}
 	}
 	return false

cmd/era/main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"math/big"
 	"os"
@@ -182,7 +183,7 @@ func open(ctx *cli.Context, epoch uint64) (*era.Era, error) {
 // that the accumulator matches the expected value.
 func verify(ctx *cli.Context) error {
 	if ctx.Args().Len() != 1 {
-		return fmt.Errorf("missing accumulators file")
+		return errors.New("missing accumulators file")
 	}
 
 	roots, err := readHashes(ctx.Args().First())
@@ -203,7 +204,7 @@ func verify(ctx *cli.Context) error {
 	}
 
 	if len(entries) != len(roots) {
-		return fmt.Errorf("number of era1 files should match the number of accumulator hashes")
+		return errors.New("number of era1 files should match the number of accumulator hashes")
 	}
 
 	// Verify each epoch matches the expected root.
@@ -308,7 +309,7 @@ func checkAccumulator(e *era.Era) error {
 func readHashes(f string) ([]common.Hash, error) {
 	b, err := os.ReadFile(f)
 	if err != nil {
-		return nil, fmt.Errorf("unable to open accumulators file")
+		return nil, errors.New("unable to open accumulators file")
 	}
 	s := strings.Split(string(b), "\n")
 	// Remove empty last element, if present.

cmd/evm/internal/t8ntool/block.go

@@ -242,7 +242,7 @@ func readInput(ctx *cli.Context) (*bbInput, error) {
 	if headerStr == stdinSelector || ommersStr == stdinSelector || txsStr == stdinSelector || cliqueStr == stdinSelector {
 		decoder := json.NewDecoder(os.Stdin)
 		if err := decoder.Decode(inputData); err != nil {
-			return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err))
+			return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err))
 		}
 	}
 	if cliqueStr != stdinSelector && cliqueStr != "" {

cmd/evm/internal/t8ntool/transaction.go

@@ -86,7 +86,7 @@ func Transaction(ctx *cli.Context) error {
 	if txStr == stdinSelector {
 		decoder := json.NewDecoder(os.Stdin)
 		if err := decoder.Decode(inputData); err != nil {
-			return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err))
+			return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err))
 		}
 		// Decode the body of already signed transactions
 		body = common.FromHex(inputData.TxRlp)

cmd/evm/internal/t8ntool/transition.go

@@ -135,7 +135,7 @@ func Transition(ctx *cli.Context) error {
 	if allocStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector {
 		decoder := json.NewDecoder(os.Stdin)
 		if err := decoder.Decode(inputData); err != nil {
-			return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err))
+			return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err))
 		}
 	}
 	if allocStr != stdinSelector {

cmd/evm/internal/t8ntool/tx_iterator.go

@@ -127,7 +127,7 @@ func loadTransactions(txStr string, inputData *input, env stEnv, chainConfig *pa
 			return newRlpTxIterator(body), nil
 		}
 		if err := json.Unmarshal(data, &txsWithKeys); err != nil {
-			return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err))
+			return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshalling txs-file: %v", err))
 		}
 	} else {
 		if len(inputData.TxRlp) > 0 {

cmd/evm/internal/t8ntool/utils.go

@@ -33,7 +33,7 @@ func readFile(path, desc string, dest interface{}) error {
 	defer inFile.Close()
 	decoder := json.NewDecoder(inFile)
 	if err := decoder.Decode(dest); err != nil {
-		return NewError(ErrorJson, fmt.Errorf("failed unmarshaling %s file: %v", desc, err))
+		return NewError(ErrorJson, fmt.Errorf("failed unmarshalling %s file: %v", desc, err))
 	}
 	return nil
 }

cmd/geth/chaincmd.go

@@ -444,7 +444,7 @@ func importHistory(ctx *cli.Context) error {
 			return fmt.Errorf("no era1 files found in %s", dir)
 		}
 		if len(networks) > 1 {
-			return fmt.Errorf("multiple networks found, use a network flag to specify desired network")
+			return errors.New("multiple networks found, use a network flag to specify desired network")
 		}
 		network = networks[0]
 	}
@@ -514,13 +514,10 @@ func importPreimages(ctx *cli.Context) error {
 	return nil
 }
 
-func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) {
+func parseDumpConfig(ctx *cli.Context, stack *node.Node, db ethdb.Database) (*state.DumpConfig, common.Hash, error) {
-	db := utils.MakeChainDatabase(ctx, stack, true)
-	defer db.Close()
-
 	var header *types.Header
 	if ctx.NArg() > 1 {
-		return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg())
+		return nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg())
 	}
 	if ctx.NArg() == 1 {
 		arg := ctx.Args().First()
@@ -529,17 +526,17 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
 			if number := rawdb.ReadHeaderNumber(db, hash); number != nil {
 				header = rawdb.ReadHeader(db, hash, *number)
 			} else {
-				return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash)
+				return nil, common.Hash{}, fmt.Errorf("block %x not found", hash)
 			}
 		} else {
 			number, err := strconv.ParseUint(arg, 10, 64)
 			if err != nil {
-				return nil, nil, common.Hash{}, err
+				return nil, common.Hash{}, err
 			}
 			if hash := rawdb.ReadCanonicalHash(db, number); hash != (common.Hash{}) {
 				header = rawdb.ReadHeader(db, hash, number)
 			} else {
-				return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number)
+				return nil, common.Hash{}, fmt.Errorf("header for block %d not found", number)
 			}
 		}
 	} else {
@@ -547,7 +544,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
 		header = rawdb.ReadHeadHeader(db)
 	}
 	if header == nil {
-		return nil, nil, common.Hash{}, errors.New("no head block found")
+		return nil, common.Hash{}, errors.New("no head block found")
 	}
 	startArg := common.FromHex(ctx.String(utils.StartKeyFlag.Name))
 	var start common.Hash
@@ -559,7 +556,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
 		start = crypto.Keccak256Hash(startArg)
 		log.Info("Converting start-address to hash", "address", common.BytesToAddress(startArg), "hash", start.Hex())
 	default:
-		return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg)
+		return nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg)
 	}
 	var conf = &state.DumpConfig{
 		SkipCode:          ctx.Bool(utils.ExcludeCodeFlag.Name),
@@ -571,14 +568,17 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
 	log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(),
 		"skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage,
 		"start", hexutil.Encode(conf.Start), "limit", conf.Max)
-	return conf, db, header.Root, nil
+	return conf, header.Root, nil
 }
 
 func dump(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	conf, db, root, err := parseDumpConfig(ctx, stack)
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+
+	conf, root, err := parseDumpConfig(ctx, stack, db)
 	if err != nil {
 		return err
 	}

cmd/geth/config.go

@@ -31,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/accounts/scwallet"
 	"github.com/ethereum/go-ethereum/accounts/usbwallet"
+	"github.com/ethereum/go-ethereum/beacon/blsync"
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
@@ -221,6 +222,8 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 		}
 		catalyst.RegisterSimulatedBeaconAPIs(stack, simBeacon)
 		stack.RegisterLifecycle(simBeacon)
+	} else if ctx.IsSet(utils.BeaconApiFlag.Name) {
+		stack.RegisterLifecycle(catalyst.NewBlsync(blsync.NewClient(ctx), eth))
 	} else {
 		err := catalyst.Register(stack, eth)
 		if err != nil {

cmd/geth/consolecmd_test.go

@@ -71,7 +71,6 @@ func TestConsoleWelcome(t *testing.T) {
 Welcome to the Geth JavaScript console!
 
 instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
-coinbase: {{.Etherbase}}
 at block: 0 ({{niltime}})
  datadir: {{.Datadir}}
  modules: {{apis}}
@@ -131,7 +130,6 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) {
 	attach.SetTemplateFunc("goarch", func() string { return runtime.GOARCH })
 	attach.SetTemplateFunc("gover", runtime.Version)
 	attach.SetTemplateFunc("gethver", func() string { return params.VersionWithCommit("", "") })
-	attach.SetTemplateFunc("etherbase", func() string { return geth.Etherbase })
 	attach.SetTemplateFunc("niltime", func() string {
 		return time.Unix(1548854791, 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)")
 	})
@@ -144,7 +142,6 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) {
 Welcome to the Geth JavaScript console!
 
 instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
-coinbase: {{etherbase}}
 at block: 0 ({{niltime}}){{if ipc}}
  datadir: {{datadir}}{{end}}
  modules: {{apis}}

cmd/geth/main.go

@@ -30,7 +30,6 @@ import (
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/console/prompt"
-	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/internal/debug"
@@ -116,13 +115,14 @@ var (
 		utils.DiscoveryPortFlag,
 		utils.MaxPeersFlag,
 		utils.MaxPendingPeersFlag,
-		utils.MiningEnabledFlag,
+		utils.MiningEnabledFlag, // deprecated
 		utils.MinerGasLimitFlag,
 		utils.MinerGasPriceFlag,
-		utils.MinerEtherbaseFlag,
+		utils.MinerEtherbaseFlag, // deprecated
 		utils.MinerExtraDataFlag,
 		utils.MinerRecommitIntervalFlag,
-		utils.MinerNewPayloadTimeout,
+		utils.MinerPendingFeeRecipientFlag,
+		utils.MinerNewPayloadTimeoutFlag, // deprecated
 		utils.NATFlag,
 		utils.NoDiscoverFlag,
 		utils.DiscoveryV4Flag,
@@ -146,6 +146,14 @@ var (
 		configFileFlag,
 		utils.LogDebugFlag,
 		utils.LogBacktraceAtFlag,
+		utils.BeaconApiFlag,
+		utils.BeaconApiHeaderFlag,
+		utils.BeaconThresholdFlag,
+		utils.BeaconNoFilterFlag,
+		utils.BeaconConfigFlag,
+		utils.BeaconGenesisRootFlag,
+		utils.BeaconGenesisTimeFlag,
+		utils.BeaconCheckpointFlag,
 	}, utils.NetworkFlags, utils.DatabaseFlags)
 
 	rpcFlags = []cli.Flag{
@@ -421,24 +429,6 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon
 			}
 		}()
 	}
-
-	// Start auxiliary services if enabled
-	if ctx.Bool(utils.MiningEnabledFlag.Name) {
-		// Mining only makes sense if a full Ethereum node is running
-		if ctx.String(utils.SyncModeFlag.Name) == "light" {
-			utils.Fatalf("Light clients do not support mining")
-		}
-		ethBackend, ok := backend.(*eth.EthAPIBackend)
-		if !ok {
-			utils.Fatalf("Ethereum service not running")
-		}
-		// Set the gas price to the limits from the CLI and start mining
-		gasprice := flags.GlobalBig(ctx, utils.MinerGasPriceFlag.Name)
-		ethBackend.TxPool().SetGasTip(gasprice)
-		if err := ethBackend.StartMining(); err != nil {
-			utils.Fatalf("Failed to start mining: %v", err)
-		}
-	}
 }
 
 // unlockAccounts unlocks any account specifically requested.

cmd/geth/snapshot.go

@@ -36,7 +36,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/trie"
-	cli "github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2"
 )
 
 var (
@@ -541,7 +541,10 @@ func dumpState(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 
-	conf, db, root, err := parseDumpConfig(ctx, stack)
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+
+	conf, root, err := parseDumpConfig(ctx, stack, db)
 	if err != nil {
 		return err
 	}

cmd/geth/testdata/clique.json

@@ -8,6 +8,7 @@
     "byzantiumBlock": 0,
     "constantinopleBlock": 0,
     "petersburgBlock": 0,
+    "terminalTotalDifficultyPassed": true,
     "clique": {
       "period": 5,
       "epoch": 30000

cmd/geth/verkle.go

@@ -29,7 +29,7 @@ import (
 	"github.com/ethereum/go-ethereum/internal/flags"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/gballet/go-verkle"
-	cli "github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2"
 )
 
 var (

cmd/utils/cmd.go

@@ -245,7 +245,7 @@ func readList(filename string) ([]string, error) {
 // starting from genesis.
 func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error {
 	if chain.CurrentSnapBlock().Number.BitLen() != 0 {
-		return fmt.Errorf("history import only supported when starting from genesis")
+		return errors.New("history import only supported when starting from genesis")
 	}
 	entries, err := era.ReadDir(dir, network)
 	if err != nil {

cmd/utils/flags.go

@@ -36,6 +36,7 @@ import (
 
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
+	bparams "github.com/ethereum/go-ethereum/beacon/params"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/fdlimit"
 	"github.com/ethereum/go-ethereum/core"
@@ -281,6 +282,58 @@ var (
 		Value:    ethconfig.Defaults.TransactionHistory,
 		Category: flags.StateCategory,
 	}
+	// Beacon client light sync settings
+	BeaconApiFlag = &cli.StringSliceFlag{
+		Name:     "beacon.api",
+		Usage:    "Beacon node (CL) light client API URL. This flag can be given multiple times.",
+		Category: flags.BeaconCategory,
+	}
+	BeaconApiHeaderFlag = &cli.StringSliceFlag{
+		Name:     "beacon.api.header",
+		Usage:    "Pass custom HTTP header fields to the emote beacon node API in \"key:value\" format. This flag can be given multiple times.",
+		Category: flags.BeaconCategory,
+	}
+	BeaconThresholdFlag = &cli.IntFlag{
+		Name:     "beacon.threshold",
+		Usage:    "Beacon sync committee participation threshold",
+		Value:    bparams.SyncCommitteeSupermajority,
+		Category: flags.BeaconCategory,
+	}
+	BeaconNoFilterFlag = &cli.BoolFlag{
+		Name:     "beacon.nofilter",
+		Usage:    "Disable future slot signature filter",
+		Category: flags.BeaconCategory,
+	}
+	BeaconConfigFlag = &cli.StringFlag{
+		Name:     "beacon.config",
+		Usage:    "Beacon chain config YAML file",
+		Category: flags.BeaconCategory,
+	}
+	BeaconGenesisRootFlag = &cli.StringFlag{
+		Name:     "beacon.genesis.gvroot",
+		Usage:    "Beacon chain genesis validators root",
+		Category: flags.BeaconCategory,
+	}
+	BeaconGenesisTimeFlag = &cli.Uint64Flag{
+		Name:     "beacon.genesis.time",
+		Usage:    "Beacon chain genesis time",
+		Category: flags.BeaconCategory,
+	}
+	BeaconCheckpointFlag = &cli.StringFlag{
+		Name:     "beacon.checkpoint",
+		Usage:    "Beacon chain weak subjectivity checkpoint block hash",
+		Category: flags.BeaconCategory,
+	}
+	BlsyncApiFlag = &cli.StringFlag{
+		Name:     "blsync.engine.api",
+		Usage:    "Target EL engine API URL",
+		Category: flags.BeaconCategory,
+	}
+	BlsyncJWTSecretFlag = &cli.StringFlag{
+		Name:     "blsync.jwtsecret",
+		Usage:    "Path to a JWT secret to use for target engine API endpoint",
+		Category: flags.BeaconCategory,
+	}
 	// Transaction pool settings
 	TxPoolLocalsFlag = &cli.StringFlag{
 		Name:     "txpool.locals",
@@ -425,11 +478,6 @@ var (
 	}
 
 	// Miner settings
-	MiningEnabledFlag = &cli.BoolFlag{
-		Name:     "mine",
-		Usage:    "Enable mining",
-		Category: flags.MinerCategory,
-	}
 	MinerGasLimitFlag = &cli.Uint64Flag{
 		Name:     "miner.gaslimit",
 		Usage:    "Target gas ceiling for mined blocks",
@@ -442,11 +490,6 @@ var (
 		Value:    ethconfig.Defaults.Miner.GasPrice,
 		Category: flags.MinerCategory,
 	}
-	MinerEtherbaseFlag = &cli.StringFlag{
-		Name:     "miner.etherbase",
-		Usage:    "0x prefixed public address for block mining rewards",
-		Category: flags.MinerCategory,
-	}
 	MinerExtraDataFlag = &cli.StringFlag{
 		Name:     "miner.extradata",
 		Usage:    "Block extra data set by the miner (default = client version)",
@@ -458,10 +501,9 @@ var (
 		Value:    ethconfig.Defaults.Miner.Recommit,
 		Category: flags.MinerCategory,
 	}
-	MinerNewPayloadTimeout = &cli.DurationFlag{
+	MinerPendingFeeRecipientFlag = &cli.StringFlag{
-		Name:     "miner.newpayload-timeout",
+		Name:     "miner.pending.feeRecipient",
-		Usage:    "Specify the maximum time allowance for creating a new payload",
+		Usage:    "0x prefixed public address for the pending block producer (not used for actual block production)",
-		Value:    ethconfig.Defaults.Miner.NewPayloadTimeout,
 		Category: flags.MinerCategory,
 	}
 
@@ -1268,19 +1310,23 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error
 
 // setEtherbase retrieves the etherbase from the directly specified command line flags.
 func setEtherbase(ctx *cli.Context, cfg *ethconfig.Config) {
-	if !ctx.IsSet(MinerEtherbaseFlag.Name) {
+	if ctx.IsSet(MinerEtherbaseFlag.Name) {
+		log.Warn("Option --miner.etherbase is deprecated as the etherbase is set by the consensus client post-merge")
 		return
 	}
-	addr := ctx.String(MinerEtherbaseFlag.Name)
+	if !ctx.IsSet(MinerPendingFeeRecipientFlag.Name) {
+		return
+	}
+	addr := ctx.String(MinerPendingFeeRecipientFlag.Name)
 	if strings.HasPrefix(addr, "0x") || strings.HasPrefix(addr, "0X") {
 		addr = addr[2:]
 	}
 	b, err := hex.DecodeString(addr)
 	if err != nil || len(b) != common.AddressLength {
-		Fatalf("-%s: invalid etherbase address %q", MinerEtherbaseFlag.Name, addr)
+		Fatalf("-%s: invalid pending block producer address %q", MinerPendingFeeRecipientFlag.Name, addr)
 		return
 	}
-	cfg.Miner.Etherbase = common.BytesToAddress(b)
+	cfg.Miner.PendingFeeRecipient = common.BytesToAddress(b)
 }
 
 // MakePasswordList reads password lines from the file specified by the global --password flag.
@@ -1496,6 +1542,9 @@ func setTxPool(ctx *cli.Context, cfg *legacypool.Config) {
 }
 
 func setMiner(ctx *cli.Context, cfg *miner.Config) {
+	if ctx.Bool(MiningEnabledFlag.Name) {
+		log.Warn("The flag --mine is deprecated and will be removed")
+	}
 	if ctx.IsSet(MinerExtraDataFlag.Name) {
 		cfg.ExtraData = []byte(ctx.String(MinerExtraDataFlag.Name))
 	}
@@ -1508,8 +1557,9 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) {
 	if ctx.IsSet(MinerRecommitIntervalFlag.Name) {
 		cfg.Recommit = ctx.Duration(MinerRecommitIntervalFlag.Name)
 	}
-	if ctx.IsSet(MinerNewPayloadTimeout.Name) {
+	if ctx.IsSet(MinerNewPayloadTimeoutFlag.Name) {
-		cfg.NewPayloadTimeout = ctx.Duration(MinerNewPayloadTimeout.Name)
+		log.Warn("The flag --miner.newpayload-timeout is deprecated and will be removed, please use --miner.recommit")
+		cfg.Recommit = ctx.Duration(MinerNewPayloadTimeoutFlag.Name)
 	}
 }
 
@@ -1668,6 +1718,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 {
 		cfg.TransactionHistory = 0
 		log.Warn("Disabled transaction unindexing for archive node")
+
+		cfg.StateScheme = rawdb.HashScheme
+		log.Warn("Forcing hash state-scheme for archive mode")
 	}
 	if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) {
 		cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100
@@ -1783,8 +1836,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 
 		// Figure out the dev account address.
 		// setEtherbase has been called above, configuring the miner address from command line flags.
-		if cfg.Miner.Etherbase != (common.Address{}) {
+		if cfg.Miner.PendingFeeRecipient != (common.Address{}) {
-			developer = accounts.Account{Address: cfg.Miner.Etherbase}
+			developer = accounts.Account{Address: cfg.Miner.PendingFeeRecipient}
 		} else if accs := ks.Accounts(); len(accs) > 0 {
 			developer = ks.Accounts()[0]
 		} else {
@@ -1795,7 +1848,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 		}
 		// Make sure the address is configured as fee recipient, otherwise
 		// the miner will fail to start.
-		cfg.Miner.Etherbase = developer.Address
+		cfg.Miner.PendingFeeRecipient = developer.Address
 
 		if err := ks.Unlock(developer, passphrase); err != nil {
 			Fatalf("Failed to unlock developer account: %v", err)

cmd/utils/flags_legacy.go

@@ -47,6 +47,9 @@ var DeprecatedFlags = []cli.Flag{
 	LightNoSyncServeFlag,
 	LogBacktraceAtFlag,
 	LogDebugFlag,
+	MinerNewPayloadTimeoutFlag,
+	MinerEtherbaseFlag,
+	MiningEnabledFlag,
 }
 
 var (
@@ -132,6 +135,23 @@ var (
 		Usage:    "Prepends log messages with call-site location (deprecated)",
 		Category: flags.DeprecatedCategory,
 	}
+	// Deprecated February 2024
+	MinerNewPayloadTimeoutFlag = &cli.DurationFlag{
+		Name:     "miner.newpayload-timeout",
+		Usage:    "Specify the maximum time allowance for creating a new payload",
+		Value:    ethconfig.Defaults.Miner.Recommit,
+		Category: flags.MinerCategory,
+	}
+	MinerEtherbaseFlag = &cli.StringFlag{
+		Name:     "miner.etherbase",
+		Usage:    "0x prefixed public address for block mining rewards",
+		Category: flags.MinerCategory,
+	}
+	MiningEnabledFlag = &cli.BoolFlag{
+		Name:     "mine",
+		Usage:    "Enable mining",
+		Category: flags.MinerCategory,
+	}
 )
 
 // showDeprecated displays deprecated flags that will be soon removed from the codebase.

consensus/clique/clique.go

@@ -30,7 +30,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
-	lru "github.com/ethereum/go-ethereum/common/lru"
+	"github.com/ethereum/go-ethereum/common/lru"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/misc"
 	"github.com/ethereum/go-ethereum/consensus/misc/eip1559"

consensus/consensus.go

@@ -119,11 +119,3 @@ type Engine interface {
 	// Close terminates any background threads maintained by the consensus engine.
 	Close() error
 }
-
-// PoW is a consensus engine based on proof-of-work.
-type PoW interface {
-	Engine
-
-	// Hashrate returns the current mining hashrate of a PoW consensus engine.
-	Hashrate() float64
-}

consensus/merger.go

@@ -1,110 +0,0 @@
-// Copyright 2021 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package consensus
-
-import (
-	"fmt"
-	"sync"
-
-	"github.com/ethereum/go-ethereum/core/rawdb"
-	"github.com/ethereum/go-ethereum/ethdb"
-	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/rlp"
-)
-
-// transitionStatus describes the status of eth1/2 transition. This switch
-// between modes is a one-way action which is triggered by corresponding
-// consensus-layer message.
-type transitionStatus struct {
-	LeftPoW    bool // The flag is set when the first NewHead message received
-	EnteredPoS bool // The flag is set when the first FinalisedBlock message received
-}
-
-// Merger is an internal help structure used to track the eth1/2 transition status.
-// It's a common structure can be used in both full node and light client.
-type Merger struct {
-	db     ethdb.KeyValueStore
-	status transitionStatus
-	mu     sync.RWMutex
-}
-
-// NewMerger creates a new Merger which stores its transition status in the provided db.
-func NewMerger(db ethdb.KeyValueStore) *Merger {
-	var status transitionStatus
-	blob := rawdb.ReadTransitionStatus(db)
-	if len(blob) != 0 {
-		if err := rlp.DecodeBytes(blob, &status); err != nil {
-			log.Crit("Failed to decode the transition status", "err", err)
-		}
-	}
-	return &Merger{
-		db:     db,
-		status: status,
-	}
-}
-
-// ReachTTD is called whenever the first NewHead message received
-// from the consensus-layer.
-func (m *Merger) ReachTTD() {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	if m.status.LeftPoW {
-		return
-	}
-	m.status = transitionStatus{LeftPoW: true}
-	blob, err := rlp.EncodeToBytes(m.status)
-	if err != nil {
-		panic(fmt.Sprintf("Failed to encode the transition status: %v", err))
-	}
-	rawdb.WriteTransitionStatus(m.db, blob)
-	log.Info("Left PoW stage")
-}
-
-// FinalizePoS is called whenever the first FinalisedBlock message received
-// from the consensus-layer.
-func (m *Merger) FinalizePoS() {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-
-	if m.status.EnteredPoS {
-		return
-	}
-	m.status = transitionStatus{LeftPoW: true, EnteredPoS: true}
-	blob, err := rlp.EncodeToBytes(m.status)
-	if err != nil {
-		panic(fmt.Sprintf("Failed to encode the transition status: %v", err))
-	}
-	rawdb.WriteTransitionStatus(m.db, blob)
-	log.Info("Entered PoS stage")
-}
-
-// TDDReached reports whether the chain has left the PoW stage.
-func (m *Merger) TDDReached() bool {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-
-	return m.status.LeftPoW
-}
-
-// PoSFinalized reports whether the chain has entered the PoS stage.
-func (m *Merger) PoSFinalized() bool {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-
-	return m.status.EnteredPoS
-}

console/console.go

@@ -325,9 +325,6 @@ func (c *Console) Welcome() {
 	// Print some generic Geth metadata
 	if res, err := c.jsre.Run(`
 		var message = "instance: " + web3.version.node + "\n";
-		try {
-			message += "coinbase: " + eth.coinbase + "\n";
-		} catch (err) {}
 		message += "at block: " + eth.blockNumber + " (" + new Date(1000 * eth.getBlock(eth.blockNumber).timestamp) + ")\n";
 		try {
 			message += " datadir: " + admin.datadir + "\n";

console/console_test.go

@@ -96,7 +96,7 @@ func newTester(t *testing.T, confOverride func(*ethconfig.Config)) *tester {
 	ethConf := &ethconfig.Config{
 		Genesis: core.DeveloperGenesisBlock(11_500_000, nil),
 		Miner: miner.Config{
-			Etherbase: common.HexToAddress(testAddress),
+			PendingFeeRecipient: common.HexToAddress(testAddress),
 		},
 	}
 	if confOverride != nil {
@@ -152,8 +152,7 @@ func (env *tester) Close(t *testing.T) {
 }
 
 // Tests that the node lists the correct welcome message, notably that it contains
-// the instance name, coinbase account, block number, data directory and supported
+// the instance name, block number, data directory and supported console modules.
-// console modules.
 func TestWelcome(t *testing.T) {
 	tester := newTester(t, nil)
 	defer tester.Close(t)
@@ -167,14 +166,14 @@ func TestWelcome(t *testing.T) {
 	if want := fmt.Sprintf("instance: %s", testInstance); !strings.Contains(output, want) {
 		t.Fatalf("console output missing instance: have\n%s\nwant also %s", output, want)
 	}
-	if want := fmt.Sprintf("coinbase: %s", testAddress); !strings.Contains(output, want) {
-		t.Fatalf("console output missing coinbase: have\n%s\nwant also %s", output, want)
-	}
 	if want := "at block: 0"; !strings.Contains(output, want) {
 		t.Fatalf("console output missing sync status: have\n%s\nwant also %s", output, want)
 	}
 	if want := fmt.Sprintf("datadir: %s", tester.workspace); !strings.Contains(output, want) {
-		t.Fatalf("console output missing coinbase: have\n%s\nwant also %s", output, want)
+		t.Fatalf("console output missing datadir: have\n%s\nwant also %s", output, want)
+	}
+	if want := "modules: "; !strings.Contains(output, want) {
+		t.Fatalf("console output missing modules: have\n%s\nwant also %s", output, want)
 	}
 }
 

core/block_validator_test.go

@@ -94,7 +94,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) {
 		preBlocks  []*types.Block
 		postBlocks []*types.Block
 		engine     consensus.Engine
-		merger     = consensus.NewMerger(rawdb.NewMemoryDatabase())
 	)
 	if isClique {
 		var (
@@ -186,11 +185,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) {
 		}
 		chain.InsertChain(preBlocks[i : i+1])
 	}
-
-	// Make the transition
-	merger.ReachTTD()
-	merger.FinalizePoS()
-
 	// Verify the blocks after the merging
 	for i := 0; i < len(postBlocks); i++ {
 		_, results := engine.VerifyHeaders(chain, []*types.Header{postHeaders[i]})

core/blockchain.go

@@ -50,7 +50,6 @@ import (
 	"github.com/ethereum/go-ethereum/triedb"
 	"github.com/ethereum/go-ethereum/triedb/hashdb"
 	"github.com/ethereum/go-ethereum/triedb/pathdb"
-	"golang.org/x/exp/slices"
 )
 
 var (
@@ -97,13 +96,11 @@ var (
 )
 
 const (
-	bodyCacheLimit      = 256
+	bodyCacheLimit     = 256
-	blockCacheLimit     = 256
+	blockCacheLimit    = 256
-	receiptsCacheLimit  = 32
+	receiptsCacheLimit = 32
-	txLookupCacheLimit  = 1024
+	txLookupCacheLimit = 1024
-	maxFutureBlocks     = 256
+	TriesInMemory      = 128
-	maxTimeFutureBlocks = 30
-	TriesInMemory       = 128
 
 	// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
 	//
@@ -245,9 +242,6 @@ type BlockChain struct {
 	blockCache    *lru.Cache[common.Hash, *types.Block]
 	txLookupCache *lru.Cache[common.Hash, txLookup]
 
-	// future blocks are blocks added for later processing
-	futureBlocks *lru.Cache[common.Hash, *types.Block]
-
 	wg            sync.WaitGroup
 	quit          chan struct{} // shutdown signal, closed in Stop.
 	stopping      atomic.Bool   // false if chain is running, true when stopped
@@ -299,7 +293,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 		receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit),
 		blockCache:    lru.NewCache[common.Hash, *types.Block](blockCacheLimit),
 		txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit),
-		futureBlocks:  lru.NewCache[common.Hash, *types.Block](maxFutureBlocks),
 		engine:        engine,
 		vmConfig:      vmConfig,
 	}
@@ -449,11 +442,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
 		}
 		bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root)
 	}
-
-	// Start future block processor.
-	bc.wg.Add(1)
-	go bc.updateFutureBlocks()
-
 	// Rewind the chain in case of an incompatible config upgrade.
 	if compat, ok := genesisErr.(*params.ConfigCompatError); ok {
 		log.Warn("Rewinding chain to upgrade configuration", "err", compat)
@@ -794,7 +782,6 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
 	bc.receiptsCache.Purge()
 	bc.blockCache.Purge()
 	bc.txLookupCache.Purge()
-	bc.futureBlocks.Purge()
 
 	// Clear safe block, finalized block if needed
 	if safe := bc.CurrentSafeBlock(); safe != nil && head < safe.Number.Uint64() {
@@ -1048,24 +1035,6 @@ func (bc *BlockChain) insertStopped() bool {
 	return bc.procInterrupt.Load()
 }
 
-func (bc *BlockChain) procFutureBlocks() {
-	blocks := make([]*types.Block, 0, bc.futureBlocks.Len())
-	for _, hash := range bc.futureBlocks.Keys() {
-		if block, exist := bc.futureBlocks.Peek(hash); exist {
-			blocks = append(blocks, block)
-		}
-	}
-	if len(blocks) > 0 {
-		slices.SortFunc(blocks, func(a, b *types.Block) int {
-			return a.Number().Cmp(b.Number())
-		})
-		// Insert one by one as chain insertion needs contiguous ancestry between blocks
-		for i := range blocks {
-			bc.InsertChain(blocks[i : i+1])
-		}
-	}
-}
-
 // WriteStatus status of write
 type WriteStatus byte
 
@@ -1466,8 +1435,6 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types
 	if status == CanonStatTy {
 		bc.writeHeadBlock(block)
 	}
-	bc.futureBlocks.Remove(block.Hash())
-
 	if status == CanonStatTy {
 		bc.chainFeed.Send(ChainEvent{Block: block, Hash: block.Hash(), Logs: logs})
 		if len(logs) > 0 {
@@ -1487,25 +1454,6 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types
 	return status, nil
 }
 
-// addFutureBlock checks if the block is within the max allowed window to get
-// accepted for future processing, and returns an error if the block is too far
-// ahead and was not added.
-//
-// TODO after the transition, the future block shouldn't be kept. Because
-// it's not checked in the Geth side anymore.
-func (bc *BlockChain) addFutureBlock(block *types.Block) error {
-	max := uint64(time.Now().Unix() + maxTimeFutureBlocks)
-	if block.Time() > max {
-		return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max)
-	}
-	if block.Difficulty().Cmp(common.Big0) == 0 {
-		// Never add PoS blocks into the future queue
-		return nil
-	}
-	bc.futureBlocks.Add(block.Hash(), block)
-	return nil
-}
-
 // InsertChain attempts to insert the given batch of blocks in to the canonical
 // chain or, otherwise, create a fork. If an error is returned it will return
 // the index number of the failing block as well an error describing what went
@@ -1643,26 +1591,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 			_, err := bc.recoverAncestors(block)
 			return it.index, err
 		}
-	// First block is future, shove it (and all children) to the future queue (unknown ancestor)
-	case errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash())):
-		for block != nil && (it.index == 0 || errors.Is(err, consensus.ErrUnknownAncestor)) {
-			log.Debug("Future block, postponing import", "number", block.Number(), "hash", block.Hash())
-			if err := bc.addFutureBlock(block); err != nil {
-				return it.index, err
-			}
-			block, err = it.next()
-		}
-		stats.queued += it.processed()
-		stats.ignored += it.remaining()
-
-		// If there are any still remaining, mark as ignored
-		return it.index, err
-
 	// Some other error(except ErrKnownBlock) occurred, abort.
 	// ErrKnownBlock is allowed here since some known blocks
 	// still need re-execution to generate snapshots that are missing
 	case err != nil && !errors.Is(err, ErrKnownBlock):
-		bc.futureBlocks.Remove(block.Hash())
 		stats.ignored += len(it.chain)
 		bc.reportBlock(block, nil, err)
 		return it.index, err
@@ -1867,23 +1799,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
 				"root", block.Root())
 		}
 	}
-
-	// Any blocks remaining here? The only ones we care about are the future ones
-	if block != nil && errors.Is(err, consensus.ErrFutureBlock) {
-		if err := bc.addFutureBlock(block); err != nil {
-			return it.index, err
-		}
-		block, err = it.next()
-
-		for ; block != nil && errors.Is(err, consensus.ErrUnknownAncestor); block, err = it.next() {
-			if err := bc.addFutureBlock(block); err != nil {
-				return it.index, err
-			}
-			stats.queued++
-		}
-	}
 	stats.ignored += it.remaining()
-
 	return it.index, err
 }
 
@@ -2334,20 +2250,6 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) {
 	return head.Hash(), nil
 }
 
-func (bc *BlockChain) updateFutureBlocks() {
-	futureTimer := time.NewTicker(5 * time.Second)
-	defer futureTimer.Stop()
-	defer bc.wg.Done()
-	for {
-		select {
-		case <-futureTimer.C:
-			bc.procFutureBlocks()
-		case <-bc.quit:
-			return
-		}
-	}
-}
-
 // skipBlock returns 'true', if the block being imported can be skipped over, meaning
 // that the block does not need to be processed but can be considered already fully 'done'.
 func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool {

core/blockchain_insert.go

@@ -179,8 +179,3 @@ func (it *insertIterator) first() *types.Block {
 func (it *insertIterator) remaining() int {
 	return len(it.chain) - it.index
 }
-
-// processed returns the number of processed blocks.
-func (it *insertIterator) processed() int {
-	return it.index + 1
-}

core/blockchain_repair_test.go

@@ -1757,7 +1757,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 
 func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) {
 	// It's hard to follow the test case, visualize the input
-	//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
+	// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
 	// fmt.Println(tt.dump(true))
 
 	// Create a temporary persistent database
@@ -1830,10 +1830,14 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
 	}
 	// Force run a freeze cycle
 	type freezer interface {
-		Freeze(threshold uint64) error
+		Freeze() error
 		Ancients() (uint64, error)
 	}
-	db.(freezer).Freeze(tt.freezeThreshold)
+	if tt.freezeThreshold < uint64(tt.canonicalBlocks) {
+		final := uint64(tt.canonicalBlocks) - tt.freezeThreshold
+		chain.SetFinalized(canonblocks[int(final)-1].Header())
+	}
+	db.(freezer).Freeze()
 
 	// Set the simulated pivot block
 	if tt.pivotBlock != nil {

core/blockchain_sethead_test.go

@@ -2044,10 +2044,14 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
 
 	// Force run a freeze cycle
 	type freezer interface {
-		Freeze(threshold uint64) error
+		Freeze() error
 		Ancients() (uint64, error)
 	}
-	db.(freezer).Freeze(tt.freezeThreshold)
+	if tt.freezeThreshold < uint64(tt.canonicalBlocks) {
+		final := uint64(tt.canonicalBlocks) - tt.freezeThreshold
+		chain.SetFinalized(canonblocks[int(final)-1].Header())
+	}
+	db.(freezer).Freeze()
 
 	// Set the simulated pivot block
 	if tt.pivotBlock != nil {

core/blockchain_test.go

@@ -2129,7 +2129,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon
 	// Generate a canonical chain to act as the main dataset
 	chainConfig := *params.TestChainConfig
 	var (
-		merger = consensus.NewMerger(rawdb.NewMemoryDatabase())
 		engine = beacon.New(ethash.NewFaker())
 		key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
 		addr   = crypto.PubkeyToAddress(key.PublicKey)
@@ -2153,8 +2152,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon
 	// Activate the transition since genesis if required
 	if mergePoint == 0 {
 		mergeBlock = 0
-		merger.ReachTTD()
-		merger.FinalizePoS()
 
 		// Set the terminal total difficulty in the config
 		gspec.Config.TerminalTotalDifficulty = big.NewInt(0)
@@ -2189,8 +2186,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon
 
 	// Activate the transition in the middle of the chain
 	if mergePoint == 1 {
-		merger.ReachTTD()
-		merger.FinalizePoS()
 		// Set the terminal total difficulty in the config
 		ttd := big.NewInt(int64(len(blocks)))
 		ttd.Mul(ttd, params.GenesisDifficulty)

core/rawdb/accessors_trie.go

@@ -315,7 +315,7 @@ func ReadStateScheme(db ethdb.Reader) string {
 // the stored state.
 //
 //   - If the provided scheme is none, use the scheme consistent with persistent
-//     state, or fallback to hash-based scheme if state is empty.
+//     state, or fallback to path-based scheme if state is empty.
 //
 //   - If the provided scheme is hash, use hash-based scheme or error out if not
 //     compatible with persistent state scheme.
@@ -329,10 +329,8 @@ func ParseStateScheme(provided string, disk ethdb.Database) (string, error) {
 	stored := ReadStateScheme(disk)
 	if provided == "" {
 		if stored == "" {
-			// use default scheme for empty database, flip it when
+			log.Info("State schema set to default", "scheme", "path")
-			// path mode is chosen as default
+			return PathScheme, nil // use default scheme for empty database
-			log.Info("State schema set to default", "scheme", "hash")
-			return HashScheme, nil
 		}
 		log.Info("State scheme set to already existing", "scheme", stored)
 		return stored, nil // reuse scheme of persistent scheme

core/rawdb/chain_freezer.go

@@ -17,9 +17,9 @@
 package rawdb
 
 import (
+	"errors"
 	"fmt"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -43,8 +43,6 @@ const (
 // The background thread will keep moving ancient chain segments from key-value
 // database to flat files for saving space on live database.
 type chainFreezer struct {
-	threshold atomic.Uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests)
-
 	*Freezer
 	quit    chan struct{}
 	wg      sync.WaitGroup
@@ -57,13 +55,11 @@ func newChainFreezer(datadir string, namespace string, readonly bool) (*chainFre
 	if err != nil {
 		return nil, err
 	}
-	cf := chainFreezer{
+	return &chainFreezer{
 		Freezer: freezer,
 		quit:    make(chan struct{}),
 		trigger: make(chan chan struct{}),
-	}
+	}, nil
-	cf.threshold.Store(params.FullImmutabilityThreshold)
-	return &cf, nil
 }
 
 // Close closes the chain freezer instance and terminates the background thread.
@@ -77,6 +73,57 @@ func (f *chainFreezer) Close() error {
 	return f.Freezer.Close()
 }
 
+// readHeadNumber returns the number of chain head block. 0 is returned if the
+// block is unknown or not available yet.
+func (f *chainFreezer) readHeadNumber(db ethdb.KeyValueReader) uint64 {
+	hash := ReadHeadBlockHash(db)
+	if hash == (common.Hash{}) {
+		log.Error("Head block is not reachable")
+		return 0
+	}
+	number := ReadHeaderNumber(db, hash)
+	if number == nil {
+		log.Error("Number of head block is missing")
+		return 0
+	}
+	return *number
+}
+
+// readFinalizedNumber returns the number of finalized block. 0 is returned
+// if the block is unknown or not available yet.
+func (f *chainFreezer) readFinalizedNumber(db ethdb.KeyValueReader) uint64 {
+	hash := ReadFinalizedBlockHash(db)
+	if hash == (common.Hash{}) {
+		return 0
+	}
+	number := ReadHeaderNumber(db, hash)
+	if number == nil {
+		log.Error("Number of finalized block is missing")
+		return 0
+	}
+	return *number
+}
+
+// freezeThreshold returns the threshold for chain freezing. It's determined
+// by formula: max(finality, HEAD-params.FullImmutabilityThreshold).
+func (f *chainFreezer) freezeThreshold(db ethdb.KeyValueReader) (uint64, error) {
+	var (
+		head      = f.readHeadNumber(db)
+		final     = f.readFinalizedNumber(db)
+		headLimit uint64
+	)
+	if head > params.FullImmutabilityThreshold {
+		headLimit = head - params.FullImmutabilityThreshold
+	}
+	if final == 0 && headLimit == 0 {
+		return 0, errors.New("freezing threshold is not available")
+	}
+	if final > headLimit {
+		return final, nil
+	}
+	return headLimit, nil
+}
+
 // freeze is a background thread that periodically checks the blockchain for any
 // import progress and moves ancient data from the fast database into the freezer.
 //
@@ -114,60 +161,39 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 				return
 			}
 		}
-		// Retrieve the freezing threshold.
+		threshold, err := f.freezeThreshold(nfdb)
-		hash := ReadHeadBlockHash(nfdb)
+		if err != nil {
-		if hash == (common.Hash{}) {
-			log.Debug("Current full block hash unavailable") // new chain, empty database
 			backoff = true
+			log.Debug("Current full block not old enough to freeze", "err", err)
 			continue
 		}
-		number := ReadHeaderNumber(nfdb, hash)
-		threshold := f.threshold.Load()
 		frozen := f.frozen.Load()
-		switch {
-		case number == nil:
-			log.Error("Current full block number unavailable", "hash", hash)
-			backoff = true
-			continue
 
-		case *number < threshold:
+		// Short circuit if the blocks below threshold are already frozen.
-			log.Debug("Current full block not old enough to freeze", "number", *number, "hash", hash, "delay", threshold)
+		if frozen != 0 && frozen-1 >= threshold {
-			backoff = true
-			continue
-
-		case *number-threshold <= frozen:
-			log.Debug("Ancient blocks frozen already", "number", *number, "hash", hash, "frozen", frozen)
 			backoff = true
+			log.Debug("Ancient blocks frozen already", "threshold", threshold, "frozen", frozen)
 			continue
 		}
-		head := ReadHeader(nfdb, hash, *number)
-		if head == nil {
-			log.Error("Current full block unavailable", "number", *number, "hash", hash)
-			backoff = true
-			continue
-		}
-
 		// Seems we have data ready to be frozen, process in usable batches
 		var (
-			start    = time.Now()
+			start = time.Now()
-			first, _ = f.Ancients()
+			first = frozen    // the first block to freeze
-			limit    = *number - threshold
+			last  = threshold // the last block to freeze
 		)
-		if limit-first > freezerBatchLimit {
+		if last-first+1 > freezerBatchLimit {
-			limit = first + freezerBatchLimit
+			last = freezerBatchLimit + first - 1
 		}
-		ancients, err := f.freezeRange(nfdb, first, limit)
+		ancients, err := f.freezeRange(nfdb, first, last)
 		if err != nil {
 			log.Error("Error in block freeze operation", "err", err)
 			backoff = true
 			continue
 		}
-
 		// Batch of blocks have been frozen, flush them before wiping from leveldb
 		if err := f.Sync(); err != nil {
 			log.Crit("Failed to flush frozen tables", "err", err)
 		}
-
 		// Wipe out all data from the active database
 		batch := db.NewBatch()
 		for i := 0; i < len(ancients); i++ {
@@ -250,8 +276,11 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 	}
 }
 
+// freezeRange moves a batch of chain segments from the fast database to the freezer.
+// The parameters (number, limit) specify the relevant block range, both of which
+// are included.
 func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes []common.Hash, err error) {
-	hashes = make([]common.Hash, 0, limit-number)
+	hashes = make([]common.Hash, 0, limit-number+1)
 
 	_, err = f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
 		for ; number <= limit; number++ {
@@ -293,11 +322,9 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
 			if err := op.AppendRaw(ChainFreezerDifficultyTable, number, td); err != nil {
 				return fmt.Errorf("can't write td to Freezer: %v", err)
 			}
-
 			hashes = append(hashes, hash)
 		}
 		return nil
 	})
-
 	return hashes, err
 }

core/rawdb/database.go

@@ -66,16 +66,10 @@ func (frdb *freezerdb) Close() error {
 // Freeze is a helper method used for external testing to trigger and block until
 // a freeze cycle completes, without having to sleep for a minute to trigger the
 // automatic background run.
-func (frdb *freezerdb) Freeze(threshold uint64) error {
+func (frdb *freezerdb) Freeze() error {
 	if frdb.AncientStore.(*chainFreezer).readonly {
 		return errReadOnly
 	}
-	// Set the freezer threshold to a temporary value
-	defer func(old uint64) {
-		frdb.AncientStore.(*chainFreezer).threshold.Store(old)
-	}(frdb.AncientStore.(*chainFreezer).threshold.Load())
-	frdb.AncientStore.(*chainFreezer).threshold.Store(threshold)
-
 	// Trigger a freeze cycle and block until it's done
 	trigger := make(chan struct{}, 1)
 	frdb.AncientStore.(*chainFreezer).trigger <- trigger

core/rawdb/schema.go

@@ -113,8 +113,8 @@ var (
 	skeletonHeaderPrefix  = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header
 
 	// Path-based storage scheme of merkle patricia trie.
-	trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node
+	TrieNodeAccountPrefix = []byte("A") // TrieNodeAccountPrefix + hexPath -> trie node
-	trieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node
+	TrieNodeStoragePrefix = []byte("O") // TrieNodeStoragePrefix + accountHash + hexPath -> trie node
 	stateIDPrefix         = []byte("L") // stateIDPrefix + state root -> state id
 
 	PreimagePrefix = []byte("secure-key-")       // PreimagePrefix + hash -> preimage
@@ -265,15 +265,15 @@ func stateIDKey(root common.Hash) []byte {
 	return append(stateIDPrefix, root.Bytes()...)
 }
 
-// accountTrieNodeKey = trieNodeAccountPrefix + nodePath.
+// accountTrieNodeKey = TrieNodeAccountPrefix + nodePath.
 func accountTrieNodeKey(path []byte) []byte {
-	return append(trieNodeAccountPrefix, path...)
+	return append(TrieNodeAccountPrefix, path...)
 }
 
-// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath.
+// storageTrieNodeKey = TrieNodeStoragePrefix + accountHash + nodePath.
 func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte {
-	buf := make([]byte, len(trieNodeStoragePrefix)+common.HashLength+len(path))
+	buf := make([]byte, len(TrieNodeStoragePrefix)+common.HashLength+len(path))
-	n := copy(buf, trieNodeStoragePrefix)
+	n := copy(buf, TrieNodeStoragePrefix)
 	n += copy(buf[n:], accountHash.Bytes())
 	copy(buf[n:], path)
 	return buf
@@ -294,16 +294,16 @@ func IsLegacyTrieNode(key []byte, val []byte) bool {
 // account trie node in path-based state scheme, and returns the resolved
 // node path if so.
 func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) {
-	if !bytes.HasPrefix(key, trieNodeAccountPrefix) {
+	if !bytes.HasPrefix(key, TrieNodeAccountPrefix) {
 		return false, nil
 	}
 	// The remaining key should only consist a hex node path
 	// whose length is in the range 0 to 64 (64 is excluded
 	// since leaves are always wrapped with shortNode).
-	if len(key) >= len(trieNodeAccountPrefix)+common.HashLength*2 {
+	if len(key) >= len(TrieNodeAccountPrefix)+common.HashLength*2 {
 		return false, nil
 	}
-	return true, key[len(trieNodeAccountPrefix):]
+	return true, key[len(TrieNodeAccountPrefix):]
 }
 
 // IsAccountTrieNode reports whether a provided database entry is an account
@@ -317,20 +317,20 @@ func IsAccountTrieNode(key []byte) bool {
 // trie node in path-based state scheme, and returns the resolved account hash
 // and node path if so.
 func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
-	if !bytes.HasPrefix(key, trieNodeStoragePrefix) {
+	if !bytes.HasPrefix(key, TrieNodeStoragePrefix) {
 		return false, common.Hash{}, nil
 	}
 	// The remaining key consists of 2 parts:
 	// - 32 bytes account hash
 	// - hex node path whose length is in the range 0 to 64
-	if len(key) < len(trieNodeStoragePrefix)+common.HashLength {
+	if len(key) < len(TrieNodeStoragePrefix)+common.HashLength {
 		return false, common.Hash{}, nil
 	}
-	if len(key) >= len(trieNodeStoragePrefix)+common.HashLength+common.HashLength*2 {
+	if len(key) >= len(TrieNodeStoragePrefix)+common.HashLength+common.HashLength*2 {
 		return false, common.Hash{}, nil
 	}
-	accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength])
+	accountHash := common.BytesToHash(key[len(TrieNodeStoragePrefix) : len(TrieNodeStoragePrefix)+common.HashLength])
-	return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:]
+	return true, accountHash, key[len(TrieNodeStoragePrefix)+common.HashLength:]
 }
 
 // IsStorageTrieNode reports whether a provided database entry is a storage

core/state/metrics.go

@@ -33,5 +33,4 @@ var (
 	slotDeletionTimer    = metrics.NewRegisteredResettingTimer("state/delete/storage/timer", nil)
 	slotDeletionCount    = metrics.NewRegisteredMeter("state/delete/storage/slot", nil)
 	slotDeletionSize     = metrics.NewRegisteredMeter("state/delete/storage/size", nil)
-	slotDeletionSkip     = metrics.NewRegisteredGauge("state/delete/storage/skip", nil)
 )

core/state/statedb.go

@@ -36,12 +36,6 @@ import (
 	"github.com/holiman/uint256"
 )
 
-const (
-	// storageDeleteLimit denotes the highest permissible memory allocation
-	// employed for contract storage deletion.
-	storageDeleteLimit = 512 * 1024 * 1024
-)
-
 type revision struct {
 	id           int
 	journalIndex int
@@ -949,10 +943,10 @@ func (s *StateDB) clearJournalAndRefund() {
 // of a specific account. It leverages the associated state snapshot for fast
 // storage iteration and constructs trie node deletion markers by creating
 // stack trie with iterated slots.
-func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) {
+func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) {
 	iter, err := s.snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{})
 	if err != nil {
-		return false, 0, nil, nil, err
+		return 0, nil, nil, err
 	}
 	defer iter.Release()
 
@@ -968,40 +962,37 @@ func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (boo
 	})
 	stack := trie.NewStackTrie(options)
 	for iter.Next() {
-		if size > storageDeleteLimit {
-			return true, size, nil, nil, nil
-		}
 		slot := common.CopyBytes(iter.Slot())
 		if err := iter.Error(); err != nil { // error might occur after Slot function
-			return false, 0, nil, nil, err
+			return 0, nil, nil, err
 		}
 		size += common.StorageSize(common.HashLength + len(slot))
 		slots[iter.Hash()] = slot
 
 		if err := stack.Update(iter.Hash().Bytes(), slot); err != nil {
-			return false, 0, nil, nil, err
+			return 0, nil, nil, err
 		}
 	}
 	if err := iter.Error(); err != nil { // error might occur during iteration
-		return false, 0, nil, nil, err
+		return 0, nil, nil, err
 	}
 	if stack.Hash() != root {
-		return false, 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash())
+		return 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash())
 	}
-	return false, size, slots, nodes, nil
+	return size, slots, nodes, nil
 }
 
 // slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage,"
 // employed when the associated state snapshot is not available. It iterates the
 // storage slots along with all internal trie nodes via trie directly.
-func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) {
+func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) {
 	tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie)
 	if err != nil {
-		return false, 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err)
+		return 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err)
 	}
 	it, err := tr.NodeIterator(nil)
 	if err != nil {
-		return false, 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err)
+		return 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err)
 	}
 	var (
 		size  common.StorageSize
@@ -1009,9 +1000,6 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r
 		slots = make(map[common.Hash][]byte)
 	)
 	for it.Next(true) {
-		if size > storageDeleteLimit {
-			return true, size, nil, nil, nil
-		}
 		if it.Leaf() {
 			slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob())
 			size += common.StorageSize(common.HashLength + len(it.LeafBlob()))
@@ -1024,9 +1012,9 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r
 		nodes.AddNode(it.Path(), trienode.NewDeleted())
 	}
 	if err := it.Error(); err != nil {
-		return false, 0, nil, nil, err
+		return 0, nil, nil, err
 	}
-	return false, size, slots, nodes, nil
+	return size, slots, nodes, nil
 }
 
 // deleteStorage is designed to delete the storage trie of a designated account.
@@ -1034,31 +1022,27 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r
 // potentially leading to an out-of-memory panic. The function will make an attempt
 // to utilize an efficient strategy if the associated state snapshot is reachable;
 // otherwise, it will resort to a less-efficient approach.
-func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, map[common.Hash][]byte, *trienode.NodeSet, error) {
+func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) {
 	var (
-		start   = time.Now()
+		start = time.Now()
-		err     error
+		err   error
-		aborted bool
+		size  common.StorageSize
-		size    common.StorageSize
+		slots map[common.Hash][]byte
-		slots   map[common.Hash][]byte
+		nodes *trienode.NodeSet
-		nodes   *trienode.NodeSet
 	)
 	// The fast approach can be failed if the snapshot is not fully
 	// generated, or it's internally corrupted. Fallback to the slow
 	// one just in case.
 	if s.snap != nil {
-		aborted, size, slots, nodes, err = s.fastDeleteStorage(addrHash, root)
+		size, slots, nodes, err = s.fastDeleteStorage(addrHash, root)
 	}
 	if s.snap == nil || err != nil {
-		aborted, size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root)
+		size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root)
 	}
 	if err != nil {
-		return false, nil, nil, err
+		return nil, nil, err
 	}
 	if metrics.EnabledExpensive {
-		if aborted {
-			slotDeletionSkip.Inc(1)
-		}
 		n := int64(len(slots))
 
 		slotDeletionMaxCount.UpdateIfGt(int64(len(slots)))
@@ -1068,7 +1052,7 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root
 		slotDeletionCount.Mark(n)
 		slotDeletionSize.Mark(int64(size))
 	}
-	return aborted, slots, nodes, nil
+	return slots, nodes, nil
 }
 
 // handleDestruction processes all destruction markers and deletes the account
@@ -1095,13 +1079,12 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root
 //
 // In case (d), **original** account along with its storages should be deleted,
 // with their values be tracked as original value.
-func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.Address]struct{}, error) {
+func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) error {
 	// Short circuit if geth is running with hash mode. This procedure can consume
 	// considerable time and storage deletion isn't supported in hash mode, thus
 	// preemptively avoiding unnecessary expenses.
-	incomplete := make(map[common.Address]struct{})
 	if s.db.TrieDB().Scheme() == rawdb.HashScheme {
-		return incomplete, nil
+		return nil
 	}
 	for addr, prev := range s.stateObjectsDestruct {
 		// The original account was non-existing, and it's marked as destructed
@@ -1124,18 +1107,9 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A
 			continue
 		}
 		// Remove storage slots belong to the account.
-		aborted, slots, set, err := s.deleteStorage(addr, addrHash, prev.Root)
+		slots, set, err := s.deleteStorage(addr, addrHash, prev.Root)
 		if err != nil {
-			return nil, fmt.Errorf("failed to delete storage, err: %w", err)
+			return fmt.Errorf("failed to delete storage, err: %w", err)
-		}
-		// The storage is too huge to handle, skip it but mark as incomplete.
-		// For case (d), the account is resurrected might with a few slots
-		// created. In this case, wipe the entire storage state diff because
-		// of aborted deletion.
-		if aborted {
-			incomplete[addr] = struct{}{}
-			delete(s.storagesOrigin, addr)
-			continue
 		}
 		if s.storagesOrigin[addr] == nil {
 			s.storagesOrigin[addr] = slots
@@ -1147,10 +1121,10 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A
 			}
 		}
 		if err := nodes.Merge(set); err != nil {
-			return nil, err
+			return err
 		}
 	}
-	return incomplete, nil
+	return nil
 }
 
 // Commit writes the state to the underlying in-memory trie database.
@@ -1179,8 +1153,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er
 		codeWriter              = s.db.DiskDB().NewBatch()
 	)
 	// Handle all state deletions first
-	incomplete, err := s.handleDestruction(nodes)
+	if err := s.handleDestruction(nodes); err != nil {
-	if err != nil {
 		return common.Hash{}, err
 	}
 	// Handle all state updates afterwards
@@ -1276,7 +1249,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er
 	}
 	if root != origin {
 		start := time.Now()
-		set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)
+		set := triestate.New(s.accountsOrigin, s.storagesOrigin)
 		if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil {
 			return common.Hash{}, err
 		}

core/state/statedb_test.go

@@ -1161,12 +1161,12 @@ func TestDeleteStorage(t *testing.T) {
 	obj := fastState.getOrNewStateObject(addr)
 	storageRoot := obj.data.Root
 
-	_, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
+	_, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	_, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
+	_, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
 	if err != nil {
 		t.Fatal(err)
 	}

core/state/sync.go

@@ -24,7 +24,7 @@ import (
 	"github.com/ethereum/go-ethereum/trie"
 )
 
-// NewStateSync create a new state trie download scheduler.
+// NewStateSync creates a new state trie download scheduler.
 func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync {
 	// Register the storage slot callback if the external callback is specified.
 	var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error

core/state_transition.go

@@ -67,7 +67,7 @@ func (result *ExecutionResult) Revert() []byte {
 }
 
 // IntrinsicGas computes the 'intrinsic gas' for a message with the given data.
-func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028 bool, isEIP3860 bool) (uint64, error) {
+func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028, isEIP3860 bool) (uint64, error) {
 	// Set the starting gas for the raw transaction
 	var gas uint64
 	if isContractCreation && isHomestead {
@@ -234,7 +234,7 @@ func (st *StateTransition) to() common.Address {
 
 func (st *StateTransition) buyGas() error {
 	mgval := new(big.Int).SetUint64(st.msg.GasLimit)
-	mgval = mgval.Mul(mgval, st.msg.GasPrice)
+	mgval.Mul(mgval, st.msg.GasPrice)
 	balanceCheck := new(big.Int).Set(mgval)
 	if st.msg.GasFeeCap != nil {
 		balanceCheck.SetUint64(st.msg.GasLimit)
@@ -263,7 +263,7 @@ func (st *StateTransition) buyGas() error {
 	if err := st.gp.SubGas(st.msg.GasLimit); err != nil {
 		return err
 	}
-	st.gasRemaining += st.msg.GasLimit
+	st.gasRemaining = st.msg.GasLimit
 
 	st.initialGas = st.msg.GasLimit
 	mgvalU256, _ := uint256.FromBig(mgval)
@@ -477,7 +477,7 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 {
 
 	// Return ETH for remaining gas, exchanged at the original rate.
 	remaining := uint256.NewInt(st.gasRemaining)
-	remaining = remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice))
+	remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice))
 	st.state.AddBalance(st.msg.From, remaining)
 
 	// Also return remaining gas to the block gas counter so it is

core/txpool/blobpool/blobpool.go

@@ -360,7 +360,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres
 		}
 	}
 	// Initialize the state with head block, or fallback to empty one in
-	// case the head state is not available(might occur when node is not
+	// case the head state is not available (might occur when node is not
 	// fully synced).
 	state, err := p.chain.StateAt(head.Root)
 	if err != nil {
@@ -371,7 +371,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres
 	}
 	p.head, p.state = head, state
 
-	// Index all transactions on disk and delete anything inprocessable
+	// Index all transactions on disk and delete anything unprocessable
 	var fails []uint64
 	index := func(id uint64, size uint32, blob []byte) {
 		if p.parseTransaction(id, size, blob) != nil {
@@ -402,7 +402,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres
 	}
 	var (
 		basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head))
-		blobfee = uint256.MustFromBig(big.NewInt(params.BlobTxMinBlobGasprice))
+		blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice)
 	)
 	if p.head.ExcessBlobGas != nil {
 		blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(*p.head.ExcessBlobGas))
@@ -540,7 +540,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
 		}
 		delete(p.index, addr)
 		delete(p.spent, addr)
-		if inclusions != nil { // only during reorgs will the heap will be initialized
+		if inclusions != nil { // only during reorgs will the heap be initialized
 			heap.Remove(p.evict, p.evict.index[addr])
 		}
 		p.reserve(addr, false)
@@ -693,7 +693,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
 		if len(txs) == 0 {
 			delete(p.index, addr)
 			delete(p.spent, addr)
-			if inclusions != nil { // only during reorgs will the heap will be initialized
+			if inclusions != nil { // only during reorgs will the heap be initialized
 				heap.Remove(p.evict, p.evict.index[addr])
 			}
 			p.reserve(addr, false)
@@ -809,7 +809,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) {
 				}
 			}
 			// Recheck the account's pooled transactions to drop included and
-			// invalidated one
+			// invalidated ones
 			p.recheck(addr, inclusions)
 		}
 		if len(adds) > 0 {
@@ -1226,7 +1226,7 @@ func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error
 // consensus validity and pool restrictions).
 func (p *BlobPool) add(tx *types.Transaction) (err error) {
 	// The blob pool blocks on adding a transaction. This is because blob txs are
-	// only even pulled form the network, so this method will act as the overload
+	// only even pulled from the network, so this method will act as the overload
 	// protection for fetches.
 	waitStart := time.Now()
 	p.lock.Lock()
@@ -1446,7 +1446,12 @@ func (p *BlobPool) drop() {
 //
 // The transactions can also be pre-filtered by the dynamic fee components to
 // reduce allocations and load on downstream subsystems.
-func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction {
+func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction {
+	// If only plain transactions are requested, this pool is unsuitable as it
+	// contains none, don't even bother.
+	if filter.OnlyPlainTxs {
+		return nil
+	}
 	// Track the amount of time waiting to retrieve the list of pending blob txs
 	// from the pool and the amount of time actually spent on assembling the data.
 	// The latter will be pretty much moot, but we've kept it to have symmetric
@@ -1456,29 +1461,30 @@ func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *u
 	pendwaitHist.Update(time.Since(pendStart).Nanoseconds())
 	defer p.lock.RUnlock()
 
-	defer func(start time.Time) {
+	execStart := time.Now()
-		pendtimeHist.Update(time.Since(start).Nanoseconds())
+	defer func() {
-	}(time.Now())
+		pendtimeHist.Update(time.Since(execStart).Nanoseconds())
+	}()
 
-	pending := make(map[common.Address][]*txpool.LazyTransaction)
+	pending := make(map[common.Address][]*txpool.LazyTransaction, len(p.index))
 	for addr, txs := range p.index {
-		var lazies []*txpool.LazyTransaction
+		lazies := make([]*txpool.LazyTransaction, 0, len(txs))
 		for _, tx := range txs {
 			// If transaction filtering was requested, discard badly priced ones
-			if minTip != nil && baseFee != nil {
+			if filter.MinTip != nil && filter.BaseFee != nil {
-				if tx.execFeeCap.Lt(baseFee) {
+				if tx.execFeeCap.Lt(filter.BaseFee) {
 					break // basefee too low, cannot be included, discard rest of txs from the account
 				}
-				tip := new(uint256.Int).Sub(tx.execFeeCap, baseFee)
+				tip := new(uint256.Int).Sub(tx.execFeeCap, filter.BaseFee)
 				if tip.Gt(tx.execTipCap) {
 					tip = tx.execTipCap
 				}
-				if tip.Lt(minTip) {
+				if tip.Lt(filter.MinTip) {
 					break // allowed or remaining tip too low, cannot be included, discard rest of txs from the account
 				}
 			}
-			if blobFee != nil {
+			if filter.BlobFee != nil {
-				if tx.blobFeeCap.Lt(blobFee) {
+				if tx.blobFeeCap.Lt(filter.BlobFee) {
 					break // blobfee too low, cannot be included, discard rest of txs from the account
 				}
 			}
@@ -1486,9 +1492,9 @@ func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *u
 			lazies = append(lazies, &txpool.LazyTransaction{
 				Pool:      p,
 				Hash:      tx.hash,
-				Time:      time.Now(), // TODO(karalabe): Maybe save these and use that?
+				Time:      execStart, // TODO(karalabe): Maybe save these and use that?
-				GasFeeCap: tx.execFeeCap.ToBig(),
+				GasFeeCap: tx.execFeeCap,
-				GasTipCap: tx.execTipCap.ToBig(),
+				GasTipCap: tx.execTipCap,
 				Gas:       tx.execGas,
 				BlobGas:   tx.blobGas,
 			})
@@ -1548,7 +1554,7 @@ func (p *BlobPool) updateStorageMetrics() {
 }
 
 // updateLimboMetrics retrieves a bunch of stats from the limbo store and pushes
-// // them out as metrics.
+// them out as metrics.
 func (p *BlobPool) updateLimboMetrics() {
 	stats := p.limbo.store.Infos()
 

core/txpool/blobpool/blobpool_test.go

@@ -185,7 +185,7 @@ func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64,
 	return types.MustSignNewTx(key, types.LatestSigner(testChainConfig), blobtx)
 }
 
-// makeUnsignedTx is a utility method to construct a random blob tranasaction
+// makeUnsignedTx is a utility method to construct a random blob transaction
 // without signing it.
 func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64) *types.BlobTx {
 	return &types.BlobTx{
@@ -391,7 +391,7 @@ func TestOpenDrops(t *testing.T) {
 		id, _ := store.Put(blob)
 		filled[id] = struct{}{}
 	}
-	// Insert a sequence of transactions with partially passed nonces to veirfy
+	// Insert a sequence of transactions with partially passed nonces to verify
 	// that the included part of the set will get dropped (case 4).
 	var (
 		overlapper, _ = crypto.GenerateKey()
@@ -1228,6 +1228,24 @@ func TestAdd(t *testing.T) {
 				},
 			},
 		},
+		// Blob transactions that don't meet the min blob gas price should be rejected
+		{
+			seeds: map[string]seed{
+				"alice": {balance: 10000000},
+			},
+			adds: []addtx{
+				{ // New account, no previous txs, nonce 0, but blob fee cap too low
+					from: "alice",
+					tx:   makeUnsignedTx(0, 1, 1, 0),
+					err:  txpool.ErrUnderpriced,
+				},
+				{ // Same as above but blob fee cap equals minimum, should be accepted
+					from: "alice",
+					tx:   makeUnsignedTx(0, 1, 1, params.BlobTxMinBlobGasprice),
+					err:  nil,
+				},
+			},
+		},
 	}
 	for i, tt := range tests {
 		// Create a temporary folder for the persistent backend
@@ -1288,3 +1306,65 @@ func TestAdd(t *testing.T) {
 		pool.Close()
 	}
 }
+
+// Benchmarks the time it takes to assemble the lazy pending transaction list
+// from the pool contents.
+func BenchmarkPoolPending100Mb(b *testing.B) { benchmarkPoolPending(b, 100_000_000) }
+func BenchmarkPoolPending1GB(b *testing.B)   { benchmarkPoolPending(b, 1_000_000_000) }
+func BenchmarkPoolPending10GB(b *testing.B)  { benchmarkPoolPending(b, 10_000_000_000) }
+
+func benchmarkPoolPending(b *testing.B, datacap uint64) {
+	// Calculate the maximum number of transaction that would fit into the pool
+	// and generate a set of random accounts to seed them with.
+	capacity := datacap / params.BlobTxBlobGasPerBlob
+
+	var (
+		basefee    = uint64(1050)
+		blobfee    = uint64(105)
+		signer     = types.LatestSigner(testChainConfig)
+		statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil)
+		chain      = &testBlockChain{
+			config:  testChainConfig,
+			basefee: uint256.NewInt(basefee),
+			blobfee: uint256.NewInt(blobfee),
+			statedb: statedb,
+		}
+		pool = New(Config{Datadir: ""}, chain)
+	)
+
+	if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil {
+		b.Fatalf("failed to create blob pool: %v", err)
+	}
+	// Fill the pool up with one random transaction from each account with the
+	// same price and everything to maximize the worst case scenario
+	for i := 0; i < int(capacity); i++ {
+		blobtx := makeUnsignedTx(0, 10, basefee+10, blobfee)
+		blobtx.R = uint256.NewInt(1)
+		blobtx.S = uint256.NewInt(uint64(100 + i))
+		blobtx.V = uint256.NewInt(0)
+		tx := types.NewTx(blobtx)
+		addr, err := types.Sender(signer, tx)
+		if err != nil {
+			b.Fatal(err)
+		}
+		statedb.AddBalance(addr, uint256.NewInt(1_000_000_000))
+		pool.add(tx)
+	}
+	statedb.Commit(0, true)
+	defer pool.Close()
+
+	// Benchmark assembling the pending
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	for i := 0; i < b.N; i++ {
+		p := pool.Pending(txpool.PendingFilter{
+			MinTip:  uint256.NewInt(1),
+			BaseFee: chain.basefee,
+			BlobFee: chain.blobfee,
+		})
+		if len(p) != int(capacity) {
+			b.Fatalf("have %d want %d", len(p), capacity)
+		}
+	}
+}

core/txpool/blobpool/config.go

@@ -30,8 +30,8 @@ type Config struct {
 // DefaultConfig contains the default configurations for the transaction pool.
 var DefaultConfig = Config{
 	Datadir:   "blobpool",
-	Datacap:   10 * 1024 * 1024 * 1024,
+	Datacap:   10 * 1024 * 1024 * 1024 / 4, // TODO(karalabe): /4 handicap for rollout, gradually bump back up to 10GB
-	PriceBump: 100, // either have patience or be aggressive, no mushy ground
+	PriceBump: 100,                         // either have patience or be aggressive, no mushy ground
 }
 
 // sanitize checks the provided user configurations and changes anything that's

core/txpool/blobpool/evictheap.go

@@ -30,7 +30,7 @@ import (
 // transaction from each account to determine which account to evict from.
 //
 // The heap internally tracks a slice of cheapest transactions from each account
-// and a mapping from addresses to indices for direct removals/udates.
+// and a mapping from addresses to indices for direct removals/updates.
 //
 // The goal of the heap is to decide which account has the worst bottleneck to
 // evict transactions from.

core/txpool/blobpool/priority_test.go

@@ -64,7 +64,7 @@ func BenchmarkDynamicFeeJumpCalculation(b *testing.B) {
 // Benchmarks how many priority recalculations can be done.
 func BenchmarkPriorityCalculation(b *testing.B) {
 	// The basefee and blob fee is constant for all transactions across a block,
-	// so we can assume theit absolute jump counts can be pre-computed.
+	// so we can assume their absolute jump counts can be pre-computed.
 	basefee := uint256.NewInt(17_200_000_000)  // 17.2 Gwei is the 22.03.2023 zero-emission basefee, random number
 	blobfee := uint256.NewInt(123_456_789_000) // Completely random, no idea what this will be
 

core/txpool/errors.go

@@ -54,4 +54,10 @@ var (
 	// ErrFutureReplacePending is returned if a future transaction replaces a pending
 	// one. Future transactions should only be able to replace other future transactions.
 	ErrFutureReplacePending = errors.New("future transaction tries to replace pending")
+
+	// ErrAlreadyReserved is returned if the sender address has a pending transaction
+	// in a different subpool. For example, this error is returned in response to any
+	// input transaction of non-blob type when a blob transaction from this sender
+	// remains pending (and vice-versa).
+	ErrAlreadyReserved = errors.New("address already reserved")
 )

core/txpool/legacypool/journal.go

@@ -164,7 +164,12 @@ func (journal *journal) rotate(all map[common.Address]types.Transactions) error
 		return err
 	}
 	journal.writer = sink
-	log.Info("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all))
+
+	logger := log.Info
+	if len(all) == 0 {
+		logger = log.Debug
+	}
+	logger("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all))
 
 	return nil
 }

core/txpool/legacypool/legacypool.go

@@ -296,7 +296,7 @@ func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserve txpool.A
 	pool.gasTip.Store(uint256.NewInt(gasTip))
 
 	// Initialize the state with head block, or fallback to empty one in
-	// case the head state is not available(might occur when node is not
+	// case the head state is not available (might occur when node is not
 	// fully synced).
 	statedb, err := pool.chain.StateAt(head.Root)
 	if err != nil {
@@ -522,7 +522,12 @@ func (pool *LegacyPool) ContentFrom(addr common.Address) ([]*types.Transaction,
 //
 // The transactions can also be pre-filtered by the dynamic fee components to
 // reduce allocations and load on downstream subsystems.
-func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction {
+func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction {
+	// If only blob transactions are requested, this pool is unsuitable as it
+	// contains none, don't even bother.
+	if filter.OnlyBlobTxs {
+		return nil
+	}
 	pool.mu.Lock()
 	defer pool.mu.Unlock()
 
@@ -531,13 +536,12 @@ func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobF
 		minTipBig  *big.Int
 		baseFeeBig *big.Int
 	)
-	if minTip != nil {
+	if filter.MinTip != nil {
-		minTipBig = minTip.ToBig()
+		minTipBig = filter.MinTip.ToBig()
 	}
-	if baseFee != nil {
+	if filter.BaseFee != nil {
-		baseFeeBig = baseFee.ToBig()
+		baseFeeBig = filter.BaseFee.ToBig()
 	}
-
 	pending := make(map[common.Address][]*txpool.LazyTransaction, len(pool.pending))
 	for addr, list := range pool.pending {
 		txs := list.Flatten()
@@ -559,8 +563,8 @@ func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobF
 					Hash:      txs[i].Hash(),
 					Tx:        txs[i],
 					Time:      txs[i].Time(),
-					GasFeeCap: txs[i].GasFeeCap(),
+					GasFeeCap: uint256.MustFromBig(txs[i].GasFeeCap()),
-					GasTipCap: txs[i].GasTipCap(),
+					GasTipCap: uint256.MustFromBig(txs[i].GasTipCap()),
 					Gas:       txs[i].Gas(),
 					BlobGas:   txs[i].BlobGas(),
 				}

core/txpool/legacypool/list.go

@@ -278,7 +278,7 @@ type list struct {
 	totalcost *uint256.Int // Total cost of all transactions in the list
 }
 
-// newList create a new transaction list for maintaining nonce-indexable fast,
+// newList creates a new transaction list for maintaining nonce-indexable fast,
 // gapped, sortable transaction lists.
 func newList(strict bool) *list {
 	return &list{

core/txpool/subpool.go

@@ -35,9 +35,9 @@ type LazyTransaction struct {
 	Hash common.Hash        // Transaction hash to pull up if needed
 	Tx   *types.Transaction // Transaction if already resolved
 
-	Time      time.Time // Time when the transaction was first seen
+	Time      time.Time    // Time when the transaction was first seen
-	GasFeeCap *big.Int  // Maximum fee per gas the transaction may consume
+	GasFeeCap *uint256.Int // Maximum fee per gas the transaction may consume
-	GasTipCap *big.Int  // Maximum miner tip per gas the transaction can pay
+	GasTipCap *uint256.Int // Maximum miner tip per gas the transaction can pay
 
 	Gas     uint64 // Amount of gas required by the transaction
 	BlobGas uint64 // Amount of blob gas required by the transaction
@@ -70,6 +70,21 @@ type LazyResolver interface {
 // may request (and relinquish) exclusive access to certain addresses.
 type AddressReserver func(addr common.Address, reserve bool) error
 
+// PendingFilter is a collection of filter rules to allow retrieving a subset
+// of transactions for announcement or mining.
+//
+// Note, the entries here are not arbitrary useful filters, rather each one has
+// a very specific call site in mind and each one can be evaluated very cheaply
+// by the pool implementations. Only add new ones that satisfy those constraints.
+type PendingFilter struct {
+	MinTip  *uint256.Int // Minimum miner tip required to include a transaction
+	BaseFee *uint256.Int // Minimum 1559 basefee needed to include a transaction
+	BlobFee *uint256.Int // Minimum 4844 blobfee needed to include a blob transaction
+
+	OnlyPlainTxs bool // Return only plain EVM transactions (peer-join announces, block space filling)
+	OnlyBlobTxs  bool // Return only blob transactions (block blob-space filling)
+}
+
 // SubPool represents a specialized transaction pool that lives on its own (e.g.
 // blob pool). Since independent of how many specialized pools we have, they do
 // need to be updated in lockstep and assemble into one coherent view for block
@@ -118,7 +133,7 @@ type SubPool interface {
 	//
 	// The transactions can also be pre-filtered by the dynamic fee components to
 	// reduce allocations and load on downstream subsystems.
-	Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*LazyTransaction
+	Pending(filter PendingFilter) map[common.Address][]*LazyTransaction
 
 	// SubscribeTransactions subscribes to new transaction events. The subscriber
 	// can decide whether to receive notifications only for newly seen transactions

core/txpool/txpool.go

@@ -28,7 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/metrics"
-	"github.com/holiman/uint256"
 )
 
 // TxStatus is the current status of a transaction as seen by the pool.
@@ -123,7 +122,7 @@ func (p *TxPool) reserver(id int, subpool SubPool) AddressReserver {
 					log.Error("pool attempted to reserve already-owned address", "address", addr)
 					return nil // Ignore fault to give the pool a chance to recover while the bug gets fixed
 				}
-				return errors.New("address already reserved")
+				return ErrAlreadyReserved
 			}
 			p.reservations[addr] = subpool
 			if metrics.Enabled {
@@ -357,10 +356,10 @@ func (p *TxPool) Add(txs []*types.Transaction, local bool, sync bool) []error {
 //
 // The transactions can also be pre-filtered by the dynamic fee components to
 // reduce allocations and load on downstream subsystems.
-func (p *TxPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*LazyTransaction {
+func (p *TxPool) Pending(filter PendingFilter) map[common.Address][]*LazyTransaction {
 	txs := make(map[common.Address][]*LazyTransaction)
 	for _, subpool := range p.subpools {
-		for addr, set := range subpool.Pending(minTip, baseFee, blobFee) {
+		for addr, set := range subpool.Pending(filter) {
 			txs[addr] = set
 		}
 	}

core/txpool/validation.go

@@ -18,6 +18,7 @@ package txpool
 
 import (
 	"crypto/sha256"
+	"errors"
 	"fmt"
 	"math/big"
 
@@ -30,6 +31,12 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 )
 
+var (
+	// blobTxMinBlobGasPrice is the big.Int version of the configured protocol
+	// parameter to avoid constucting a new big integer for every transaction.
+	blobTxMinBlobGasPrice = big.NewInt(params.BlobTxMinBlobGasprice)
+)
+
 // ValidationOptions define certain differences between transaction validation
 // across the different pools without having to duplicate those checks.
 type ValidationOptions struct {
@@ -101,28 +108,31 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
 		return err
 	}
 	if tx.Gas() < intrGas {
-		return fmt.Errorf("%w: needed %v, allowed %v", core.ErrIntrinsicGas, intrGas, tx.Gas())
+		return fmt.Errorf("%w: gas %v, minimum needed %v", core.ErrIntrinsicGas, tx.Gas(), intrGas)
 	}
-	// Ensure the gasprice is high enough to cover the requirement of the calling
+	// Ensure the gasprice is high enough to cover the requirement of the calling pool
-	// pool and/or block producer
 	if tx.GasTipCapIntCmp(opts.MinTip) < 0 {
-		return fmt.Errorf("%w: tip needed %v, tip permitted %v", ErrUnderpriced, opts.MinTip, tx.GasTipCap())
+		return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrUnderpriced, tx.GasTipCap(), opts.MinTip)
 	}
-	// Ensure blob transactions have valid commitments
 	if tx.Type() == types.BlobTxType {
+		// Ensure the blob fee cap satisfies the minimum blob gas price
+		if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
+			return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrUnderpriced, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
+		}
 		sidecar := tx.BlobTxSidecar()
 		if sidecar == nil {
-			return fmt.Errorf("missing sidecar in blob transaction")
+			return errors.New("missing sidecar in blob transaction")
 		}
 		// Ensure the number of items in the blob transaction and various side
 		// data match up before doing any expensive validations
 		hashes := tx.BlobHashes()
 		if len(hashes) == 0 {
-			return fmt.Errorf("blobless blob transaction")
+			return errors.New("blobless blob transaction")
 		}
 		if len(hashes) > params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob {
 			return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob)
 		}
+		// Ensure commitments, proofs and hashes are valid
 		if err := validateBlobSidecar(hashes, sidecar); err != nil {
 			return err
 		}

core/types/account.go

@@ -52,7 +52,7 @@ type accountMarshaling struct {
 }
 
 // storageJSON represents a 256 bit byte array, but allows less than 256 bits when
-// unmarshaling from hex.
+// unmarshalling from hex.
 type storageJSON common.Hash
 
 func (h *storageJSON) UnmarshalText(text []byte) error {

core/types/block_test.go

@@ -196,7 +196,7 @@ func TestEIP2718BlockEncoding(t *testing.T) {
 func TestUncleHash(t *testing.T) {
 	uncles := make([]*Header, 0)
 	h := CalcUncleHash(uncles)
-	exp := common.HexToHash("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347")
+	exp := EmptyUncleHash
 	if h != exp {
 		t.Fatalf("empty uncle hash is wrong, got %x != %x", h, exp)
 	}

core/types/receipt_test.go

@@ -343,7 +343,7 @@ func TestReceiptJSON(t *testing.T) {
 		r := Receipt{}
 		err = r.UnmarshalJSON(b)
 		if err != nil {
-			t.Fatal("error unmarshaling receipt from json:", err)
+			t.Fatal("error unmarshalling receipt from json:", err)
 		}
 	}
 }
@@ -360,7 +360,7 @@ func TestEffectiveGasPriceNotRequired(t *testing.T) {
 	r2 := Receipt{}
 	err = r2.UnmarshalJSON(b)
 	if err != nil {
-		t.Fatal("error unmarshaling receipt from json:", err)
+		t.Fatal("error unmarshalling receipt from json:", err)
 	}
 }
 

core/types/transaction_signing.go

@@ -107,13 +107,7 @@ func SignTx(tx *Transaction, s Signer, prv *ecdsa.PrivateKey) (*Transaction, err
 
 // SignNewTx creates a transaction and signs it.
 func SignNewTx(prv *ecdsa.PrivateKey, s Signer, txdata TxData) (*Transaction, error) {
-	tx := NewTx(txdata)
+	return SignTx(NewTx(txdata), s, prv)
-	h := s.Hash(tx)
-	sig, err := crypto.Sign(h[:], prv)
-	if err != nil {
-		return nil, err
-	}
-	return tx.WithSignature(s, sig)
 }
 
 // MustSignNewTx creates a transaction and signs it.

core/vm/errors.go

@@ -29,7 +29,6 @@ var (
 	ErrInsufficientBalance      = errors.New("insufficient balance for transfer")
 	ErrContractAddressCollision = errors.New("contract address collision")
 	ErrExecutionReverted        = errors.New("execution reverted")
-	ErrMaxInitCodeSizeExceeded  = errors.New("max initcode size exceeded")
 	ErrMaxCodeSizeExceeded      = errors.New("max code size exceeded")
 	ErrInvalidJump              = errors.New("invalid jump destination")
 	ErrWriteProtection          = errors.New("write protection")

core/vm/instructions.go

@@ -17,6 +17,8 @@
 package vm
 
 import (
+	"math"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
@@ -303,7 +305,7 @@ func opCallDataCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext
 	)
 	dataOffset64, overflow := dataOffset.Uint64WithOverflow()
 	if overflow {
-		dataOffset64 = 0xffffffffffffffff
+		dataOffset64 = math.MaxUint64
 	}
 	// These values are checked for overflow during gas cost calculation
 	memOffset64 := memOffset.Uint64()
@@ -359,7 +361,7 @@ func opCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([
 	)
 	uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow()
 	if overflow {
-		uint64CodeOffset = 0xffffffffffffffff
+		uint64CodeOffset = math.MaxUint64
 	}
 	codeCopy := getData(scope.Contract.Code, uint64CodeOffset, length.Uint64())
 	scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy)
@@ -377,7 +379,7 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext)
 	)
 	uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow()
 	if overflow {
-		uint64CodeOffset = 0xffffffffffffffff
+		uint64CodeOffset = math.MaxUint64
 	}
 	addr := common.Address(a.Bytes20())
 	codeCopy := getData(interpreter.evm.StateDB.GetCode(addr), uint64CodeOffset, length.Uint64())

core/vm/operations_acl.go

@@ -187,7 +187,12 @@ func makeCallVariantGasCallEIP2929(oldCalculator gasFunc) gasFunc {
 		// outside of this function, as part of the dynamic gas, and that will make it
 		// also become correctly reported to tracers.
 		contract.Gas += coldCost
-		return gas + coldCost, nil
+
+		var overflow bool
+		if gas, overflow = math.SafeAdd(gas, coldCost); overflow {
+			return 0, ErrGasUintOverflow
+		}
+		return gas, nil
 	}
 }
 

crypto/kzg4844/kzg4844.go

@@ -85,7 +85,7 @@ type Claim [32]byte
 var useCKZG atomic.Bool
 
 // UseCKZG can be called to switch the default Go implementation of KZG to the C
-// library if fo some reason the user wishes to do so (e.g. consensus bug in one
+// library if for some reason the user wishes to do so (e.g. consensus bug in one
 // or the other).
 func UseCKZG(use bool) error {
 	if use && !ckzgAvailable {

crypto/secp256k1/libsecp256k1/include/secp256k1.h

@@ -357,7 +357,7 @@ SECP256K1_API int secp256k1_ecdsa_signature_serialize_compact(
 /** Verify an ECDSA signature.
  *
  *  Returns: 1: correct signature
- *           0: incorrect or unparseable signature
+ *           0: incorrect or unparsable signature
  *  Args:    ctx:       a secp256k1 context object, initialized for verification.
  *  In:      sig:       the signature being verified (cannot be NULL)
  *           msg32:     the 32-byte message hash being verified (cannot be NULL)

crypto/secp256k1/libsecp256k1/sage/group_prover.sage

@@ -17,7 +17,7 @@
 #   - A constraint describing the requirements of the law, called "require"
 # * Implementations are transliterated into functions that operate as well on
 #   algebraic input points, and are called once per combination of branches
-#   exectured. Each execution returns:
+#   executed. Each execution returns:
 #   - A constraint describing the assumptions this implementation requires
 #     (such as Z1=1), called "assumeFormula"
 #   - A constraint describing the assumptions this specific branch requires,

eth/api.go

@@ -1,52 +0,0 @@
-// Copyright 2015 The go-ethereum Authors
-// This file is part of the go-ethereum library.
-//
-// The go-ethereum library is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Lesser General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// The go-ethereum library is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Lesser General Public License for more details.
-//
-// You should have received a copy of the GNU Lesser General Public License
-// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
-
-package eth
-
-import (
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
-)
-
-// EthereumAPI provides an API to access Ethereum full node-related information.
-type EthereumAPI struct {
-	e *Ethereum
-}
-
-// NewEthereumAPI creates a new Ethereum protocol API for full nodes.
-func NewEthereumAPI(e *Ethereum) *EthereumAPI {
-	return &EthereumAPI{e}
-}
-
-// Etherbase is the address that mining rewards will be sent to.
-func (api *EthereumAPI) Etherbase() (common.Address, error) {
-	return api.e.Etherbase()
-}
-
-// Coinbase is the address that mining rewards will be sent to (alias for Etherbase).
-func (api *EthereumAPI) Coinbase() (common.Address, error) {
-	return api.Etherbase()
-}
-
-// Hashrate returns the POW hashrate.
-func (api *EthereumAPI) Hashrate() hexutil.Uint64 {
-	return hexutil.Uint64(api.e.Miner().Hashrate())
-}
-
-// Mining returns an indication if this node is currently mining.
-func (api *EthereumAPI) Mining() bool {
-	return api.e.IsMining()
-}

eth/api_backend.go

@@ -37,7 +37,6 @@ import (
 	"github.com/ethereum/go-ethereum/eth/tracers"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/event"
-	"github.com/ethereum/go-ethereum/miner"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rpc"
 )
@@ -67,7 +66,7 @@ func (b *EthAPIBackend) SetHead(number uint64) {
 func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) {
 	// Pending block is only known by the miner
 	if number == rpc.PendingBlockNumber {
-		block := b.eth.miner.PendingBlock()
+		block, _, _ := b.eth.miner.Pending()
 		if block == nil {
 			return nil, errors.New("pending block is not available")
 		}
@@ -118,7 +117,7 @@ func (b *EthAPIBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*ty
 func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) {
 	// Pending block is only known by the miner
 	if number == rpc.PendingBlockNumber {
-		block := b.eth.miner.PendingBlock()
+		block, _, _ := b.eth.miner.Pending()
 		if block == nil {
 			return nil, errors.New("pending block is not available")
 		}
@@ -182,14 +181,14 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r
 	return nil, errors.New("invalid arguments; neither block nor hash specified")
 }
 
-func (b *EthAPIBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) {
+func (b *EthAPIBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) {
-	return b.eth.miner.PendingBlockAndReceipts()
+	return b.eth.miner.Pending()
 }
 
 func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) {
 	// Pending state is only known by the miner
 	if number == rpc.PendingBlockNumber {
-		block, state := b.eth.miner.Pending()
+		block, _, state := b.eth.miner.Pending()
 		if block == nil || state == nil {
 			return nil, nil, errors.New("pending state is not available")
 		}
@@ -267,10 +266,6 @@ func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEven
 	return b.eth.BlockChain().SubscribeRemovedLogsEvent(ch)
 }
 
-func (b *EthAPIBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription {
-	return b.eth.miner.SubscribePendingLogs(ch)
-}
-
 func (b *EthAPIBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
 	return b.eth.BlockChain().SubscribeChainEvent(ch)
 }
@@ -292,7 +287,7 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction)
 }
 
 func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) {
-	pending := b.eth.txPool.Pending(nil, nil, nil)
+	pending := b.eth.txPool.Pending(txpool.PendingFilter{})
 	var txs types.Transactions
 	for _, batch := range pending {
 		for _, lazy := range batch {
@@ -421,14 +416,6 @@ func (b *EthAPIBackend) CurrentHeader() *types.Header {
 	return b.eth.blockchain.CurrentHeader()
 }
 
-func (b *EthAPIBackend) Miner() *miner.Miner {
-	return b.eth.Miner()
-}
-
-func (b *EthAPIBackend) StartMining() error {
-	return b.eth.StartMining()
-}
-
 func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) {
 	return b.eth.stateAtBlock(ctx, block, reexec, base, readOnly, preferDisk)
 }

eth/api_debug.go

@@ -56,7 +56,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) {
 		// If we're dumping the pending state, we need to request
 		// both the pending block as well as the pending state from
 		// the miner and operate on those
-		_, stateDb := api.eth.miner.Pending()
+		_, _, stateDb := api.eth.miner.Pending()
 		if stateDb == nil {
 			return state.Dump{}, errors.New("pending state is not available")
 		}
@@ -142,7 +142,7 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex
 			// If we're dumping the pending state, we need to request
 			// both the pending block as well as the pending state from
 			// the miner and operate on those
-			_, stateDb = api.eth.miner.Pending()
+			_, _, stateDb = api.eth.miner.Pending()
 			if stateDb == nil {
 				return state.Dump{}, errors.New("pending state is not available")
 			}

eth/api_miner.go

@@ -18,9 +18,7 @@ package eth
 
 import (
 	"math/big"
-	"time"
 
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 )
 
@@ -29,26 +27,11 @@ type MinerAPI struct {
 	e *Ethereum
 }
 
-// NewMinerAPI create a new MinerAPI instance.
+// NewMinerAPI creates a new MinerAPI instance.
 func NewMinerAPI(e *Ethereum) *MinerAPI {
 	return &MinerAPI{e}
 }
 
-// Start starts the miner with the given number of threads. If threads is nil,
-// the number of workers started is equal to the number of logical CPUs that are
-// usable by this process. If mining is already running, this method adjust the
-// number of threads allowed to use and updates the minimum price required by the
-// transaction pool.
-func (api *MinerAPI) Start() error {
-	return api.e.StartMining()
-}
-
-// Stop terminates the miner, both at the consensus engine level as well as at
-// the block creation level.
-func (api *MinerAPI) Stop() {
-	api.e.StopMining()
-}
-
 // SetExtra sets the extra data string that is included when this miner mines a block.
 func (api *MinerAPI) SetExtra(extra string) (bool, error) {
 	if err := api.e.Miner().SetExtra([]byte(extra)); err != nil {
@@ -73,14 +56,3 @@ func (api *MinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool {
 	api.e.Miner().SetGasCeil(uint64(gasLimit))
 	return true
 }
-
-// SetEtherbase sets the etherbase of the miner.
-func (api *MinerAPI) SetEtherbase(etherbase common.Address) bool {
-	api.e.SetEtherbase(etherbase)
-	return true
-}
-
-// SetRecommitInterval updates the interval for miner sealing work recommitting.
-func (api *MinerAPI) SetRecommitInterval(interval int) {
-	api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond)
-}

eth/backend.go

@@ -28,8 +28,6 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
-	"github.com/ethereum/go-ethereum/consensus/beacon"
-	"github.com/ethereum/go-ethereum/consensus/clique"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/bloombits"
 	"github.com/ethereum/go-ethereum/core/rawdb"
@@ -74,7 +72,6 @@ type Ethereum struct {
 	handler            *handler
 	ethDialCandidates  enode.Iterator
 	snapDialCandidates enode.Iterator
-	merger             *consensus.Merger
 
 	// DB interfaces
 	chainDb ethdb.Database // Block chain database
@@ -89,9 +86,8 @@ type Ethereum struct {
 
 	APIBackend *EthAPIBackend
 
-	miner     *miner.Miner
+	miner    *miner.Miner
-	gasPrice  *big.Int
+	gasPrice *big.Int
-	etherbase common.Address
 
 	networkID     uint64
 	netRPCService *ethapi.NetAPI
@@ -158,7 +154,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	}
 	eth := &Ethereum{
 		config:            config,
-		merger:            consensus.NewMerger(chainDb),
 		chainDb:           chainDb,
 		eventMux:          stack.EventMux(),
 		accountManager:    stack.AccountManager(),
@@ -166,7 +161,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		closeBloomHandler: make(chan struct{}),
 		networkID:         networkID,
 		gasPrice:          config.Miner.GasPrice,
-		etherbase:         config.Miner.Etherbase,
 		bloomRequests:     make(chan chan *bloombits.Retrieval),
 		bloomIndexer:      core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms),
 		p2pServer:         stack.Server(),
@@ -213,7 +207,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	if config.OverrideVerkle != nil {
 		overrides.OverrideVerkle = config.OverrideVerkle
 	}
-	eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TransactionHistory)
+	// TODO (MariusVanDerWijden) get rid of shouldPreserve in a follow-up PR
+	shouldPreserve := func(header *types.Header) bool {
+		return false
+	}
+	eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, shouldPreserve, &config.TransactionHistory)
 	if err != nil {
 		return nil, err
 	}
@@ -236,10 +234,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	// Permit the downloader to use the trie cache allowance during fast sync
 	cacheLimit := cacheConfig.TrieCleanLimit + cacheConfig.TrieDirtyLimit + cacheConfig.SnapshotLimit
 	if eth.handler, err = newHandler(&handlerConfig{
+		NodeID:         eth.p2pServer.Self().ID(),
 		Database:       chainDb,
 		Chain:          eth.blockchain,
 		TxPool:         eth.txPool,
-		Merger:         eth.merger,
 		Network:        networkID,
 		Sync:           config.SyncMode,
 		BloomCache:     uint64(cacheLimit),
@@ -249,7 +247,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		return nil, err
 	}
 
-	eth.miner = miner.New(eth, &config.Miner, eth.blockchain.Config(), eth.EventMux(), eth.engine, eth.isLocalBlock)
+	eth.miner = miner.New(eth, config.Miner, eth.engine)
 	eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData))
 
 	eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil}
@@ -315,9 +313,6 @@ func (s *Ethereum) APIs() []rpc.API {
 	// Append all the local APIs and return
 	return append(apis, []rpc.API{
 		{
-			Namespace: "eth",
-			Service:   NewEthereumAPI(s),
-		}, {
 			Namespace: "miner",
 			Service:   NewMinerAPI(s),
 		}, {
@@ -340,138 +335,6 @@ func (s *Ethereum) ResetWithGenesisBlock(gb *types.Block) {
 	s.blockchain.ResetWithGenesisBlock(gb)
 }
 
-func (s *Ethereum) Etherbase() (eb common.Address, err error) {
-	s.lock.RLock()
-	etherbase := s.etherbase
-	s.lock.RUnlock()
-
-	if etherbase != (common.Address{}) {
-		return etherbase, nil
-	}
-	return common.Address{}, errors.New("etherbase must be explicitly specified")
-}
-
-// isLocalBlock checks whether the specified block is mined
-// by local miner accounts.
-//
-// We regard two types of accounts as local miner account: etherbase
-// and accounts specified via `txpool.locals` flag.
-func (s *Ethereum) isLocalBlock(header *types.Header) bool {
-	author, err := s.engine.Author(header)
-	if err != nil {
-		log.Warn("Failed to retrieve block author", "number", header.Number.Uint64(), "hash", header.Hash(), "err", err)
-		return false
-	}
-	// Check whether the given address is etherbase.
-	s.lock.RLock()
-	etherbase := s.etherbase
-	s.lock.RUnlock()
-	if author == etherbase {
-		return true
-	}
-	// Check whether the given address is specified by `txpool.local`
-	// CLI flag.
-	for _, account := range s.config.TxPool.Locals {
-		if account == author {
-			return true
-		}
-	}
-	return false
-}
-
-// shouldPreserve checks whether we should preserve the given block
-// during the chain reorg depending on whether the author of block
-// is a local account.
-func (s *Ethereum) shouldPreserve(header *types.Header) bool {
-	// The reason we need to disable the self-reorg preserving for clique
-	// is it can be probable to introduce a deadlock.
-	//
-	// e.g. If there are 7 available signers
-	//
-	// r1   A
-	// r2     B
-	// r3       C
-	// r4         D
-	// r5   A      [X] F G
-	// r6    [X]
-	//
-	// In the round5, the in-turn signer E is offline, so the worst case
-	// is A, F and G sign the block of round5 and reject the block of opponents
-	// and in the round6, the last available signer B is offline, the whole
-	// network is stuck.
-	if _, ok := s.engine.(*clique.Clique); ok {
-		return false
-	}
-	return s.isLocalBlock(header)
-}
-
-// SetEtherbase sets the mining reward address.
-func (s *Ethereum) SetEtherbase(etherbase common.Address) {
-	s.lock.Lock()
-	s.etherbase = etherbase
-	s.lock.Unlock()
-
-	s.miner.SetEtherbase(etherbase)
-}
-
-// StartMining starts the miner with the given number of CPU threads. If mining
-// is already running, this method adjust the number of threads allowed to use
-// and updates the minimum price required by the transaction pool.
-func (s *Ethereum) StartMining() error {
-	// If the miner was not running, initialize it
-	if !s.IsMining() {
-		// Propagate the initial price point to the transaction pool
-		s.lock.RLock()
-		price := s.gasPrice
-		s.lock.RUnlock()
-		s.txPool.SetGasTip(price)
-
-		// Configure the local mining address
-		eb, err := s.Etherbase()
-		if err != nil {
-			log.Error("Cannot start mining without etherbase", "err", err)
-			return fmt.Errorf("etherbase missing: %v", err)
-		}
-		var cli *clique.Clique
-		if c, ok := s.engine.(*clique.Clique); ok {
-			cli = c
-		} else if cl, ok := s.engine.(*beacon.Beacon); ok {
-			if c, ok := cl.InnerEngine().(*clique.Clique); ok {
-				cli = c
-			}
-		}
-		if cli != nil {
-			wallet, err := s.accountManager.Find(accounts.Account{Address: eb})
-			if wallet == nil || err != nil {
-				log.Error("Etherbase account unavailable locally", "err", err)
-				return fmt.Errorf("signer missing: %v", err)
-			}
-			cli.Authorize(eb, wallet.SignData)
-		}
-		// If mining is started, we can disable the transaction rejection mechanism
-		// introduced to speed sync times.
-		s.handler.enableSyncedFeatures()
-
-		go s.miner.Start()
-	}
-	return nil
-}
-
-// StopMining terminates the miner, both at the consensus engine level as well as
-// at the block creation level.
-func (s *Ethereum) StopMining() {
-	// Update the thread count within the consensus engine
-	type threaded interface {
-		SetThreads(threads int)
-	}
-	if th, ok := s.engine.(threaded); ok {
-		th.SetThreads(-1)
-	}
-	// Stop the block creating itself
-	s.miner.Stop()
-}
-
-func (s *Ethereum) IsMining() bool      { return s.miner.Mining() }
 func (s *Ethereum) Miner() *miner.Miner { return s.miner }
 
 func (s *Ethereum) AccountManager() *accounts.Manager  { return s.accountManager }
@@ -486,11 +349,6 @@ func (s *Ethereum) Synced() bool                       { return s.handler.synced
 func (s *Ethereum) SetSynced()                         { s.handler.enableSyncedFeatures() }
 func (s *Ethereum) ArchiveMode() bool                  { return s.config.NoPruning }
 func (s *Ethereum) BloomIndexer() *core.ChainIndexer   { return s.bloomIndexer }
-func (s *Ethereum) Merger() *consensus.Merger          { return s.merger }
-func (s *Ethereum) SyncMode() downloader.SyncMode {
-	mode, _ := s.handler.chainSync.modeAndLocalHead()
-	return mode
-}
 
 // Protocols returns all the currently configured
 // network protocols to start.
@@ -538,7 +396,6 @@ func (s *Ethereum) Stop() error {
 	s.bloomIndexer.Close()
 	close(s.closeBloomHandler)
 	s.txPool.Close()
-	s.miner.Close()
 	s.blockchain.Stop()
 	s.engine.Close()
 
@@ -550,3 +407,29 @@ func (s *Ethereum) Stop() error {
 
 	return nil
 }
+
+// SyncMode retrieves the current sync mode, either explicitly set, or derived
+// from the chain status.
+func (s *Ethereum) SyncMode() downloader.SyncMode {
+	// If we're in snap sync mode, return that directly
+	if s.handler.snapSync.Load() {
+		return downloader.SnapSync
+	}
+	// We are probably in full sync, but we might have rewound to before the
+	// snap sync pivot, check if we should re-enable snap sync.
+	head := s.blockchain.CurrentBlock()
+	if pivot := rawdb.ReadLastPivotNumber(s.chainDb); pivot != nil {
+		if head.Number.Uint64() < *pivot {
+			return downloader.SnapSync
+		}
+	}
+	// We are in a full sync, but the associated head state is missing. To complete
+	// the head state, forcefully rerun the snap sync. Note it doesn't mean the
+	// persistent state is corrupted, just mismatch with the head block.
+	if !s.blockchain.HasState(head.Root) {
+		log.Info("Reenabled snap sync as chain is stateless")
+		return downloader.SnapSync
+	}
+	// Nope, we're really full syncing
+	return downloader.FullSync
+}

eth/catalyst/api.go

@@ -190,21 +190,21 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, pa
 // attributes. It supports both PayloadAttributesV1 and PayloadAttributesV2.
 func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) {
 	if params != nil {
+		if params.BeaconRoot != nil {
+			return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("unexpected beacon root"))
+		}
 		switch api.eth.BlockChain().Config().LatestFork(params.Timestamp) {
 		case forks.Paris:
 			if params.Withdrawals != nil {
-				return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals before shanghai"))
+				return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("withdrawals before shanghai"))
 			}
 		case forks.Shanghai:
 			if params.Withdrawals == nil {
-				return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals"))
+				return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing withdrawals"))
 			}
 		default:
 			return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV2 must only be called with paris and shanghai payloads"))
 		}
-		if params.BeaconRoot != nil {
-			return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("unexpected beacon root"))
-		}
 	}
 	return api.forkchoiceUpdated(update, params, engine.PayloadV2, false)
 }
@@ -213,15 +213,11 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, pa
 // in the payload attributes. It supports only PayloadAttributesV3.
 func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) {
 	if params != nil {
-		// TODO(matt): according to https://github.com/ethereum/execution-apis/pull/498,
-		// payload attributes that are invalid should return error
-		// engine.InvalidPayloadAttributes. Once hive updates this, we should update
-		// on our end.
 		if params.Withdrawals == nil {
-			return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals"))
+			return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing withdrawals"))
 		}
 		if params.BeaconRoot == nil {
-			return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing beacon root"))
+			return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing beacon root"))
 		}
 		if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun {
 			return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV3 must only be called for cancun payloads"))
@@ -271,12 +267,6 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl
 		finalized := api.remoteBlocks.get(update.FinalizedBlockHash)
 
 		// Header advertised via a past newPayload request. Start syncing to it.
-		// Before we do however, make sure any legacy sync in switched off so we
-		// don't accidentally have 2 cycles running.
-		if merger := api.eth.Merger(); !merger.TDDReached() {
-			merger.ReachTTD()
-			api.eth.Downloader().Cancel()
-		}
 		context := []interface{}{"number", header.Number, "hash", header.Hash()}
 		if update.FinalizedBlockHash != (common.Hash{}) {
 			if finalized == nil {
@@ -338,9 +328,6 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl
 	// If the beacon client also advertised a finalized block, mark the local
 	// chain final and completely in PoS mode.
 	if update.FinalizedBlockHash != (common.Hash{}) {
-		if merger := api.eth.Merger(); !merger.PoSFinalized() {
-			merger.FinalizePoS()
-		}
 		// If the finalized block is not in our canonical tree, something is wrong
 		finalBlock := api.eth.BlockChain().GetBlockByHash(update.FinalizedBlockHash)
 		if finalBlock == nil {
@@ -488,7 +475,7 @@ func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.Payl
 // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
 func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) {
 	if api.eth.BlockChain().Config().IsCancun(api.eth.BlockChain().Config().LondonBlock, params.Timestamp) {
-		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use new payload v2 post-shanghai"))
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use newPayloadV2 post-cancun"))
 	}
 	if api.eth.BlockChain().Config().LatestFork(params.Timestamp) == forks.Shanghai {
 		if params.Withdrawals == nil {
@@ -503,7 +490,7 @@ func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.Payl
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil excessBlobGas pre-cancun"))
 	}
 	if params.BlobGasUsed != nil {
-		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil params.BlobGasUsed pre-cancun"))
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil blobGasUsed pre-cancun"))
 	}
 	return api.newPayload(params, nil, nil)
 }
@@ -517,14 +504,14 @@ func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHas
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil excessBlobGas post-cancun"))
 	}
 	if params.BlobGasUsed == nil {
-		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil params.BlobGasUsed post-cancun"))
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil blobGasUsed post-cancun"))
 	}
 
 	if versionedHashes == nil {
 		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil versionedHashes post-cancun"))
 	}
 	if beaconRoot == nil {
-		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil parentBeaconBlockRoot post-cancun"))
+		return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil beaconRoot post-cancun"))
 	}
 
 	if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun {
@@ -624,13 +611,6 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe
 
 		return api.invalid(err, parent.Header()), nil
 	}
-	// We've accepted a valid payload from the beacon client. Mark the local
-	// chain transitions to notify other subsystems (e.g. downloader) of the
-	// behavioral change.
-	if merger := api.eth.Merger(); !merger.TDDReached() {
-		merger.ReachTTD()
-		api.eth.Downloader().Cancel()
-	}
 	hash := block.Hash()
 	return engine.PayloadStatusV1{Status: engine.VALID, LatestValidHash: &hash}, nil
 }
@@ -788,26 +768,23 @@ func (api *ConsensusAPI) heartbeat() {
 
 		// If there have been no updates for the past while, warn the user
 		// that the beacon client is probably offline
-		if api.eth.BlockChain().Config().TerminalTotalDifficultyPassed || api.eth.Merger().TDDReached() {
+		if time.Since(lastForkchoiceUpdate) <= beaconUpdateConsensusTimeout || time.Since(lastNewPayloadUpdate) <= beaconUpdateConsensusTimeout {
-			if time.Since(lastForkchoiceUpdate) <= beaconUpdateConsensusTimeout || time.Since(lastNewPayloadUpdate) <= beaconUpdateConsensusTimeout {
+			offlineLogged = time.Time{}
-				offlineLogged = time.Time{}
-				continue
-			}
-
-			if time.Since(offlineLogged) > beaconUpdateWarnFrequency {
-				if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() {
-					if lastTransitionUpdate.IsZero() {
-						log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!")
-					} else {
-						log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!")
-					}
-				} else {
-					log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!")
-				}
-				offlineLogged = time.Now()
-			}
 			continue
 		}
+		if time.Since(offlineLogged) > beaconUpdateWarnFrequency {
+			if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() {
+				if lastTransitionUpdate.IsZero() {
+					log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!")
+				} else {
+					log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!")
+				}
+			} else {
+				log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!")
+			}
+			offlineLogged = time.Now()
+		}
+		continue
 	}
 }
 
@@ -879,8 +856,7 @@ func getBody(block *types.Block) *engine.ExecutionPayloadBodyV1 {
 	)
 
 	for j, tx := range body.Transactions {
-		data, _ := tx.MarshalBinary()
+		txs[j], _ = tx.MarshalBinary()
-		txs[j] = hexutil.Bytes(data)
 	}
 
 	// Post-shanghai withdrawals MUST be set to empty slice instead of nil

eth/catalyst/api_test.go

@@ -262,11 +262,8 @@ func TestInvalidPayloadTimestamp(t *testing.T) {
 		{0, true},
 		{parent.Time, true},
 		{parent.Time - 1, true},
-
+		{parent.Time + 1, false},
-		// TODO (MariusVanDerWijden) following tests are currently broken,
+		{uint64(time.Now().Unix()) + uint64(time.Minute), false},
-		// fixed in upcoming merge-kiln-v2 pr
-		//{parent.Time() + 1, false},
-		//{uint64(time.Now().Unix()) + uint64(time.Minute), false},
 	}
 
 	for i, test := range tests {
@@ -450,7 +447,9 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block)
 		t.Fatal("can't create node:", err)
 	}
 
-	ethcfg := &ethconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256}
+	mcfg := miner.DefaultConfig
+	mcfg.PendingFeeRecipient = testAddr
+	ethcfg := &ethconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: mcfg}
 	ethservice, err := eth.New(n, ethcfg)
 	if err != nil {
 		t.Fatal("can't create eth service:", err)
@@ -463,7 +462,6 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block)
 		t.Fatal("can't import test blocks:", err)
 	}
 
-	ethservice.SetEtherbase(testAddr)
 	ethservice.SetSynced()
 	return n, ethservice
 }
@@ -865,7 +863,6 @@ func TestTrickRemoteBlockCache(t *testing.T) {
 func TestInvalidBloom(t *testing.T) {
 	genesis, preMergeBlocks := generateMergeChain(10, false)
 	n, ethservice := startEthService(t, genesis, preMergeBlocks)
-	ethservice.Merger().ReachTTD()
 	defer n.Close()
 
 	commonAncestor := ethservice.BlockChain().CurrentBlock()
@@ -1047,7 +1044,6 @@ func TestWithdrawals(t *testing.T) {
 	genesis.Config.ShanghaiTime = &time
 
 	n, ethservice := startEthService(t, genesis, blocks)
-	ethservice.Merger().ReachTTD()
 	defer n.Close()
 
 	api := NewConsensusAPI(ethservice)
@@ -1165,7 +1161,6 @@ func TestNilWithdrawals(t *testing.T) {
 	genesis.Config.ShanghaiTime = &time
 
 	n, ethservice := startEthService(t, genesis, blocks)
-	ethservice.Merger().ReachTTD()
 	defer n.Close()
 
 	api := NewConsensusAPI(ethservice)
@@ -1592,7 +1587,6 @@ func TestParentBeaconBlockRoot(t *testing.T) {
 	genesis.Config.CancunTime = &time
 
 	n, ethservice := startEthService(t, genesis, blocks)
-	ethservice.Merger().ReachTTD()
 	defer n.Close()
 
 	api := NewConsensusAPI(ethservice)

eth/catalyst/blsync.go

@@ -0,0 +1,88 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package catalyst
+
+import (
+	"github.com/ethereum/go-ethereum/beacon/engine"
+	"github.com/ethereum/go-ethereum/beacon/types"
+	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/ethereum/go-ethereum/log"
+)
+
+// Blsync tracks the head of the beacon chain through the beacon light client
+// and drives the local node via ConsensusAPI.
+type Blsync struct {
+	engine  *ConsensusAPI
+	client  Client
+	headCh  chan types.ChainHeadEvent
+	headSub event.Subscription
+
+	quitCh chan struct{}
+}
+
+type Client interface {
+	SubscribeChainHeadEvent(ch chan<- types.ChainHeadEvent) event.Subscription
+	Start()
+	Stop()
+}
+
+// NewBlsync creates a new beacon light syncer.
+func NewBlsync(client Client, eth *eth.Ethereum) *Blsync {
+	return &Blsync{
+		engine: newConsensusAPIWithoutHeartbeat(eth),
+		client: client,
+		headCh: make(chan types.ChainHeadEvent, 16),
+		quitCh: make(chan struct{}),
+	}
+}
+
+// Start starts underlying beacon light client and the sync logic for driving
+// the local node.
+func (b *Blsync) Start() error {
+	log.Info("Beacon light sync started")
+	b.headSub = b.client.SubscribeChainHeadEvent(b.headCh)
+	go b.client.Start()
+
+	for {
+		select {
+		case <-b.quitCh:
+			return nil
+		case head := <-b.headCh:
+			if _, err := b.engine.NewPayloadV2(*head.HeadBlock); err != nil {
+				log.Error("failed to send new payload", "err", err)
+				continue
+			}
+			update := engine.ForkchoiceStateV1{
+				HeadBlockHash:      head.HeadBlock.BlockHash,
+				SafeBlockHash:      head.Finalized, //TODO pass finalized or empty hash here?
+				FinalizedBlockHash: head.Finalized,
+			}
+			if _, err := b.engine.ForkchoiceUpdatedV1(update, nil); err != nil {
+				log.Error("failed to send forkchoice updated", "err", err)
+				continue
+			}
+		}
+	}
+}
+
+// Stop signals to the light client and syncer to exit.
+func (b *Blsync) Stop() error {
+	b.client.Stop()
+	close(b.quitCh)
+	return nil
+}