forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-06-23 07:04:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge branch 'ethereum:master' into master

Browse source
This commit is contained in:
Sascha Ronnie Daoudia 2024-03-08 19:43:21 +01:00 committed by GitHub
commit b78bfdab0e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
210 changed files with 5936 additions and 6368 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.mailmap
View file

@ -56,7 +56,6 @@ Diederik Loerakker <proto@protolambda.com>
Dimitry Khokhlov <winsvega@mail.ru>
Domino Valdano <dominoplural@gmail.com>
Domino Valdano <dominoplural@gmail.com> <jeff@okcupid.com>
Edgar Aroutiounian <edgar.factorial@gmail.com>

3
accounts/abi/type.go
View file

@ -179,9 +179,6 @@ func NewType(t string, internalType string, components []ArgumentMarshaling) (ty
return Type{}, errors.New("abi: purely anonymous or underscored field is not supported")
}
fieldName := ResolveNameConflict(name, func(s string) bool { return used[s] })
if err != nil {
return Type{}, err
}
used[fieldName] = true
if !isValidFieldName(fieldName) {
return Type{}, fmt.Errorf("field %d has invalid name", idx)

2
accounts/accounts.go
View file

@ -195,7 +195,7 @@ func TextHash(data []byte) []byte {
//
// This gives context to the signed message and prevents signing of transactions.
func TextAndHash(data []byte) ([]byte, string) {
msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), string(data))
msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data)
hasher := sha3.NewLegacyKeccak256()
hasher.Write([]byte(msg))
return hasher.Sum(nil), msg

2
accounts/keystore/account_cache_test.go
View file

@ -86,7 +86,7 @@ func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error {
func TestWatchNewFile(t *testing.T) {
t.Parallel()
dir, ks := tmpKeyStore(t, false)
dir, ks := tmpKeyStore(t)
// Ensure the watcher is started before adding any files.
ks.Accounts()

9
accounts/keystore/keystore.go
View file

@ -87,15 +87,6 @@ func NewKeyStore(keydir string, scryptN, scryptP int) *KeyStore {
return ks
}
// NewPlaintextKeyStore creates a keystore for the given directory.
// Deprecated: Use NewKeyStore.
func NewPlaintextKeyStore(keydir string) *KeyStore {
keydir, _ = filepath.Abs(keydir)
ks := &KeyStore{storage: &keyStorePlain{keydir}}
ks.init(keydir)
return ks
}
func (ks *KeyStore) init(keydir string) {
// Lock the mutex since the account cache might call back with events
ks.mu.Lock()

34
accounts/keystore/keystore_test.go
View file

@ -37,7 +37,7 @@ var testSigData = make([]byte, 32)
func TestKeyStore(t *testing.T) {
t.Parallel()
dir, ks := tmpKeyStore(t, true)
dir, ks := tmpKeyStore(t)
a, err := ks.NewAccount("foo")
if err != nil {
@ -72,7 +72,7 @@ func TestKeyStore(t *testing.T) {
func TestSign(t *testing.T) {
t.Parallel()
_, ks := tmpKeyStore(t, true)
_, ks := tmpKeyStore(t)
pass := "" // not used but required by API
a1, err := ks.NewAccount(pass)
@ -89,7 +89,7 @@ func TestSign(t *testing.T) {
func TestSignWithPassphrase(t *testing.T) {
t.Parallel()
_, ks := tmpKeyStore(t, true)
_, ks := tmpKeyStore(t)
pass := "passwd"
acc, err := ks.NewAccount(pass)
@ -117,7 +117,7 @@ func TestSignWithPassphrase(t *testing.T) {
func TestTimedUnlock(t *testing.T) {
t.Parallel()
_, ks := tmpKeyStore(t, true)
_, ks := tmpKeyStore(t)
pass := "foo"
a1, err := ks.NewAccount(pass)
@ -152,7 +152,7 @@ func TestTimedUnlock(t *testing.T) {
func TestOverrideUnlock(t *testing.T) {
t.Parallel()
_, ks := tmpKeyStore(t, false)
_, ks := tmpKeyStore(t)
pass := "foo"
a1, err := ks.NewAccount(pass)
@ -193,7 +193,7 @@ func TestOverrideUnlock(t *testing.T) {
// This test should fail under -race if signing races the expiration goroutine.
func TestSignRace(t *testing.T) {
t.Parallel()
_, ks := tmpKeyStore(t, false)
_, ks := tmpKeyStore(t)
// Create a test account.
a1, err := ks.NewAccount("")
@ -238,7 +238,7 @@ func waitForKsUpdating(t *testing.T, ks *KeyStore, wantStatus bool, maxTime time
func TestWalletNotifierLifecycle(t *testing.T) {
t.Parallel()
// Create a temporary keystore to test with
_, ks := tmpKeyStore(t, false)
_, ks := tmpKeyStore(t)
// Ensure that the notification updater is not running yet
time.Sleep(250 * time.Millisecond)
@ -284,7 +284,7 @@ type walletEvent struct {
// or deleted from the keystore.
func TestWalletNotifications(t *testing.T) {
t.Parallel()
_, ks := tmpKeyStore(t, false)
_, ks := tmpKeyStore(t)
// Subscribe to the wallet feed and collect events.
var (
@ -346,7 +346,7 @@ func TestWalletNotifications(t *testing.T) {
// TestImportExport tests the import functionality of a keystore.
func TestImportECDSA(t *testing.T) {
t.Parallel()
_, ks := tmpKeyStore(t, true)
_, ks := tmpKeyStore(t)
key, err := crypto.GenerateKey()
if err != nil {
t.Fatalf("failed to generate key: %v", key)
@ -365,7 +365,7 @@ func TestImportECDSA(t *testing.T) {
// TestImportECDSA tests the import and export functionality of a keystore.
func TestImportExport(t *testing.T) {
t.Parallel()
_, ks := tmpKeyStore(t, true)
_, ks := tmpKeyStore(t)
acc, err := ks.NewAccount("old")
if err != nil {
t.Fatalf("failed to create account: %v", acc)
@ -374,7 +374,7 @@ func TestImportExport(t *testing.T) {
if err != nil {
t.Fatalf("failed to export account: %v", acc)
}
_, ks2 := tmpKeyStore(t, true)
_, ks2 := tmpKeyStore(t)
if _, err = ks2.Import(json, "old", "old"); err == nil {
t.Errorf("importing with invalid password succeeded")
}
@ -394,7 +394,7 @@ func TestImportExport(t *testing.T) {
// This test should fail under -race if importing races.
func TestImportRace(t *testing.T) {
t.Parallel()
_, ks := tmpKeyStore(t, true)
_, ks := tmpKeyStore(t)
acc, err := ks.NewAccount("old")
if err != nil {
t.Fatalf("failed to create account: %v", acc)
@ -403,7 +403,7 @@ func TestImportRace(t *testing.T) {
if err != nil {
t.Fatalf("failed to export account: %v", acc)
}
_, ks2 := tmpKeyStore(t, true)
_, ks2 := tmpKeyStore(t)
var atom atomic.Uint32
var wg sync.WaitGroup
wg.Add(2)
@ -457,11 +457,7 @@ func checkEvents(t *testing.T, want []walletEvent, have []walletEvent) {
}
}
func tmpKeyStore(t *testing.T, encrypted bool) (string, *KeyStore) {
func tmpKeyStore(t *testing.T) (string, *KeyStore) {
d := t.TempDir()
newKs := NewPlaintextKeyStore
if encrypted {
newKs = func(kd string) *KeyStore { return NewKeyStore(kd, veryLightScryptN, veryLightScryptP) }
}
return d, newKs(d)
return d, NewKeyStore(d, veryLightScryptN, veryLightScryptP)
}

8
accounts/usbwallet/hub.go
View file

@ -26,7 +26,7 @@ import (
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/karalabe/usb"
"github.com/karalabe/hid"
)
// LedgerScheme is the protocol scheme prefixing account and wallet URLs.
@ -109,7 +109,7 @@ func NewTrezorHubWithWebUSB() (*Hub, error) {
// newHub creates a new hardware wallet manager for generic USB devices.
func newHub(scheme string, vendorID uint16, productIDs []uint16, usageID uint16, endpointID int, makeDriver func(log.Logger) driver) (*Hub, error) {
if !usb.Supported() {
if !hid.Supported() {
return nil, errors.New("unsupported platform")
}
hub := &Hub{
@ -155,7 +155,7 @@ func (hub *Hub) refreshWallets() {
return
}
// Retrieve the current list of USB wallet devices
var devices []usb.DeviceInfo
var devices []hid.DeviceInfo
if runtime.GOOS == "linux" {
// hidapi on Linux opens the device during enumeration to retrieve some infos,
@ -170,7 +170,7 @@ func (hub *Hub) refreshWallets() {
return
}
}
infos, err := usb.Enumerate(hub.vendorID, 0)
infos, err := hid.Enumerate(hub.vendorID, 0)
if err != nil {
failcount := hub.enumFails.Add(1)
if runtime.GOOS == "linux" {

6
accounts/usbwallet/wallet.go
View file

@ -31,7 +31,7 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/karalabe/usb"
"github.com/karalabe/hid"
)
// Maximum time between wallet health checks to detect USB unplugs.
@ -79,8 +79,8 @@ type wallet struct {
driver driver // Hardware implementation of the low level device operations
url *accounts.URL // Textual URL uniquely identifying this wallet
info usb.DeviceInfo // Known USB device infos about the wallet
device usb.Device // USB device advertising itself as a hardware wallet
info hid.DeviceInfo // Known USB device infos about the wallet
device hid.Device // USB device advertising itself as a hardware wallet
accounts []accounts.Account // List of derive accounts pinned on the hardware wallet
paths map[common.Address]accounts.DerivationPath // Known derivation paths for signing operations

203
beacon/blsync/block_sync.go Executable file
View file

@ -0,0 +1,203 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package blsync
import (
"fmt"
"math/big"
"github.com/ethereum/go-ethereum/beacon/engine"
"github.com/ethereum/go-ethereum/beacon/light/request"
"github.com/ethereum/go-ethereum/beacon/light/sync"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/lru"
ctypes "github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/trie"
"github.com/holiman/uint256"
"github.com/protolambda/zrnt/eth2/beacon/capella"
"github.com/protolambda/zrnt/eth2/configs"
"github.com/protolambda/ztyp/tree"
)
// beaconBlockSync implements request.Module; it fetches the beacon blocks belonging
// to the validated and prefetch heads.
type beaconBlockSync struct {
recentBlocks *lru.Cache[common.Hash, *capella.BeaconBlock]
locked map[common.Hash]request.ServerAndID
serverHeads map[request.Server]common.Hash
headTracker headTracker
lastHeadInfo types.HeadInfo
chainHeadFeed *event.Feed
}
type headTracker interface {
PrefetchHead() types.HeadInfo
ValidatedHead() (types.SignedHeader, bool)
ValidatedFinality() (types.FinalityUpdate, bool)
}
// newBeaconBlockSync returns a new beaconBlockSync.
func newBeaconBlockSync(headTracker headTracker, chainHeadFeed *event.Feed) *beaconBlockSync {
return &beaconBlockSync{
headTracker: headTracker,
chainHeadFeed: chainHeadFeed,
recentBlocks: lru.NewCache[common.Hash, *capella.BeaconBlock](10),
locked: make(map[common.Hash]request.ServerAndID),
serverHeads: make(map[request.Server]common.Hash),
}
}
// Process implements request.Module.
func (s *beaconBlockSync) Process(requester request.Requester, events []request.Event) {
for _, event := range events {
switch event.Type {
case request.EvResponse, request.EvFail, request.EvTimeout:
sid, req, resp := event.RequestInfo()
blockRoot := common.Hash(req.(sync.ReqBeaconBlock))
if resp != nil {
s.recentBlocks.Add(blockRoot, resp.(*capella.BeaconBlock))
}
if s.locked[blockRoot] == sid {
delete(s.locked, blockRoot)
}
case sync.EvNewHead:
s.serverHeads[event.Server] = event.Data.(types.HeadInfo).BlockRoot
case request.EvUnregistered:
delete(s.serverHeads, event.Server)
}
}
s.updateEventFeed()
// request validated head block if unavailable and not yet requested
if vh, ok := s.headTracker.ValidatedHead(); ok {
s.tryRequestBlock(requester, vh.Header.Hash(), false)
}
// request prefetch head if the given server has announced it
if prefetchHead := s.headTracker.PrefetchHead().BlockRoot; prefetchHead != (common.Hash{}) {
s.tryRequestBlock(requester, prefetchHead, true)
}
}
func (s *beaconBlockSync) tryRequestBlock(requester request.Requester, blockRoot common.Hash, needSameHead bool) {
if _, ok := s.recentBlocks.Get(blockRoot); ok {
return
}
if _, ok := s.locked[blockRoot]; ok {
return
}
for _, server := range requester.CanSendTo() {
if needSameHead && (s.serverHeads[server] != blockRoot) {
continue
}
id := requester.Send(server, sync.ReqBeaconBlock(blockRoot))
s.locked[blockRoot] = request.ServerAndID{Server: server, ID: id}
return
}
}
func blockHeadInfo(block *capella.BeaconBlock) types.HeadInfo {
if block == nil {
return types.HeadInfo{}
}
return types.HeadInfo{Slot: uint64(block.Slot), BlockRoot: beaconBlockHash(block)}
}
// beaconBlockHash calculates the hash of a beacon block.
func beaconBlockHash(beaconBlock *capella.BeaconBlock) common.Hash {
return common.Hash(beaconBlock.HashTreeRoot(configs.Mainnet, tree.GetHashFn()))
}
// getExecBlock extracts the execution block from the beacon block's payload.
func getExecBlock(beaconBlock *capella.BeaconBlock) (*ctypes.Block, error) {
payload := &beaconBlock.Body.ExecutionPayload
txs := make([]*ctypes.Transaction, len(payload.Transactions))
for i, opaqueTx := range payload.Transactions {
var tx ctypes.Transaction
if err := tx.UnmarshalBinary(opaqueTx); err != nil {
return nil, fmt.Errorf("failed to parse tx %d: %v", i, err)
}
txs[i] = &tx
}
withdrawals := make([]*ctypes.Withdrawal, len(payload.Withdrawals))
for i, w := range payload.Withdrawals {
withdrawals[i] = &ctypes.Withdrawal{
Index: uint64(w.Index),
Validator: uint64(w.ValidatorIndex),
Address: common.Address(w.Address),
Amount: uint64(w.Amount),
}
}
wroot := ctypes.DeriveSha(ctypes.Withdrawals(withdrawals), trie.NewStackTrie(nil))
execHeader := &ctypes.Header{
ParentHash: common.Hash(payload.ParentHash),
UncleHash: ctypes.EmptyUncleHash,
Coinbase: common.Address(payload.FeeRecipient),
Root: common.Hash(payload.StateRoot),
TxHash: ctypes.DeriveSha(ctypes.Transactions(txs), trie.NewStackTrie(nil)),
ReceiptHash: common.Hash(payload.ReceiptsRoot),
Bloom: ctypes.Bloom(payload.LogsBloom),
Difficulty: common.Big0,
Number: new(big.Int).SetUint64(uint64(payload.BlockNumber)),
GasLimit: uint64(payload.GasLimit),
GasUsed: uint64(payload.GasUsed),
Time: uint64(payload.Timestamp),
Extra: []byte(payload.ExtraData),
MixDigest: common.Hash(payload.PrevRandao), // reused in merge
Nonce: ctypes.BlockNonce{}, // zero
BaseFee: (*uint256.Int)(&payload.BaseFeePerGas).ToBig(),
WithdrawalsHash: &wroot,
}
execBlock := ctypes.NewBlockWithHeader(execHeader).WithBody(txs, nil).WithWithdrawals(withdrawals)
if execBlockHash := execBlock.Hash(); execBlockHash != common.Hash(payload.BlockHash) {
return execBlock, fmt.Errorf("Sanity check failed, payload hash does not match (expected %x, got %x)", common.Hash(payload.BlockHash), execBlockHash)
}
return execBlock, nil
}
func (s *beaconBlockSync) updateEventFeed() {
head, ok := s.headTracker.ValidatedHead()
if !ok {
return
}
finality, ok := s.headTracker.ValidatedFinality() //TODO fetch directly if subscription does not deliver
if !ok || head.Header.Epoch() != finality.Attested.Header.Epoch() {
return
}
validatedHead := head.Header.Hash()
headBlock, ok := s.recentBlocks.Get(validatedHead)
if !ok {
return
}
headInfo := blockHeadInfo(headBlock)
if headInfo == s.lastHeadInfo {
return
}
s.lastHeadInfo = headInfo
// new head block and finality info available; extract executable data and send event to feed
execBlock, err := getExecBlock(headBlock)
if err != nil {
log.Error("Error extracting execution block from validated beacon block", "error", err)
return
}
s.chainHeadFeed.Send(types.ChainHeadEvent{
HeadBlock: engine.BlockToExecutableData(execBlock, nil, nil).ExecutionPayload,
Finalized: common.Hash(finality.Finalized.PayloadHeader.BlockHash),
})
}

160
beacon/blsync/block_sync_test.go Normal file
View file

@ -0,0 +1,160 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package blsync
import (
"testing"
"github.com/ethereum/go-ethereum/beacon/light/request"
"github.com/ethereum/go-ethereum/beacon/light/sync"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/event"
"github.com/protolambda/zrnt/eth2/beacon/capella"
"github.com/protolambda/zrnt/eth2/configs"
"github.com/protolambda/ztyp/tree"
)
var (
testServer1 = "testServer1"
testServer2 = "testServer2"
testBlock1 = &capella.BeaconBlock{
Slot: 123,
Body: capella.BeaconBlockBody{
ExecutionPayload: capella.ExecutionPayload{BlockNumber: 456},
},
}
testBlock2 = &capella.BeaconBlock{
Slot: 124,
Body: capella.BeaconBlockBody{
ExecutionPayload: capella.ExecutionPayload{BlockNumber: 457},
},
}
)
func init() {
eb1, _ := getExecBlock(testBlock1)
testBlock1.Body.ExecutionPayload.BlockHash = tree.Root(eb1.Hash())
eb2, _ := getExecBlock(testBlock2)
testBlock2.Body.ExecutionPayload.BlockHash = tree.Root(eb2.Hash())
}
func TestBlockSync(t *testing.T) {
ht := &testHeadTracker{}
eventFeed := new(event.Feed)
blockSync := newBeaconBlockSync(ht, eventFeed)
headCh := make(chan types.ChainHeadEvent, 16)
eventFeed.Subscribe(headCh)
ts := sync.NewTestScheduler(t, blockSync)
ts.AddServer(testServer1, 1)
ts.AddServer(testServer2, 1)
expHeadBlock := func(tci int, expHead *capella.BeaconBlock) {
var expNumber, headNumber uint64
if expHead != nil {
expNumber = uint64(expHead.Body.ExecutionPayload.BlockNumber)
}
select {
case event := <-headCh:
headNumber = event.HeadBlock.Number
default:
}
if headNumber != expNumber {
t.Errorf("Wrong head block in test case #%d (expected block number %d, got %d)", tci, expNumber, headNumber)
}
}
// no block requests expected until head tracker knows about a head
ts.Run(1)
expHeadBlock(1, nil)
// set block 1 as prefetch head, announced by server 2
head1 := blockHeadInfo(testBlock1)
ht.prefetch = head1
ts.ServerEvent(sync.EvNewHead, testServer2, head1)
// expect request to server 2 which has announced the head
ts.Run(2, testServer2, sync.ReqBeaconBlock(head1.BlockRoot))
// valid response
ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testBlock1)
ts.AddAllowance(testServer2, 1)
ts.Run(3)
// head block still not expected as the fetched block is not the validated head yet
expHeadBlock(3, nil)
// set as validated head, expect no further requests but block 1 set as head block
ht.validated.Header = blockHeader(testBlock1)
ts.Run(4)
expHeadBlock(4, testBlock1)
// set block 2 as prefetch head, announced by server 1
head2 := blockHeadInfo(testBlock2)
ht.prefetch = head2
ts.ServerEvent(sync.EvNewHead, testServer1, head2)
// expect request to server 1
ts.Run(5, testServer1, sync.ReqBeaconBlock(head2.BlockRoot))
// req2 fails, no further requests expected because server 2 has not announced it
ts.RequestEvent(request.EvFail, ts.Request(5, 1), nil)
ts.Run(6)
// set as validated head before retrieving block; now it's assumed to be available from server 2 too
ht.validated.Header = blockHeader(testBlock2)
// expect req2 retry to server 2
ts.Run(7, testServer2, sync.ReqBeaconBlock(head2.BlockRoot))
// now head block should be unavailable again
expHeadBlock(4, nil)
// valid response, now head block should be block 2 immediately as it is already validated
ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testBlock2)
ts.Run(8)
expHeadBlock(5, testBlock2)
}
func blockHeader(block *capella.BeaconBlock) types.Header {
return types.Header{
Slot: uint64(block.Slot),
ProposerIndex: uint64(block.ProposerIndex),
ParentRoot: common.Hash(block.ParentRoot),
StateRoot: common.Hash(block.StateRoot),
BodyRoot: common.Hash(block.Body.HashTreeRoot(configs.Mainnet, tree.GetHashFn())),
}
}
type testHeadTracker struct {
prefetch types.HeadInfo
validated types.SignedHeader
}
func (h *testHeadTracker) PrefetchHead() types.HeadInfo {
return h.prefetch
}
func (h *testHeadTracker) ValidatedHead() (types.SignedHeader, bool) {
return h.validated, h.validated.Header != (types.Header{})
}
// TODO add test case for finality
func (h *testHeadTracker) ValidatedFinality() (types.FinalityUpdate, bool) {
return types.FinalityUpdate{
Attested: types.HeaderWithExecProof{Header: h.validated.Header},
Finalized: types.HeaderWithExecProof{PayloadHeader: &capella.ExecutionPayloadHeader{}},
Signature: h.validated.Signature,
SignatureSlot: h.validated.SignatureSlot,
}, h.validated.Header != (types.Header{})
}

103
beacon/blsync/client.go Normal file
View file

@ -0,0 +1,103 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package blsync
import (
"strings"
"github.com/ethereum/go-ethereum/beacon/light"
"github.com/ethereum/go-ethereum/beacon/light/api"
"github.com/ethereum/go-ethereum/beacon/light/request"
"github.com/ethereum/go-ethereum/beacon/light/sync"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/ethereum/go-ethereum/event"
"github.com/urfave/cli/v2"
)
type Client struct {
scheduler *request.Scheduler
chainHeadFeed *event.Feed
urls []string
customHeader map[string]string
}
func NewClient(ctx *cli.Context) *Client {
if !ctx.IsSet(utils.BeaconApiFlag.Name) {
utils.Fatalf("Beacon node light client API URL not specified")
}
var (
chainConfig = makeChainConfig(ctx)
customHeader = make(map[string]string)
)
for _, s := range ctx.StringSlice(utils.BeaconApiHeaderFlag.Name) {
kv := strings.Split(s, ":")
if len(kv) != 2 {
utils.Fatalf("Invalid custom API header entry: %s", s)
}
customHeader[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
}
// create data structures
var (
db = memorydb.New()
threshold = ctx.Int(utils.BeaconThresholdFlag.Name)
committeeChain = light.NewCommitteeChain(db, chainConfig.ChainConfig, threshold, !ctx.Bool(utils.BeaconNoFilterFlag.Name))
headTracker = light.NewHeadTracker(committeeChain, threshold)
)
headSync := sync.NewHeadSync(headTracker, committeeChain)
// set up scheduler and sync modules
chainHeadFeed := new(event.Feed)
scheduler := request.NewScheduler()
checkpointInit := sync.NewCheckpointInit(committeeChain, chainConfig.Checkpoint)
forwardSync := sync.NewForwardUpdateSync(committeeChain)
beaconBlockSync := newBeaconBlockSync(headTracker, chainHeadFeed)
scheduler.RegisterTarget(headTracker)
scheduler.RegisterTarget(committeeChain)
scheduler.RegisterModule(checkpointInit, "checkpointInit")
scheduler.RegisterModule(forwardSync, "forwardSync")
scheduler.RegisterModule(headSync, "headSync")
scheduler.RegisterModule(beaconBlockSync, "beaconBlockSync")
return &Client{
scheduler: scheduler,
urls: ctx.StringSlice(utils.BeaconApiFlag.Name),
customHeader: customHeader,
chainHeadFeed: chainHeadFeed,
}
}
// SubscribeChainHeadEvent allows callers to subscribe a provided channel to new
// head updates.
func (c *Client) SubscribeChainHeadEvent(ch chan<- types.ChainHeadEvent) event.Subscription {
return c.chainHeadFeed.Subscribe(ch)
}
func (c *Client) Start() {
c.scheduler.Start()
// register server(s)
for _, url := range c.urls {
beaconApi := api.NewBeaconLightApi(url, c.customHeader)
c.scheduler.RegisterServer(request.NewServer(api.NewApiServer(beaconApi), &mclock.System{}))
}
}
func (c *Client) Stop() {
c.scheduler.Stop()
}

113
beacon/blsync/config.go Normal file
View file

@ -0,0 +1,113 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package blsync
import (
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/urfave/cli/v2"
)
// lightClientConfig contains beacon light client configuration
type lightClientConfig struct {
*types.ChainConfig
Checkpoint common.Hash
}
var (
MainnetConfig = lightClientConfig{
ChainConfig: (&types.ChainConfig{
GenesisValidatorsRoot: common.HexToHash("0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95"),
GenesisTime: 1606824023,
}).
AddFork("GENESIS", 0, []byte{0, 0, 0, 0}).
AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}).
AddFork("BELLATRIX", 144896, []byte{2, 0, 0, 0}).
AddFork("CAPELLA", 194048, []byte{3, 0, 0, 0}),
Checkpoint: common.HexToHash("0x388be41594ec7d6a6894f18c73f3469f07e2c19a803de4755d335817ed8e2e5a"),
}
SepoliaConfig = lightClientConfig{
ChainConfig: (&types.ChainConfig{
GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"),
GenesisTime: 1655733600,
}).
AddFork("GENESIS", 0, []byte{144, 0, 0, 105}).
AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}).
AddFork("BELLATRIX", 100, []byte{144, 0, 0, 113}).
AddFork("CAPELLA", 56832, []byte{144, 0, 0, 114}),
Checkpoint: common.HexToHash("0x1005a6d9175e96bfbce4d35b80f468e9bff0b674e1e861d16e09e10005a58e81"),
}
GoerliConfig = lightClientConfig{
ChainConfig: (&types.ChainConfig{
GenesisValidatorsRoot: common.HexToHash("0x043db0d9a83813551ee2f33450d23797757d430911a9320530ad8a0eabc43efb"),
GenesisTime: 1614588812,
}).
AddFork("GENESIS", 0, []byte{0, 0, 16, 32}).
AddFork("ALTAIR", 36660, []byte{1, 0, 16, 32}).
AddFork("BELLATRIX", 112260, []byte{2, 0, 16, 32}).
AddFork("CAPELLA", 162304, []byte{3, 0, 16, 32}),
Checkpoint: common.HexToHash("0x53a0f4f0a378e2c4ae0a9ee97407eb69d0d737d8d8cd0a5fb1093f42f7b81c49"),
}
)
func makeChainConfig(ctx *cli.Context) lightClientConfig {
utils.CheckExclusive(ctx, utils.MainnetFlag, utils.GoerliFlag, utils.SepoliaFlag)
customConfig := ctx.IsSet(utils.BeaconConfigFlag.Name) || ctx.IsSet(utils.BeaconGenesisRootFlag.Name) || ctx.IsSet(utils.BeaconGenesisTimeFlag.Name)
var config lightClientConfig
switch {
case ctx.Bool(utils.MainnetFlag.Name):
config = MainnetConfig
case ctx.Bool(utils.SepoliaFlag.Name):
config = SepoliaConfig
case ctx.Bool(utils.GoerliFlag.Name):
config = GoerliConfig
default:
if !customConfig {
config = MainnetConfig
}
}
if customConfig && config.Forks != nil {
utils.Fatalf("Cannot use custom beacon chain config flags in combination with pre-defined network config")
}
if ctx.IsSet(utils.BeaconGenesisRootFlag.Name) {
if c, err := hexutil.Decode(ctx.String(utils.BeaconGenesisRootFlag.Name)); err == nil && len(c) <= 32 {
copy(config.GenesisValidatorsRoot[:len(c)], c)
} else {
utils.Fatalf("Invalid hex string", "beacon.genesis.gvroot", ctx.String(utils.BeaconGenesisRootFlag.Name), "error", err)
}
}
if ctx.IsSet(utils.BeaconGenesisTimeFlag.Name) {
config.GenesisTime = ctx.Uint64(utils.BeaconGenesisTimeFlag.Name)
}
if ctx.IsSet(utils.BeaconConfigFlag.Name) {
if err := config.ChainConfig.LoadForks(ctx.String(utils.BeaconConfigFlag.Name)); err != nil {
utils.Fatalf("Could not load beacon chain config file", "file name", ctx.String(utils.BeaconConfigFlag.Name), "error", err)
}
}
if ctx.IsSet(utils.BeaconCheckpointFlag.Name) {
if c, err := hexutil.Decode(ctx.String(utils.BeaconCheckpointFlag.Name)); err == nil && len(c) <= 32 {
copy(config.Checkpoint[:len(c)], c)
} else {
utils.Fatalf("Invalid hex string", "beacon.checkpoint", ctx.String(utils.BeaconCheckpointFlag.Name), "error", err)
}
}
return config
}

103
beacon/light/api/api_server.go Executable file
View file

@ -0,0 +1,103 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"reflect"
"github.com/ethereum/go-ethereum/beacon/light/request"
"github.com/ethereum/go-ethereum/beacon/light/sync"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
)
// ApiServer is a wrapper around BeaconLightApi that implements request.requestServer.
type ApiServer struct {
api *BeaconLightApi
eventCallback func(event request.Event)
unsubscribe func()
}
// NewApiServer creates a new ApiServer.
func NewApiServer(api *BeaconLightApi) *ApiServer {
return &ApiServer{api: api}
}
// Subscribe implements request.requestServer.
func (s *ApiServer) Subscribe(eventCallback func(event request.Event)) {
s.eventCallback = eventCallback
listener := HeadEventListener{
OnNewHead: func(slot uint64, blockRoot common.Hash) {
log.Debug("New head received", "slot", slot, "blockRoot", blockRoot)
eventCallback(request.Event{Type: sync.EvNewHead, Data: types.HeadInfo{Slot: slot, BlockRoot: blockRoot}})
},
OnSignedHead: func(head types.SignedHeader) {
log.Debug("New signed head received", "slot", head.Header.Slot, "blockRoot", head.Header.Hash(), "signerCount", head.Signature.SignerCount())
eventCallback(request.Event{Type: sync.EvNewSignedHead, Data: head})
},
OnFinality: func(head types.FinalityUpdate) {
log.Debug("New finality update received", "slot", head.Attested.Slot, "blockRoot", head.Attested.Hash(), "signerCount", head.Signature.SignerCount())
eventCallback(request.Event{Type: sync.EvNewFinalityUpdate, Data: head})
},
OnError: func(err error) {
log.Warn("Head event stream error", "err", err)
},
}
s.unsubscribe = s.api.StartHeadListener(listener)
}
// SendRequest implements request.requestServer.
func (s *ApiServer) SendRequest(id request.ID, req request.Request) {
go func() {
var resp request.Response
var err error
switch data := req.(type) {
case sync.ReqUpdates:
log.Debug("Beacon API: requesting light client update", "reqid", id, "period", data.FirstPeriod, "count", data.Count)
var r sync.RespUpdates
r.Updates, r.Committees, err = s.api.GetBestUpdatesAndCommittees(data.FirstPeriod, data.Count)
resp = r
case sync.ReqHeader:
log.Debug("Beacon API: requesting header", "reqid", id, "hash", common.Hash(data))
resp, err = s.api.GetHeader(common.Hash(data))
case sync.ReqCheckpointData:
log.Debug("Beacon API: requesting checkpoint data", "reqid", id, "hash", common.Hash(data))
resp, err = s.api.GetCheckpointData(common.Hash(data))
case sync.ReqBeaconBlock:
log.Debug("Beacon API: requesting block", "reqid", id, "hash", common.Hash(data))
resp, err = s.api.GetBeaconBlock(common.Hash(data))
default:
}
if err != nil {
log.Warn("Beacon API request failed", "type", reflect.TypeOf(req), "reqid", id, "err", err)
s.eventCallback(request.Event{Type: request.EvFail, Data: request.RequestResponse{ID: id, Request: req}})
} else {
s.eventCallback(request.Event{Type: request.EvResponse, Data: request.RequestResponse{ID: id, Request: req, Response: resp}})
}
}()
}
// Unsubscribe implements request.requestServer.
// Note: Unsubscribe should not be called concurrently with Subscribe.
func (s *ApiServer) Unsubscribe() {
if s.unsubscribe != nil {
s.unsubscribe()
s.unsubscribe = nil
}
}

496
beacon/light/api/light_api.go Executable file
View file

@ -0,0 +1,496 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more detaiapi.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package api
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"time"
"github.com/donovanhide/eventsource"
"github.com/ethereum/go-ethereum/beacon/merkle"
"github.com/ethereum/go-ethereum/beacon/params"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/protolambda/zrnt/eth2/beacon/capella"
"github.com/protolambda/zrnt/eth2/configs"
"github.com/protolambda/ztyp/tree"
)
var (
ErrNotFound = errors.New("404 Not Found")
ErrInternal = errors.New("500 Internal Server Error")
)
type CommitteeUpdate struct {
Version string
Update types.LightClientUpdate
NextSyncCommittee types.SerializedSyncCommittee
}
// See data structure definition here:
// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientupdate
type committeeUpdateJson struct {
Version string `json:"version"`
Data committeeUpdateData `json:"data"`
}
type committeeUpdateData struct {
Header jsonBeaconHeader `json:"attested_header"`
NextSyncCommittee types.SerializedSyncCommittee `json:"next_sync_committee"`
NextSyncCommitteeBranch merkle.Values `json:"next_sync_committee_branch"`
FinalizedHeader *jsonBeaconHeader `json:"finalized_header,omitempty"`
FinalityBranch merkle.Values `json:"finality_branch,omitempty"`
SyncAggregate types.SyncAggregate `json:"sync_aggregate"`
SignatureSlot common.Decimal `json:"signature_slot"`
}
type jsonBeaconHeader struct {
Beacon types.Header `json:"beacon"`
}
type jsonHeaderWithExecProof struct {
Beacon types.Header `json:"beacon"`
Execution *capella.ExecutionPayloadHeader `json:"execution"`
ExecutionBranch merkle.Values `json:"execution_branch"`
}
// UnmarshalJSON unmarshals from JSON.
func (u *CommitteeUpdate) UnmarshalJSON(input []byte) error {
var dec committeeUpdateJson
if err := json.Unmarshal(input, &dec); err != nil {
return err
}
u.Version = dec.Version
u.NextSyncCommittee = dec.Data.NextSyncCommittee
u.Update = types.LightClientUpdate{
AttestedHeader: types.SignedHeader{
Header: dec.Data.Header.Beacon,
Signature: dec.Data.SyncAggregate,
SignatureSlot: uint64(dec.Data.SignatureSlot),
},
NextSyncCommitteeRoot: u.NextSyncCommittee.Root(),
NextSyncCommitteeBranch: dec.Data.NextSyncCommitteeBranch,
FinalityBranch: dec.Data.FinalityBranch,
}
if dec.Data.FinalizedHeader != nil {
u.Update.FinalizedHeader = &dec.Data.FinalizedHeader.Beacon
}
return nil
}
// fetcher is an interface useful for debug-harnessing the http api.
type fetcher interface {
Do(req *http.Request) (*http.Response, error)
}
// BeaconLightApi requests light client information from a beacon node REST API.
// Note: all required API endpoints are currently only implemented by Lodestar.
type BeaconLightApi struct {
url string
client fetcher
customHeaders map[string]string
}
func NewBeaconLightApi(url string, customHeaders map[string]string) *BeaconLightApi {
return &BeaconLightApi{
url: url,
client: &http.Client{
Timeout: time.Second * 10,
},
customHeaders: customHeaders,
}
}
func (api *BeaconLightApi) httpGet(path string) ([]byte, error) {
req, err := http.NewRequest("GET", api.url+path, nil)
if err != nil {
return nil, err
}
for k, v := range api.customHeaders {
req.Header.Set(k, v)
}
resp, err := api.client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
switch resp.StatusCode {
case 200:
return io.ReadAll(resp.Body)
case 404:
return nil, ErrNotFound
case 500:
return nil, ErrInternal
default:
return nil, fmt.Errorf("unexpected error from API endpoint \"%s\": status code %d", path, resp.StatusCode)
}
}
func (api *BeaconLightApi) httpGetf(format string, params ...any) ([]byte, error) {
return api.httpGet(fmt.Sprintf(format, params...))
}
// GetBestUpdateAndCommittee fetches and validates LightClientUpdate for given
// period and full serialized committee for the next period (committee root hash
// equals update.NextSyncCommitteeRoot).
// Note that the results are validated but the update signature should be verified
// by the caller as its validity depends on the update chain.
func (api *BeaconLightApi) GetBestUpdatesAndCommittees(firstPeriod, count uint64) ([]*types.LightClientUpdate, []*types.SerializedSyncCommittee, error) {
resp, err := api.httpGetf("/eth/v1/beacon/light_client/updates?start_period=%d&count=%d", firstPeriod, count)
if err != nil {
return nil, nil, err
}
var data []CommitteeUpdate
if err := json.Unmarshal(resp, &data); err != nil {
return nil, nil, err
}
if len(data) != int(count) {
return nil, nil, errors.New("invalid number of committee updates")
}
updates := make([]*types.LightClientUpdate, int(count))
committees := make([]*types.SerializedSyncCommittee, int(count))
for i, d := range data {
if d.Update.AttestedHeader.Header.SyncPeriod() != firstPeriod+uint64(i) {
return nil, nil, errors.New("wrong committee update header period")
}
if err := d.Update.Validate(); err != nil {
return nil, nil, err
}
if d.NextSyncCommittee.Root() != d.Update.NextSyncCommitteeRoot {
return nil, nil, errors.New("wrong sync committee root")
}
updates[i], committees[i] = new(types.LightClientUpdate), new(types.SerializedSyncCommittee)
*updates[i], *committees[i] = d.Update, d.NextSyncCommittee
}
return updates, committees, nil
}
// GetOptimisticHeadUpdate fetches a signed header based on the latest available
// optimistic update. Note that the signature should be verified by the caller
// as its validity depends on the update chain.
//
// See data structure definition here:
// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientoptimisticupdate
func (api *BeaconLightApi) GetOptimisticHeadUpdate() (types.SignedHeader, error) {
resp, err := api.httpGet("/eth/v1/beacon/light_client/optimistic_update")
if err != nil {
return types.SignedHeader{}, err
}
return decodeOptimisticHeadUpdate(resp)
}
func decodeOptimisticHeadUpdate(enc []byte) (types.SignedHeader, error) {
var data struct {
Data struct {
Header jsonBeaconHeader `json:"attested_header"`
Aggregate types.SyncAggregate `json:"sync_aggregate"`
SignatureSlot common.Decimal `json:"signature_slot"`
} `json:"data"`
}
if err := json.Unmarshal(enc, &data); err != nil {
return types.SignedHeader{}, err
}
if data.Data.Header.Beacon.StateRoot == (common.Hash{}) {
// workaround for different event encoding format in Lodestar
if err := json.Unmarshal(enc, &data.Data); err != nil {
return types.SignedHeader{}, err
}
}
if len(data.Data.Aggregate.Signers) != params.SyncCommitteeBitmaskSize {
return types.SignedHeader{}, errors.New("invalid sync_committee_bits length")
}
if len(data.Data.Aggregate.Signature) != params.BLSSignatureSize {
return types.SignedHeader{}, errors.New("invalid sync_committee_signature length")
}
return types.SignedHeader{
Header: data.Data.Header.Beacon,
Signature: data.Data.Aggregate,
SignatureSlot: uint64(data.Data.SignatureSlot),
}, nil
}
// GetFinalityUpdate fetches the latest available finality update.
//
// See data structure definition here:
// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientfinalityupdate
func (api *BeaconLightApi) GetFinalityUpdate() (types.FinalityUpdate, error) {
resp, err := api.httpGet("/eth/v1/beacon/light_client/finality_update")
if err != nil {
return types.FinalityUpdate{}, err
}
return decodeFinalityUpdate(resp)
}
func decodeFinalityUpdate(enc []byte) (types.FinalityUpdate, error) {
var data struct {
Data struct {
Attested jsonHeaderWithExecProof `json:"attested_header"`
Finalized jsonHeaderWithExecProof `json:"finalized_header"`
FinalityBranch merkle.Values `json:"finality_branch"`
Aggregate types.SyncAggregate `json:"sync_aggregate"`
SignatureSlot common.Decimal `json:"signature_slot"`
} `json:"data"`
}
if err := json.Unmarshal(enc, &data); err != nil {
return types.FinalityUpdate{}, err
}
if len(data.Data.Aggregate.Signers) != params.SyncCommitteeBitmaskSize {
return types.FinalityUpdate{}, errors.New("invalid sync_committee_bits length")
}
if len(data.Data.Aggregate.Signature) != params.BLSSignatureSize {
return types.FinalityUpdate{}, errors.New("invalid sync_committee_signature length")
}
return types.FinalityUpdate{
Attested: types.HeaderWithExecProof{
Header: data.Data.Attested.Beacon,
PayloadHeader: data.Data.Attested.Execution,
PayloadBranch: data.Data.Attested.ExecutionBranch,
},
Finalized: types.HeaderWithExecProof{
Header: data.Data.Finalized.Beacon,
PayloadHeader: data.Data.Finalized.Execution,
PayloadBranch: data.Data.Finalized.ExecutionBranch,
},
FinalityBranch: data.Data.FinalityBranch,
Signature: data.Data.Aggregate,
SignatureSlot: uint64(data.Data.SignatureSlot),
}, nil
}
// GetHead fetches and validates the beacon header with the given blockRoot.
// If blockRoot is null hash then the latest head header is fetched.
func (api *BeaconLightApi) GetHeader(blockRoot common.Hash) (types.Header, error) {
var blockId string
if blockRoot == (common.Hash{}) {
blockId = "head"
} else {
blockId = blockRoot.Hex()
}
resp, err := api.httpGetf("/eth/v1/beacon/headers/%s", blockId)
if err != nil {
return types.Header{}, err
}
var data struct {
Data struct {
Root common.Hash `json:"root"`
Canonical bool `json:"canonical"`
Header struct {
Message types.Header `json:"message"`
Signature hexutil.Bytes `json:"signature"`
} `json:"header"`
} `json:"data"`
}
if err := json.Unmarshal(resp, &data); err != nil {
return types.Header{}, err
}
header := data.Data.Header.Message
if blockRoot == (common.Hash{}) {
blockRoot = data.Data.Root
}
if header.Hash() != blockRoot {
return types.Header{}, errors.New("retrieved beacon header root does not match")
}
return header, nil
}
// GetCheckpointData fetches and validates bootstrap data belonging to the given checkpoint.
func (api *BeaconLightApi) GetCheckpointData(checkpointHash common.Hash) (*types.BootstrapData, error) {
resp, err := api.httpGetf("/eth/v1/beacon/light_client/bootstrap/0x%x", checkpointHash[:])
if err != nil {
return nil, err
}
// See data structure definition here:
// https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/light-client/sync-protocol.md#lightclientbootstrap
type bootstrapData struct {
Data struct {
Header jsonBeaconHeader `json:"header"`
Committee *types.SerializedSyncCommittee `json:"current_sync_committee"`
CommitteeBranch merkle.Values `json:"current_sync_committee_branch"`
} `json:"data"`
}
var data bootstrapData
if err := json.Unmarshal(resp, &data); err != nil {
return nil, err
}
if data.Data.Committee == nil {
return nil, errors.New("sync committee is missing")
}
header := data.Data.Header.Beacon
if header.Hash() != checkpointHash {
return nil, fmt.Errorf("invalid checkpoint block header, have %v want %v", header.Hash(), checkpointHash)
}
checkpoint := &types.BootstrapData{
Header: header,
CommitteeBranch: data.Data.CommitteeBranch,
CommitteeRoot: data.Data.Committee.Root(),
Committee: data.Data.Committee,
}
if err := checkpoint.Validate(); err != nil {
return nil, fmt.Errorf("invalid checkpoint: %w", err)
}
if checkpoint.Header.Hash() != checkpointHash {
return nil, errors.New("wrong checkpoint hash")
}
return checkpoint, nil
}
func (api *BeaconLightApi) GetBeaconBlock(blockRoot common.Hash) (*capella.BeaconBlock, error) {
resp, err := api.httpGetf("/eth/v2/beacon/blocks/0x%x", blockRoot)
if err != nil {
return nil, err
}
var beaconBlockMessage struct {
Data struct {
Message capella.BeaconBlock `json:"message"`
} `json:"data"`
}
if err := json.Unmarshal(resp, &beaconBlockMessage); err != nil {
return nil, fmt.Errorf("invalid block json data: %v", err)
}
beaconBlock := new(capella.BeaconBlock)
*beaconBlock = beaconBlockMessage.Data.Message
root := common.Hash(beaconBlock.HashTreeRoot(configs.Mainnet, tree.GetHashFn()))
if root != blockRoot {
return nil, fmt.Errorf("Beacon block root hash mismatch (expected: %x, got: %x)", blockRoot, root)
}
return beaconBlock, nil
}
func decodeHeadEvent(enc []byte) (uint64, common.Hash, error) {
var data struct {
Slot common.Decimal `json:"slot"`
Block common.Hash `json:"block"`
}
if err := json.Unmarshal(enc, &data); err != nil {
return 0, common.Hash{}, err
}
return uint64(data.Slot), data.Block, nil
}
type HeadEventListener struct {
OnNewHead func(slot uint64, blockRoot common.Hash)
OnSignedHead func(head types.SignedHeader)
OnFinality func(head types.FinalityUpdate)
OnError func(err error)
}
// StartHeadListener creates an event subscription for heads and signed (optimistic)
// head updates and calls the specified callback functions when they are received.
// The callbacks are also called for the current head and optimistic head at startup.
// They are never called concurrently.
func (api *BeaconLightApi) StartHeadListener(listener HeadEventListener) func() {
closeCh := make(chan struct{}) // initiate closing the stream
closedCh := make(chan struct{}) // stream closed (or failed to create)
stoppedCh := make(chan struct{}) // sync loop stopped
streamCh := make(chan *eventsource.Stream, 1)
go func() {
defer close(closedCh)
// when connected to a Lodestar node the subscription blocks until the
// first actual event arrives; therefore we create the subscription in
// a separate goroutine while letting the main goroutine sync up to the
// current head
req, err := http.NewRequest("GET", api.url+
"/eth/v1/events?topics=head&topics=light_client_optimistic_update&topics=light_client_finality_update", nil)
if err != nil {
listener.OnError(fmt.Errorf("error creating event subscription request: %v", err))
return
}
for k, v := range api.customHeaders {
req.Header.Set(k, v)
}
stream, err := eventsource.SubscribeWithRequest("", req)
if err != nil {
listener.OnError(fmt.Errorf("error creating event subscription: %v", err))
close(streamCh)
return
}
streamCh <- stream
<-closeCh
stream.Close()
}()
go func() {
defer close(stoppedCh)
if head, err := api.GetHeader(common.Hash{}); err == nil {
listener.OnNewHead(head.Slot, head.Hash())
}
if signedHead, err := api.GetOptimisticHeadUpdate(); err == nil {
listener.OnSignedHead(signedHead)
}
if finalityUpdate, err := api.GetFinalityUpdate(); err == nil {
listener.OnFinality(finalityUpdate)
}
stream := <-streamCh
if stream == nil {
return
}
for {
select {
case event, ok := <-stream.Events:
if !ok {
break
}
switch event.Event() {
case "head":
if slot, blockRoot, err := decodeHeadEvent([]byte(event.Data())); err == nil {
listener.OnNewHead(slot, blockRoot)
} else {
listener.OnError(fmt.Errorf("error decoding head event: %v", err))
}
case "light_client_optimistic_update":
if signedHead, err := decodeOptimisticHeadUpdate([]byte(event.Data())); err == nil {
listener.OnSignedHead(signedHead)
} else {
listener.OnError(fmt.Errorf("error decoding optimistic update event: %v", err))
}
case "light_client_finality_update":
if finalityUpdate, err := decodeFinalityUpdate([]byte(event.Data())); err == nil {
listener.OnFinality(finalityUpdate)
} else {
listener.OnError(fmt.Errorf("error decoding finality update event: %v", err))
}
default:
listener.OnError(fmt.Errorf("unexpected event: %s", event.Event()))
}
case err, ok := <-stream.Errors:
if !ok {
break
}
listener.OnError(err)
}
}
}()
return func() {
close(closeCh)
<-closedCh
<-stoppedCh
}
}

25
beacon/light/committee_chain.go
View file

@ -70,6 +70,7 @@ type CommitteeChain struct {
committees *canonicalStore[*types.SerializedSyncCommittee]
fixedCommitteeRoots *canonicalStore[common.Hash]
committeeCache *lru.Cache[uint64, syncCommittee] // cache deserialized committees
changeCounter uint64
clock mclock.Clock // monotonic clock (simulated clock in tests)
unixNano func() int64 // system clock (simulated clock in tests)
@ -86,6 +87,11 @@ func NewCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signer
return newCommitteeChain(db, config, signerThreshold, enforceTime, blsVerifier{}, &mclock.System{}, func() int64 { return time.Now().UnixNano() })
}
// NewTestCommitteeChain creates a new CommitteeChain for testing.
func NewTestCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool, clock *mclock.Simulated) *CommitteeChain {
return newCommitteeChain(db, config, signerThreshold, enforceTime, dummyVerifier{}, clock, func() int64 { return int64(clock.Now()) })
}
// newCommitteeChain creates a new CommitteeChain with the option of replacing the
// clock source and signature verification for testing purposes.
func newCommitteeChain(db ethdb.KeyValueStore, config *types.ChainConfig, signerThreshold int, enforceTime bool, sigVerifier committeeSigVerifier, clock mclock.Clock, unixNano func() int64) *CommitteeChain {
@ -181,20 +187,20 @@ func (s *CommitteeChain) Reset() {
if err := s.rollback(0); err != nil {
log.Error("Error writing batch into chain database", "error", err)
}
s.changeCounter++
}
// CheckpointInit initializes a CommitteeChain based on the checkpoint.
// CheckpointInit initializes a CommitteeChain based on a checkpoint.
// Note: if the chain is already initialized and the committees proven by the
// checkpoint do match the existing chain then the chain is retained and the
// new checkpoint becomes fixed.
func (s *CommitteeChain) CheckpointInit(bootstrap *types.BootstrapData) error {
func (s *CommitteeChain) CheckpointInit(bootstrap types.BootstrapData) error {
s.chainmu.Lock()
defer s.chainmu.Unlock()
if err := bootstrap.Validate(); err != nil {
return err
}
period := bootstrap.Header.SyncPeriod()
if err := s.deleteFixedCommitteeRootsFrom(period + 2); err != nil {
s.Reset()
@ -215,6 +221,7 @@ func (s *CommitteeChain) CheckpointInit(bootstrap *types.BootstrapData) error {
s.Reset()
return err
}
s.changeCounter++
return nil
}
@ -367,6 +374,7 @@ func (s *CommitteeChain) InsertUpdate(update *types.LightClientUpdate, nextCommi
return ErrWrongCommitteeRoot
}
}
s.changeCounter++
if reorg {
if err := s.rollback(period + 1); err != nil {
return err
@ -405,6 +413,13 @@ func (s *CommitteeChain) NextSyncPeriod() (uint64, bool) {
return s.committees.periods.End - 1, true
}
func (s *CommitteeChain) ChangeCounter() uint64 {
s.chainmu.RLock()
defer s.chainmu.RUnlock()
return s.changeCounter
}
// rollback removes all committees and fixed roots from the given period and updates
// starting from the previous period.
func (s *CommitteeChain) rollback(period uint64) error {
@ -452,12 +467,12 @@ func (s *CommitteeChain) getSyncCommittee(period uint64) (syncCommittee, error)
if sc, ok := s.committees.get(s.db, period); ok {
c, err := s.sigVerifier.deserializeSyncCommittee(sc)
if err != nil {
return nil, fmt.Errorf("Sync committee #%d deserialization error: %v", period, err)
return nil, fmt.Errorf("sync committee #%d deserialization error: %v", period, err)
}
s.committeeCache.Add(period, c)
return c, nil
}
return nil, fmt.Errorf("Missing serialized sync committee #%d", period)
return nil, fmt.Errorf("missing serialized sync committee #%d", period)
}
// VerifySignedHeader returns true if the given signed header has a valid signature

4
beacon/light/committee_chain_test.go
View file

@ -241,12 +241,12 @@ func newCommitteeChainTest(t *testing.T, config types.ChainConfig, signerThresho
signerThreshold: signerThreshold,
enforceTime: enforceTime,
}
c.chain = newCommitteeChain(c.db, &config, signerThreshold, enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) })
c.chain = NewTestCommitteeChain(c.db, &config, signerThreshold, enforceTime, c.clock)
return c
}
func (c *committeeChainTest) reloadChain() {
c.chain = newCommitteeChain(c.db, &c.config, c.signerThreshold, c.enforceTime, dummyVerifier{}, c.clock, func() int64 { return int64(c.clock.Now()) })
c.chain = NewTestCommitteeChain(c.db, &c.config, c.signerThreshold, c.enforceTime, c.clock)
}
func (c *committeeChainTest) setClockPeriod(period float64) {

150
beacon/light/head_tracker.go Normal file
View file

@ -0,0 +1,150 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package light
import (
"errors"
"sync"
"time"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/log"
)
// HeadTracker keeps track of the latest validated head and the "prefetch" head
// which is the (not necessarily validated) head announced by the majority of
// servers.
type HeadTracker struct {
lock sync.RWMutex
committeeChain *CommitteeChain
minSignerCount int
signedHead types.SignedHeader
hasSignedHead bool
finalityUpdate types.FinalityUpdate
hasFinalityUpdate bool
prefetchHead types.HeadInfo
changeCounter uint64
}
// NewHeadTracker creates a new HeadTracker.
func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int) *HeadTracker {
return &HeadTracker{
committeeChain: committeeChain,
minSignerCount: minSignerCount,
}
}
// ValidatedHead returns the latest validated head.
func (h *HeadTracker) ValidatedHead() (types.SignedHeader, bool) {
h.lock.RLock()
defer h.lock.RUnlock()
return h.signedHead, h.hasSignedHead
}
// ValidatedHead returns the latest validated head.
func (h *HeadTracker) ValidatedFinality() (types.FinalityUpdate, bool) {
h.lock.RLock()
defer h.lock.RUnlock()
return h.finalityUpdate, h.hasFinalityUpdate
}
// Validate validates the given signed head. If the head is successfully validated
// and it is better than the old validated head (higher slot or same slot and more
// signers) then ValidatedHead is updated. The boolean return flag signals if
// ValidatedHead has been changed.
func (h *HeadTracker) ValidateHead(head types.SignedHeader) (bool, error) {
h.lock.Lock()
defer h.lock.Unlock()
replace, err := h.validate(head, h.signedHead)
if replace {
h.signedHead, h.hasSignedHead = head, true
h.changeCounter++
}
return replace, err
}
func (h *HeadTracker) ValidateFinality(update types.FinalityUpdate) (bool, error) {
h.lock.Lock()
defer h.lock.Unlock()
replace, err := h.validate(update.SignedHeader(), h.finalityUpdate.SignedHeader())
if replace {
h.finalityUpdate, h.hasFinalityUpdate = update, true
h.changeCounter++
}
return replace, err
}
func (h *HeadTracker) validate(head, oldHead types.SignedHeader) (bool, error) {
signerCount := head.Signature.SignerCount()
if signerCount < h.minSignerCount {
return false, errors.New("low signer count")
}
if head.Header.Slot < oldHead.Header.Slot || (head.Header.Slot == oldHead.Header.Slot && signerCount <= oldHead.Signature.SignerCount()) {
return false, nil
}
sigOk, age, err := h.committeeChain.VerifySignedHeader(head)
if err != nil {
return false, err
}
if age < 0 {
log.Warn("Future signed head received", "age", age)
}
if age > time.Minute*2 {
log.Warn("Old signed head received", "age", age)
}
if !sigOk {
return false, errors.New("invalid header signature")
}
return true, nil
}
// PrefetchHead returns the latest known prefetch head's head info.
// This head can be used to start fetching related data hoping that it will be
// validated soon.
// Note that the prefetch head cannot be validated cryptographically so it should
// only be used as a performance optimization hint.
func (h *HeadTracker) PrefetchHead() types.HeadInfo {
h.lock.RLock()
defer h.lock.RUnlock()
return h.prefetchHead
}
// SetPrefetchHead sets the prefetch head info.
// Note that HeadTracker does not verify the prefetch head, just acts as a thread
// safe bulletin board.
func (h *HeadTracker) SetPrefetchHead(head types.HeadInfo) {
h.lock.Lock()
defer h.lock.Unlock()
if head == h.prefetchHead {
return
}
h.prefetchHead = head
h.changeCounter++
}
func (h *HeadTracker) ChangeCounter() uint64 {
h.lock.RLock()
defer h.lock.RUnlock()
return h.changeCounter
}

401
beacon/light/request/scheduler.go Normal file
View file

@ -0,0 +1,401 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package request
import (
"sync"
"github.com/ethereum/go-ethereum/log"
)
// Module represents a mechanism which is typically responsible for downloading
// and updating a passive data structure. It does not directly interact with the
// servers. It can start requests using the Requester interface, maintain its
// internal state by receiving and processing Events and update its target data
// structure based on the obtained data.
// It is the Scheduler's responsibility to feed events to the modules, call
// Process as long as there might be something to process and then generate request
// candidates using MakeRequest and start the best possible requests.
// Modules are called by Scheduler whenever a global trigger is fired. All events
// fire the trigger. Changing a target data structure also triggers a next
// processing round as it could make further actions possible either by the same
// or another Module.
type Module interface {
// Process is a non-blocking function responsible for starting requests,
// processing events and updating the target data structures(s) and the
// internal state of the module. Module state typically consists of information
// about pending requests and registered servers.
// Process is always called after an event is received or after a target data
// structure has been changed.
//
// Note: Process functions of different modules are never called concurrently;
// they are called by Scheduler in the same order of priority as they were
// registered in.
Process(Requester, []Event)
}
// Requester allows Modules to obtain the list of momentarily available servers,
// start new requests and report server failure when a response has been proven
// to be invalid in the processing phase.
// Note that all Requester functions should be safe to call from Module.Process.
type Requester interface {
CanSendTo() []Server
Send(Server, Request) ID
Fail(Server, string)
}
// Scheduler is a modular network data retrieval framework that coordinates multiple
// servers and retrieval mechanisms (modules). It implements a trigger mechanism
// that calls the Process function of registered modules whenever either the state
// of existing data structures or events coming from registered servers could
// allow new operations.
type Scheduler struct {
lock sync.Mutex
modules []Module // first has highest priority
names map[Module]string
servers map[server]struct{}
targets map[targetData]uint64
requesterLock sync.RWMutex
serverOrder []server
pending map[ServerAndID]pendingRequest
// eventLock guards access to the events list. Note that eventLock can be
// locked either while lock is locked or unlocked but lock cannot be locked
// while eventLock is locked.
eventLock sync.Mutex
events []Event
stopCh chan chan struct{}
triggerCh chan struct{} // restarts waiting sync loop
// if trigger has already been fired then send to testWaitCh blocks until
// the triggered processing round is finished
testWaitCh chan struct{}
}
type (
// Server identifies a server without allowing any direct interaction.
// Note: server interface is used by Scheduler and Tracker but not used by
// the modules that do not interact with them directly.
// In order to make module testing easier, Server interface is used in
// events and modules.
Server any
Request any
Response any
ID uint64
ServerAndID struct {
Server Server
ID ID
}
)
// targetData represents a registered target data structure that increases its
// ChangeCounter whenever it has been changed.
type targetData interface {
ChangeCounter() uint64
}
// pendingRequest keeps track of sent and not yet finalized requests and their
// sender modules.
type pendingRequest struct {
request Request
module Module
}
// NewScheduler creates a new Scheduler.
func NewScheduler() *Scheduler {
s := &Scheduler{
servers: make(map[server]struct{}),
names: make(map[Module]string),
pending: make(map[ServerAndID]pendingRequest),
targets: make(map[targetData]uint64),
stopCh: make(chan chan struct{}),
// Note: testWaitCh should not have capacity in order to ensure
// that after a trigger happens testWaitCh will block until the resulting
// processing round has been finished
triggerCh: make(chan struct{}, 1),
testWaitCh: make(chan struct{}),
}
return s
}
// RegisterTarget registers a target data structure, ensuring that any changes
// made to it trigger a new round of Module.Process calls, giving a chance to
// modules to react to the changes.
func (s *Scheduler) RegisterTarget(t targetData) {
s.lock.Lock()
defer s.lock.Unlock()
s.targets[t] = 0
}
// RegisterModule registers a module. Should be called before starting the scheduler.
// In each processing round the order of module processing depends on the order of
// registration.
func (s *Scheduler) RegisterModule(m Module, name string) {
s.lock.Lock()
defer s.lock.Unlock()
s.modules = append(s.modules, m)
s.names[m] = name
}
// RegisterServer registers a new server.
func (s *Scheduler) RegisterServer(server server) {
s.lock.Lock()
defer s.lock.Unlock()
s.addEvent(Event{Type: EvRegistered, Server: server})
server.subscribe(func(event Event) {
event.Server = server
s.addEvent(event)
})
}
// UnregisterServer removes a registered server.
func (s *Scheduler) UnregisterServer(server server) {
s.lock.Lock()
defer s.lock.Unlock()
server.unsubscribe()
s.addEvent(Event{Type: EvUnregistered, Server: server})
}
// Start starts the scheduler. It should be called after registering all modules
// and before registering any servers.
func (s *Scheduler) Start() {
go s.syncLoop()
}
// Stop stops the scheduler.
func (s *Scheduler) Stop() {
stop := make(chan struct{})
s.stopCh <- stop
<-stop
s.lock.Lock()
for server := range s.servers {
server.unsubscribe()
}
s.servers = nil
s.lock.Unlock()
}
// syncLoop is the main event loop responsible for event/data processing and
// sending new requests.
// A round of processing starts whenever the global trigger is fired. Triggers
// fired during a processing round ensure that there is going to be a next round.
func (s *Scheduler) syncLoop() {
for {
s.lock.Lock()
s.processRound()
s.lock.Unlock()
loop:
for {
select {
case stop := <-s.stopCh:
close(stop)
return
case <-s.triggerCh:
break loop
case <-s.testWaitCh:
}
}
}
}
// targetChanged returns true if a registered target data structure has been
// changed since the last call to this function.
func (s *Scheduler) targetChanged() (changed bool) {
for target, counter := range s.targets {
if newCounter := target.ChangeCounter(); newCounter != counter {
s.targets[target] = newCounter
changed = true
}
}
return
}
// processRound runs an entire processing round. It calls the Process functions
// of all modules, passing all relevant events and repeating Process calls as
// long as any changes have been made to the registered target data structures.
// Once all events have been processed and a stable state has been achieved,
// requests are generated and sent if necessary and possible.
func (s *Scheduler) processRound() {
for {
log.Trace("Processing modules")
filteredEvents := s.filterEvents()
for _, module := range s.modules {
log.Trace("Processing module", "name", s.names[module], "events", len(filteredEvents[module]))
module.Process(requester{s, module}, filteredEvents[module])
}
if !s.targetChanged() {
break
}
}
}
// Trigger starts a new processing round. If fired during processing, it ensures
// another full round of processing all modules.
func (s *Scheduler) Trigger() {
select {
case s.triggerCh <- struct{}{}:
default:
}
}
// addEvent adds an event to be processed in the next round. Note that it can be
// called regardless of the state of the lock mutex, making it safe for use in
// the server event callback.
func (s *Scheduler) addEvent(event Event) {
s.eventLock.Lock()
s.events = append(s.events, event)
s.eventLock.Unlock()
s.Trigger()
}
// filterEvent sorts each Event either as a request event or a server event,
// depending on its type. Request events are also sorted in a map based on the
// module that originally initiated the request. It also ensures that no events
// related to a server are returned before EvRegistered or after EvUnregistered.
// In case of an EvUnregistered server event it also closes all pending requests
// to the given server by adding a failed request event (EvFail), ensuring that
// all requests get finalized and thereby allowing the module logic to be safe
// and simple.
func (s *Scheduler) filterEvents() map[Module][]Event {
s.eventLock.Lock()
events := s.events
s.events = nil
s.eventLock.Unlock()
s.requesterLock.Lock()
defer s.requesterLock.Unlock()
filteredEvents := make(map[Module][]Event)
for _, event := range events {
server := event.Server.(server)
if _, ok := s.servers[server]; !ok && event.Type != EvRegistered {
continue // before EvRegister or after EvUnregister, discard
}
if event.IsRequestEvent() {
sid, _, _ := event.RequestInfo()
pending, ok := s.pending[sid]
if !ok {
continue // request already closed, ignore further events
}
if event.Type == EvResponse || event.Type == EvFail {
delete(s.pending, sid) // final event, close pending request
}
filteredEvents[pending.module] = append(filteredEvents[pending.module], event)
} else {
switch event.Type {
case EvRegistered:
s.servers[server] = struct{}{}
s.serverOrder = append(s.serverOrder, nil)
copy(s.serverOrder[1:], s.serverOrder[:len(s.serverOrder)-1])
s.serverOrder[0] = server
case EvUnregistered:
s.closePending(event.Server, filteredEvents)
delete(s.servers, server)
for i, srv := range s.serverOrder {
if srv == server {
copy(s.serverOrder[i:len(s.serverOrder)-1], s.serverOrder[i+1:])
s.serverOrder = s.serverOrder[:len(s.serverOrder)-1]
break
}
}
}
for _, module := range s.modules {
filteredEvents[module] = append(filteredEvents[module], event)
}
}
}
return filteredEvents
}
// closePending closes all pending requests to the given server and adds an EvFail
// event to properly finalize them
func (s *Scheduler) closePending(server Server, filteredEvents map[Module][]Event) {
for sid, pending := range s.pending {
if sid.Server == server {
filteredEvents[pending.module] = append(filteredEvents[pending.module], Event{
Type: EvFail,
Server: server,
Data: RequestResponse{
ID: sid.ID,
Request: pending.request,
},
})
delete(s.pending, sid)
}
}
}
// requester implements Requester. Note that while requester basically wraps
// Scheduler (with the added information of the currently processed Module), all
// functions are safe to call from Module.Process which is running while
// the Scheduler.lock mutex is held.
type requester struct {
*Scheduler
module Module
}
// CanSendTo returns the list of currently available servers. It also returns
// them in an order of least to most recently used, ensuring a round-robin usage
// of suitable servers if the module always chooses the first suitable one.
func (s requester) CanSendTo() []Server {
s.requesterLock.RLock()
defer s.requesterLock.RUnlock()
list := make([]Server, 0, len(s.serverOrder))
for _, server := range s.serverOrder {
if server.canRequestNow() {
list = append(list, server)
}
}
return list
}
// Send sends a request and adds an entry to Scheduler.pending map, ensuring that
// related request events will be delivered to the sender Module.
func (s requester) Send(srv Server, req Request) ID {
s.requesterLock.Lock()
defer s.requesterLock.Unlock()
server := srv.(server)
id := server.sendRequest(req)
sid := ServerAndID{Server: srv, ID: id}
s.pending[sid] = pendingRequest{request: req, module: s.module}
for i, ss := range s.serverOrder {
if ss == server {
copy(s.serverOrder[i:len(s.serverOrder)-1], s.serverOrder[i+1:])
s.serverOrder[len(s.serverOrder)-1] = server
return id
}
}
log.Error("Target server not found in ordered list of registered servers")
return id
}
// Fail should be called when a server delivers invalid or useless information.
// Calling Fail disables the given server for a period that is initially short
// but is exponentially growing if it happens frequently. This results in a
// somewhat fault tolerant operation that avoids hammering servers with requests
// that they cannot serve but still gives them a chance periodically.
func (s requester) Fail(srv Server, desc string) {
srv.(server).fail(desc)
}

122
beacon/light/request/scheduler_test.go Normal file
View file

@ -0,0 +1,122 @@
package request
import (
"reflect"
"testing"
)
func TestEventFilter(t *testing.T) {
s := NewScheduler()
module1 := &testModule{name: "module1"}
module2 := &testModule{name: "module2"}
s.RegisterModule(module1, "module1")
s.RegisterModule(module2, "module2")
s.Start()
// startup process round without events
s.testWaitCh <- struct{}{}
module1.expProcess(t, nil)
module2.expProcess(t, nil)
srv := &testServer{}
// register server; both modules should receive server event
s.RegisterServer(srv)
s.testWaitCh <- struct{}{}
module1.expProcess(t, []Event{
{Type: EvRegistered, Server: srv},
})
module2.expProcess(t, []Event{
{Type: EvRegistered, Server: srv},
})
// let module1 send a request
srv.canRequest = 1
module1.sendReq = testRequest
s.Trigger()
// in first triggered round module1 sends the request, no events yet
s.testWaitCh <- struct{}{}
module1.expProcess(t, nil)
module2.expProcess(t, nil)
// server emits EvTimeout; only module1 should receive it
srv.eventCb(Event{Type: EvTimeout, Data: RequestResponse{ID: 1, Request: testRequest}})
s.testWaitCh <- struct{}{}
module1.expProcess(t, []Event{
{Type: EvTimeout, Server: srv, Data: RequestResponse{ID: 1, Request: testRequest}},
})
module2.expProcess(t, nil)
// unregister server; both modules should receive server event
s.UnregisterServer(srv)
s.testWaitCh <- struct{}{}
module1.expProcess(t, []Event{
// module1 should also receive EvFail on its pending request
{Type: EvFail, Server: srv, Data: RequestResponse{ID: 1, Request: testRequest}},
{Type: EvUnregistered, Server: srv},
})
module2.expProcess(t, []Event{
{Type: EvUnregistered, Server: srv},
})
// response after server unregistered; should be discarded
srv.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}})
s.testWaitCh <- struct{}{}
module1.expProcess(t, nil)
module2.expProcess(t, nil)
// no more process rounds expected; shut down
s.testWaitCh <- struct{}{}
module1.expNoMoreProcess(t)
module2.expNoMoreProcess(t)
s.Stop()
}
type testServer struct {
eventCb func(Event)
lastID ID
canRequest int
}
func (s *testServer) subscribe(eventCb func(Event)) {
s.eventCb = eventCb
}
func (s *testServer) canRequestNow() bool {
return s.canRequest > 0
}
func (s *testServer) sendRequest(req Request) ID {
s.canRequest--
s.lastID++
return s.lastID
}
func (s *testServer) fail(string) {}
func (s *testServer) unsubscribe() {}
type testModule struct {
name string
processed [][]Event
sendReq Request
}
func (m *testModule) Process(requester Requester, events []Event) {
m.processed = append(m.processed, events)
if m.sendReq != nil {
if cs := requester.CanSendTo(); len(cs) > 0 {
requester.Send(cs[0], m.sendReq)
}
}
}
func (m *testModule) expProcess(t *testing.T, expEvents []Event) {
if len(m.processed) == 0 {
t.Errorf("Missing call to %s.Process", m.name)
return
}
events := m.processed[0]
m.processed = m.processed[1:]
if !reflect.DeepEqual(events, expEvents) {
t.Errorf("Call to %s.Process with wrong events (expected %v, got %v)", m.name, expEvents, events)
}
}
func (m *testModule) expNoMoreProcess(t *testing.T) {
for len(m.processed) > 0 {
t.Errorf("Unexpected call to %s.Process with events %v", m.name, m.processed[0])
m.processed = m.processed[1:]
}
}

439
beacon/light/request/server.go Normal file
View file

@ -0,0 +1,439 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package request
import (
"math"
"sync"
"time"
"github.com/ethereum/go-ethereum/common/mclock"
"github.com/ethereum/go-ethereum/log"
)
var (
// request events
EvResponse = &EventType{Name: "response", requestEvent: true} // data: RequestResponse; sent by requestServer
EvFail = &EventType{Name: "fail", requestEvent: true} // data: RequestResponse; sent by requestServer
EvTimeout = &EventType{Name: "timeout", requestEvent: true} // data: RequestResponse; sent by serverWithTimeout
// server events
EvRegistered = &EventType{Name: "registered"} // data: nil; sent by Scheduler
EvUnregistered = &EventType{Name: "unregistered"} // data: nil; sent by Scheduler
EvCanRequestAgain = &EventType{Name: "canRequestAgain"} // data: nil; sent by serverWithLimits
)
const (
softRequestTimeout = time.Second // allow resending request to a different server but do not cancel yet
hardRequestTimeout = time.Second * 10 // cancel request
)
const (
// serverWithLimits parameters
parallelAdjustUp = 0.1 // adjust parallelLimit up in case of success under full load
parallelAdjustDown = 1 // adjust parallelLimit down in case of timeout/failure
minParallelLimit = 1 // parallelLimit lower bound
defaultParallelLimit = 3 // parallelLimit initial value
minFailureDelay = time.Millisecond * 100 // minimum disable time in case of request failure
maxFailureDelay = time.Minute // maximum disable time in case of request failure
maxServerEventBuffer = 5 // server event allowance buffer limit
maxServerEventRate = time.Second // server event allowance buffer recharge rate
)
// requestServer can send requests in a non-blocking way and feed back events
// through the event callback. After each request it should send back either
// EvResponse or EvFail. Additionally, it may also send application-defined
// events that the Modules can interpret.
type requestServer interface {
Subscribe(eventCallback func(Event))
SendRequest(ID, Request)
Unsubscribe()
}
// server is implemented by a requestServer wrapped into serverWithTimeout and
// serverWithLimits and is used by Scheduler.
// In addition to requestServer functionality, server can also handle timeouts,
// limit the number of parallel in-flight requests and temporarily disable
// new requests based on timeouts and response failures.
type server interface {
subscribe(eventCallback func(Event))
canRequestNow() bool
sendRequest(Request) ID
fail(string)
unsubscribe()
}
// NewServer wraps a requestServer and returns a server
func NewServer(rs requestServer, clock mclock.Clock) server {
s := &serverWithLimits{}
s.parent = rs
s.serverWithTimeout.init(clock)
s.init()
return s
}
// EventType identifies an event type, either related to a request or the server
// in general. Server events can also be externally defined.
type EventType struct {
Name string
requestEvent bool // all request events are pre-defined in request package
}
// Event describes an event where the type of Data depends on Type.
// Server field is not required when sent through the event callback; it is filled
// out when processed by the Scheduler. Note that the Scheduler can also create
// and send events (EvRegistered, EvUnregistered) directly.
type Event struct {
Type *EventType
Server Server // filled by Scheduler
Data any
}
// IsRequestEvent returns true if the event is a request event
func (e *Event) IsRequestEvent() bool {
return e.Type.requestEvent
}
// RequestInfo assumes that the event is a request event and returns its contents
// in a convenient form.
func (e *Event) RequestInfo() (ServerAndID, Request, Response) {
data := e.Data.(RequestResponse)
return ServerAndID{Server: e.Server, ID: data.ID}, data.Request, data.Response
}
// RequestResponse is the Data type of request events.
type RequestResponse struct {
ID ID
Request Request
Response Response
}
// serverWithTimeout wraps a requestServer and introduces timeouts.
// The request's lifecycle is concluded if EvResponse or EvFail emitted by the
// parent requestServer. If this does not happen until softRequestTimeout then
// EvTimeout is emitted, after which the final EvResponse or EvFail is still
// guaranteed to follow.
// If the parent fails to send this final event for hardRequestTimeout then
// serverWithTimeout emits EvFail and discards any further events from the
// parent related to the given request.
type serverWithTimeout struct {
parent requestServer
lock sync.Mutex
clock mclock.Clock
childEventCb func(event Event)
timeouts map[ID]mclock.Timer
lastID ID
}
// init initializes serverWithTimeout
func (s *serverWithTimeout) init(clock mclock.Clock) {
s.clock = clock
s.timeouts = make(map[ID]mclock.Timer)
}
// subscribe subscribes to events which include parent (requestServer) events
// plus EvTimeout.
func (s *serverWithTimeout) subscribe(eventCallback func(event Event)) {
s.lock.Lock()
defer s.lock.Unlock()
s.childEventCb = eventCallback
s.parent.Subscribe(s.eventCallback)
}
// sendRequest generated a new request ID, emits EvRequest, sets up the timeout
// timer, then sends the request through the parent (requestServer).
func (s *serverWithTimeout) sendRequest(request Request) (reqId ID) {
s.lock.Lock()
s.lastID++
id := s.lastID
s.startTimeout(RequestResponse{ID: id, Request: request})
s.lock.Unlock()
s.parent.SendRequest(id, request)
return id
}
// eventCallback is called by parent (requestServer) event subscription.
func (s *serverWithTimeout) eventCallback(event Event) {
s.lock.Lock()
defer s.lock.Unlock()
switch event.Type {
case EvResponse, EvFail:
id := event.Data.(RequestResponse).ID
if timer, ok := s.timeouts[id]; ok {
// Note: if stopping the timer is unsuccessful then the resulting AfterFunc
// call will just do nothing
timer.Stop()
delete(s.timeouts, id)
s.childEventCb(event)
}
default:
s.childEventCb(event)
}
}
// startTimeout starts a timeout timer for the given request.
func (s *serverWithTimeout) startTimeout(reqData RequestResponse) {
id := reqData.ID
s.timeouts[id] = s.clock.AfterFunc(softRequestTimeout, func() {
s.lock.Lock()
if _, ok := s.timeouts[id]; !ok {
s.lock.Unlock()
return
}
s.timeouts[id] = s.clock.AfterFunc(hardRequestTimeout-softRequestTimeout, func() {
s.lock.Lock()
if _, ok := s.timeouts[id]; !ok {
s.lock.Unlock()
return
}
delete(s.timeouts, id)
childEventCb := s.childEventCb
s.lock.Unlock()
childEventCb(Event{Type: EvFail, Data: reqData})
})
childEventCb := s.childEventCb
s.lock.Unlock()
childEventCb(Event{Type: EvTimeout, Data: reqData})
})
}
// stop stops all goroutines associated with the server.
func (s *serverWithTimeout) unsubscribe() {
s.lock.Lock()
defer s.lock.Unlock()
for _, timer := range s.timeouts {
if timer != nil {
timer.Stop()
}
}
s.childEventCb = nil
s.parent.Unsubscribe()
}
// serverWithLimits wraps serverWithTimeout and implements server. It limits the
// number of parallel in-flight requests and prevents sending new requests when a
// pending one has already timed out. Server events are also rate limited.
// It also implements a failure delay mechanism that adds an exponentially growing
// delay each time a request fails (wrong answer or hard timeout). This makes the
// syncing mechanism less brittle as temporary failures of the server might happen
// sometimes, but still avoids hammering a non-functional server with requests.
type serverWithLimits struct {
serverWithTimeout
lock sync.Mutex
childEventCb func(event Event)
softTimeouts map[ID]struct{}
pendingCount, timeoutCount int
parallelLimit float32
sendEvent bool
delayTimer mclock.Timer
delayCounter int
failureDelayEnd mclock.AbsTime
failureDelay float64
serverEventBuffer int
eventBufferUpdated mclock.AbsTime
}
// init initializes serverWithLimits
func (s *serverWithLimits) init() {
s.softTimeouts = make(map[ID]struct{})
s.parallelLimit = defaultParallelLimit
s.serverEventBuffer = maxServerEventBuffer
}
// subscribe subscribes to events which include parent (serverWithTimeout) events
// plus EvCanRequstAgain.
func (s *serverWithLimits) subscribe(eventCallback func(event Event)) {
s.lock.Lock()
defer s.lock.Unlock()
s.childEventCb = eventCallback
s.serverWithTimeout.subscribe(s.eventCallback)
}
// eventCallback is called by parent (serverWithTimeout) event subscription.
func (s *serverWithLimits) eventCallback(event Event) {
s.lock.Lock()
var sendCanRequestAgain bool
passEvent := true
switch event.Type {
case EvTimeout:
id := event.Data.(RequestResponse).ID
s.softTimeouts[id] = struct{}{}
s.timeoutCount++
s.parallelLimit -= parallelAdjustDown
if s.parallelLimit < minParallelLimit {
s.parallelLimit = minParallelLimit
}
log.Debug("Server timeout", "count", s.timeoutCount, "parallelLimit", s.parallelLimit)
case EvResponse, EvFail:
id := event.Data.(RequestResponse).ID
if _, ok := s.softTimeouts[id]; ok {
delete(s.softTimeouts, id)
s.timeoutCount--
log.Debug("Server timeout finalized", "count", s.timeoutCount, "parallelLimit", s.parallelLimit)
}
if event.Type == EvResponse && s.pendingCount >= int(s.parallelLimit) {
s.parallelLimit += parallelAdjustUp
}
s.pendingCount--
if s.canRequest() {
sendCanRequestAgain = s.sendEvent
s.sendEvent = false
}
if event.Type == EvFail {
s.failLocked("failed request")
}
default:
// server event; check rate limit
if s.serverEventBuffer < maxServerEventBuffer {
now := s.clock.Now()
sinceUpdate := time.Duration(now - s.eventBufferUpdated)
if sinceUpdate >= maxServerEventRate*time.Duration(maxServerEventBuffer-s.serverEventBuffer) {
s.serverEventBuffer = maxServerEventBuffer
s.eventBufferUpdated = now
} else {
addBuffer := int(sinceUpdate / maxServerEventRate)
s.serverEventBuffer += addBuffer
s.eventBufferUpdated += mclock.AbsTime(maxServerEventRate * time.Duration(addBuffer))
}
}
if s.serverEventBuffer > 0 {
s.serverEventBuffer--
} else {
passEvent = false
}
}
childEventCb := s.childEventCb
s.lock.Unlock()
if passEvent {
childEventCb(event)
}
if sendCanRequestAgain {
childEventCb(Event{Type: EvCanRequestAgain})
}
}
// sendRequest sends a request through the parent (serverWithTimeout).
func (s *serverWithLimits) sendRequest(request Request) (reqId ID) {
s.lock.Lock()
s.pendingCount++
s.lock.Unlock()
return s.serverWithTimeout.sendRequest(request)
}
// stop stops all goroutines associated with the server.
func (s *serverWithLimits) unsubscribe() {
s.lock.Lock()
defer s.lock.Unlock()
if s.delayTimer != nil {
s.delayTimer.Stop()
s.delayTimer = nil
}
s.childEventCb = nil
s.serverWithTimeout.unsubscribe()
}
// canRequest checks whether a new request can be started.
func (s *serverWithLimits) canRequest() bool {
if s.delayTimer != nil || s.pendingCount >= int(s.parallelLimit) || s.timeoutCount > 0 {
return false
}
if s.parallelLimit < minParallelLimit {
s.parallelLimit = minParallelLimit
}
return true
}
// canRequestNow checks whether a new request can be started, according to the
// current in-flight request count and parallelLimit, and also the failure delay
// timer.
// If it returns false then it is guaranteed that an EvCanRequestAgain will be
// sent whenever the server becomes available for requesting again.
func (s *serverWithLimits) canRequestNow() bool {
var sendCanRequestAgain bool
s.lock.Lock()
canRequest := s.canRequest()
if canRequest {
sendCanRequestAgain = s.sendEvent
s.sendEvent = false
}
childEventCb := s.childEventCb
s.lock.Unlock()
if sendCanRequestAgain {
childEventCb(Event{Type: EvCanRequestAgain})
}
return canRequest
}
// delay sets the delay timer to the given duration, disabling new requests for
// the given period.
func (s *serverWithLimits) delay(delay time.Duration) {
if s.delayTimer != nil {
// Note: if stopping the timer is unsuccessful then the resulting AfterFunc
// call will just do nothing
s.delayTimer.Stop()
s.delayTimer = nil
}
s.delayCounter++
delayCounter := s.delayCounter
log.Debug("Server delay started", "length", delay)
s.delayTimer = s.clock.AfterFunc(delay, func() {
log.Debug("Server delay ended", "length", delay)
var sendCanRequestAgain bool
s.lock.Lock()
if s.delayTimer != nil && s.delayCounter == delayCounter { // do nothing if there is a new timer now
s.delayTimer = nil
if s.canRequest() {
sendCanRequestAgain = s.sendEvent
s.sendEvent = false
}
}
childEventCb := s.childEventCb
s.lock.Unlock()
if sendCanRequestAgain {
childEventCb(Event{Type: EvCanRequestAgain})
}
})
}
// fail reports that a response from the server was found invalid by the processing
// Module, disabling new requests for a dynamically adjused time period.
func (s *serverWithLimits) fail(desc string) {
s.lock.Lock()
defer s.lock.Unlock()
s.failLocked(desc)
}
// failLocked calculates the dynamic failure delay and applies it.
func (s *serverWithLimits) failLocked(desc string) {
log.Debug("Server error", "description", desc)
s.failureDelay *= 2
now := s.clock.Now()
if now > s.failureDelayEnd {
s.failureDelay *= math.Pow(2, -float64(now-s.failureDelayEnd)/float64(maxFailureDelay))
}
if s.failureDelay < float64(minFailureDelay) {
s.failureDelay = float64(minFailureDelay)
}
s.failureDelayEnd = now + mclock.AbsTime(s.failureDelay)
s.delay(time.Duration(s.failureDelay))
}

158
beacon/light/request/server_test.go Normal file
View file

@ -0,0 +1,158 @@
package request
import (
"testing"
"github.com/ethereum/go-ethereum/common/mclock"
)
const (
testRequest = "Life, the Universe, and Everything"
testResponse = 42
)
var testEventType = &EventType{Name: "testEvent"}
func TestServerEvents(t *testing.T) {
rs := &testRequestServer{}
clock := &mclock.Simulated{}
srv := NewServer(rs, clock)
var lastEventType *EventType
srv.subscribe(func(event Event) { lastEventType = event.Type })
evTypeName := func(evType *EventType) string {
if evType == nil {
return "none"
}
return evType.Name
}
expEvent := func(expType *EventType) {
if lastEventType != expType {
t.Errorf("Wrong event type (expected %s, got %s)", evTypeName(expType), evTypeName(lastEventType))
}
lastEventType = nil
}
// user events should simply be passed through
rs.eventCb(Event{Type: testEventType})
expEvent(testEventType)
// send request, soft timeout, then valid response
srv.sendRequest(testRequest)
clock.WaitForTimers(1)
clock.Run(softRequestTimeout)
expEvent(EvTimeout)
rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}})
expEvent(EvResponse)
// send request, hard timeout (response after hard timeout should be ignored)
srv.sendRequest(testRequest)
clock.WaitForTimers(1)
clock.Run(softRequestTimeout)
expEvent(EvTimeout)
clock.WaitForTimers(1)
clock.Run(hardRequestTimeout)
expEvent(EvFail)
rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}})
expEvent(nil)
}
func TestServerParallel(t *testing.T) {
rs := &testRequestServer{}
srv := NewServer(rs, &mclock.Simulated{})
srv.subscribe(func(event Event) {})
expSend := func(expSent int) {
var sent int
for sent <= expSent {
if !srv.canRequestNow() {
break
}
sent++
srv.sendRequest(testRequest)
}
if sent != expSent {
t.Errorf("Wrong number of parallel requests accepted (expected %d, got %d)", expSent, sent)
}
}
// max out parallel allowance
expSend(defaultParallelLimit)
// 1 answered, should accept 1 more
rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}})
expSend(1)
// 2 answered, should accept 2 more
rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 2, Request: testRequest, Response: testResponse}})
rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 3, Request: testRequest, Response: testResponse}})
expSend(2)
// failed request, should decrease allowance and not accept more
rs.eventCb(Event{Type: EvFail, Data: RequestResponse{ID: 4, Request: testRequest}})
expSend(0)
srv.unsubscribe()
}
func TestServerFail(t *testing.T) {
rs := &testRequestServer{}
clock := &mclock.Simulated{}
srv := NewServer(rs, clock)
srv.subscribe(func(event Event) {})
expCanRequest := func(expCanRequest bool) {
if canRequest := srv.canRequestNow(); canRequest != expCanRequest {
t.Errorf("Wrong result for canRequestNow (expected %v, got %v)", expCanRequest, canRequest)
}
}
// timed out request
expCanRequest(true)
srv.sendRequest(testRequest)
clock.WaitForTimers(1)
expCanRequest(true)
clock.Run(softRequestTimeout)
expCanRequest(false) // cannot request when there is a timed out request
rs.eventCb(Event{Type: EvResponse, Data: RequestResponse{ID: 1, Request: testRequest, Response: testResponse}})
expCanRequest(true)
// explicit server.Fail
srv.fail("")
clock.WaitForTimers(1)
expCanRequest(false) // cannot request for a while after a failure
clock.Run(minFailureDelay)
expCanRequest(true)
// request returned with EvFail
srv.sendRequest(testRequest)
rs.eventCb(Event{Type: EvFail, Data: RequestResponse{ID: 2, Request: testRequest}})
clock.WaitForTimers(1)
expCanRequest(false) // EvFail should also start failure delay
clock.Run(minFailureDelay)
expCanRequest(false) // second failure delay is longer, should still be disabled
clock.Run(minFailureDelay)
expCanRequest(true)
srv.unsubscribe()
}
func TestServerEventRateLimit(t *testing.T) {
rs := &testRequestServer{}
clock := &mclock.Simulated{}
srv := NewServer(rs, clock)
var eventCount int
srv.subscribe(func(event Event) {
if !event.IsRequestEvent() {
eventCount++
}
})
expEvents := func(send, expAllowed int) {
eventCount = 0
for sent := 0; sent < send; sent++ {
rs.eventCb(Event{Type: testEventType})
}
if eventCount != expAllowed {
t.Errorf("Wrong number of server events passing rate limitation (sent %d, expected %d, got %d)", send, expAllowed, eventCount)
}
}
expEvents(maxServerEventBuffer+5, maxServerEventBuffer)
clock.Run(maxServerEventRate)
expEvents(5, 1)
clock.Run(maxServerEventRate * maxServerEventBuffer * 2)
expEvents(maxServerEventBuffer+5, maxServerEventBuffer)
}
type testRequestServer struct {
eventCb func(Event)
}
func (rs *testRequestServer) Subscribe(eventCb func(Event)) { rs.eventCb = eventCb }
func (rs *testRequestServer) SendRequest(ID, Request) {}
func (rs *testRequestServer) Unsubscribe() {}

176
beacon/light/sync/head_sync.go Normal file
View file

@ -0,0 +1,176 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package sync
import (
"github.com/ethereum/go-ethereum/beacon/light/request"
"github.com/ethereum/go-ethereum/beacon/types"
)
type headTracker interface {
ValidateHead(head types.SignedHeader) (bool, error)
ValidateFinality(head types.FinalityUpdate) (bool, error)
SetPrefetchHead(head types.HeadInfo)
}
// HeadSync implements request.Module; it updates the validated and prefetch
// heads of HeadTracker based on the EvHead and EvSignedHead events coming from
// registered servers.
// It can also postpone the validation of the latest announced signed head
// until the committee chain is synced up to at least the required period.
type HeadSync struct {
headTracker headTracker
chain committeeChain
nextSyncPeriod uint64
chainInit bool
unvalidatedHeads map[request.Server]types.SignedHeader
unvalidatedFinality map[request.Server]types.FinalityUpdate
serverHeads map[request.Server]types.HeadInfo
headServerCount map[types.HeadInfo]headServerCount
headCounter uint64
prefetchHead types.HeadInfo
}
// headServerCount is associated with most recently seen head infos; it counts
// the number of servers currently having the given head info as their announced
// head and a counter signaling how recent that head is.
// This data is used for selecting the prefetch head.
type headServerCount struct {
serverCount int
headCounter uint64
}
// NewHeadSync creates a new HeadSync.
func NewHeadSync(headTracker headTracker, chain committeeChain) *HeadSync {
s := &HeadSync{
headTracker: headTracker,
chain: chain,
unvalidatedHeads: make(map[request.Server]types.SignedHeader),
unvalidatedFinality: make(map[request.Server]types.FinalityUpdate),
serverHeads: make(map[request.Server]types.HeadInfo),
headServerCount: make(map[types.HeadInfo]headServerCount),
}
return s
}
// Process implements request.Module.
func (s *HeadSync) Process(requester request.Requester, events []request.Event) {
for _, event := range events {
switch event.Type {
case EvNewHead:
s.setServerHead(event.Server, event.Data.(types.HeadInfo))
case EvNewSignedHead:
s.newSignedHead(event.Server, event.Data.(types.SignedHeader))
case EvNewFinalityUpdate:
s.newFinalityUpdate(event.Server, event.Data.(types.FinalityUpdate))
case request.EvUnregistered:
s.setServerHead(event.Server, types.HeadInfo{})
delete(s.serverHeads, event.Server)
delete(s.unvalidatedHeads, event.Server)
}
}
nextPeriod, chainInit := s.chain.NextSyncPeriod()
if nextPeriod != s.nextSyncPeriod || chainInit != s.chainInit {
s.nextSyncPeriod, s.chainInit = nextPeriod, chainInit
s.processUnvalidated()
}
}
// newSignedHead handles received signed head; either validates it if the chain
// is properly synced or stores it for further validation.
func (s *HeadSync) newSignedHead(server request.Server, signedHead types.SignedHeader) {
if !s.chainInit || types.SyncPeriod(signedHead.SignatureSlot) > s.nextSyncPeriod {
s.unvalidatedHeads[server] = signedHead
return
}
s.headTracker.ValidateHead(signedHead)
}
// newSignedHead handles received signed head; either validates it if the chain
// is properly synced or stores it for further validation.
func (s *HeadSync) newFinalityUpdate(server request.Server, finalityUpdate types.FinalityUpdate) {
if !s.chainInit || types.SyncPeriod(finalityUpdate.SignatureSlot) > s.nextSyncPeriod {
s.unvalidatedFinality[server] = finalityUpdate
return
}
s.headTracker.ValidateFinality(finalityUpdate)
}
// processUnvalidatedHeads iterates the list of unvalidated heads and validates
// those which can be validated.
func (s *HeadSync) processUnvalidated() {
if !s.chainInit {
return
}
for server, signedHead := range s.unvalidatedHeads {
if types.SyncPeriod(signedHead.SignatureSlot) <= s.nextSyncPeriod {
s.headTracker.ValidateHead(signedHead)
delete(s.unvalidatedHeads, server)
}
}
for server, finalityUpdate := range s.unvalidatedFinality {
if types.SyncPeriod(finalityUpdate.SignatureSlot) <= s.nextSyncPeriod {
s.headTracker.ValidateFinality(finalityUpdate)
delete(s.unvalidatedFinality, server)
}
}
}
// setServerHead processes non-validated server head announcements and updates
// the prefetch head if necessary.
func (s *HeadSync) setServerHead(server request.Server, head types.HeadInfo) bool {
if oldHead, ok := s.serverHeads[server]; ok {
if head == oldHead {
return false
}
h := s.headServerCount[oldHead]
if h.serverCount--; h.serverCount > 0 {
s.headServerCount[oldHead] = h
} else {
delete(s.headServerCount, oldHead)
}
}
if head != (types.HeadInfo{}) {
h, ok := s.headServerCount[head]
if !ok {
s.headCounter++
h.headCounter = s.headCounter
}
h.serverCount++
s.headServerCount[head] = h
s.serverHeads[server] = head
} else {
delete(s.serverHeads, server)
}
var (
bestHead types.HeadInfo
bestHeadInfo headServerCount
)
for head, headServerCount := range s.headServerCount {
if headServerCount.serverCount > bestHeadInfo.serverCount ||
(headServerCount.serverCount == bestHeadInfo.serverCount && headServerCount.headCounter > bestHeadInfo.headCounter) {
bestHead, bestHeadInfo = head, headServerCount
}
}
if bestHead == s.prefetchHead {
return false
}
s.prefetchHead = bestHead
s.headTracker.SetPrefetchHead(bestHead)
return true
}

151
beacon/light/sync/head_sync_test.go Normal file
View file

@ -0,0 +1,151 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package sync
import (
"testing"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
)
var (
testServer1 = "testServer1"
testServer2 = "testServer2"
testServer3 = "testServer3"
testServer4 = "testServer4"
testHead0 = types.HeadInfo{}
testHead1 = types.HeadInfo{Slot: 123, BlockRoot: common.Hash{1}}
testHead2 = types.HeadInfo{Slot: 124, BlockRoot: common.Hash{2}}
testHead3 = types.HeadInfo{Slot: 124, BlockRoot: common.Hash{3}}
testHead4 = types.HeadInfo{Slot: 125, BlockRoot: common.Hash{4}}
testSHead1 = types.SignedHeader{SignatureSlot: 0x0124, Header: types.Header{Slot: 0x0123, StateRoot: common.Hash{1}}}
testSHead2 = types.SignedHeader{SignatureSlot: 0x2010, Header: types.Header{Slot: 0x200e, StateRoot: common.Hash{2}}}
// testSHead3 is at the end of period 1 but signed in period 2
testSHead3 = types.SignedHeader{SignatureSlot: 0x4000, Header: types.Header{Slot: 0x3fff, StateRoot: common.Hash{3}}}
testSHead4 = types.SignedHeader{SignatureSlot: 0x6444, Header: types.Header{Slot: 0x6443, StateRoot: common.Hash{4}}}
)
func TestValidatedHead(t *testing.T) {
chain := &TestCommitteeChain{}
ht := &TestHeadTracker{}
headSync := NewHeadSync(ht, chain)
ts := NewTestScheduler(t, headSync)
ht.ExpValidated(t, 0, nil)
ts.AddServer(testServer1, 1)
ts.ServerEvent(EvNewSignedHead, testServer1, testSHead1)
ts.Run(1)
// announced head should be queued because of uninitialized chain
ht.ExpValidated(t, 1, nil)
chain.SetNextSyncPeriod(0) // initialize chain
ts.Run(2)
// expect previously queued head to be validated
ht.ExpValidated(t, 2, []types.SignedHeader{testSHead1})
chain.SetNextSyncPeriod(1)
ts.ServerEvent(EvNewSignedHead, testServer1, testSHead2)
ts.AddServer(testServer2, 1)
ts.ServerEvent(EvNewSignedHead, testServer2, testSHead2)
ts.Run(3)
// expect both head announcements to be validated instantly
ht.ExpValidated(t, 3, []types.SignedHeader{testSHead2, testSHead2})
ts.ServerEvent(EvNewSignedHead, testServer1, testSHead3)
ts.AddServer(testServer3, 1)
ts.ServerEvent(EvNewSignedHead, testServer3, testSHead4)
ts.Run(4)
// future period annonced heads should be queued
ht.ExpValidated(t, 4, nil)
chain.SetNextSyncPeriod(2)
ts.Run(5)
// testSHead3 can be validated now but not testSHead4
ht.ExpValidated(t, 5, []types.SignedHeader{testSHead3})
// server 3 disconnected without proving period 3, its announced head should be dropped
ts.RemoveServer(testServer3)
ts.Run(6)
ht.ExpValidated(t, 6, nil)
chain.SetNextSyncPeriod(3)
ts.Run(7)
// testSHead4 could be validated now but it's not queued by any registered server
ht.ExpValidated(t, 7, nil)
ts.ServerEvent(EvNewSignedHead, testServer2, testSHead4)
ts.Run(8)
// now testSHead4 should be validated
ht.ExpValidated(t, 8, []types.SignedHeader{testSHead4})
}
func TestPrefetchHead(t *testing.T) {
chain := &TestCommitteeChain{}
ht := &TestHeadTracker{}
headSync := NewHeadSync(ht, chain)
ts := NewTestScheduler(t, headSync)
ht.ExpPrefetch(t, 0, testHead0) // no servers registered
ts.AddServer(testServer1, 1)
ts.ServerEvent(EvNewHead, testServer1, testHead1)
ts.Run(1)
ht.ExpPrefetch(t, 1, testHead1) // s1: h1
ts.AddServer(testServer2, 1)
ts.ServerEvent(EvNewHead, testServer2, testHead2)
ts.Run(2)
ht.ExpPrefetch(t, 2, testHead2) // s1: h1, s2: h2
ts.ServerEvent(EvNewHead, testServer1, testHead2)
ts.Run(3)
ht.ExpPrefetch(t, 3, testHead2) // s1: h2, s2: h2
ts.AddServer(testServer3, 1)
ts.ServerEvent(EvNewHead, testServer3, testHead3)
ts.Run(4)
ht.ExpPrefetch(t, 4, testHead2) // s1: h2, s2: h2, s3: h3
ts.AddServer(testServer4, 1)
ts.ServerEvent(EvNewHead, testServer4, testHead4)
ts.Run(5)
ht.ExpPrefetch(t, 5, testHead2) // s1: h2, s2: h2, s3: h3, s4: h4
ts.ServerEvent(EvNewHead, testServer2, testHead3)
ts.Run(6)
ht.ExpPrefetch(t, 6, testHead3) // s1: h2, s2: h3, s3: h3, s4: h4
ts.RemoveServer(testServer3)
ts.Run(7)
ht.ExpPrefetch(t, 7, testHead4) // s1: h2, s2: h3, s4: h4
ts.RemoveServer(testServer1)
ts.Run(8)
ht.ExpPrefetch(t, 8, testHead4) // s2: h3, s4: h4
ts.RemoveServer(testServer4)
ts.Run(9)
ht.ExpPrefetch(t, 9, testHead3) // s2: h3
ts.RemoveServer(testServer2)
ts.Run(10)
ht.ExpPrefetch(t, 10, testHead0) // no servers registered
}

254
beacon/light/sync/test_helpers.go Normal file
View file

@ -0,0 +1,254 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package sync
import (
"reflect"
"testing"
"github.com/ethereum/go-ethereum/beacon/light"
"github.com/ethereum/go-ethereum/beacon/light/request"
"github.com/ethereum/go-ethereum/beacon/types"
)
type requestWithID struct {
sid request.ServerAndID
request request.Request
}
type TestScheduler struct {
t *testing.T
module request.Module
events []request.Event
servers []request.Server
allowance map[request.Server]int
sent map[int][]requestWithID
testIndex int
expFail map[request.Server]int // expected Server.Fail calls during next Run
lastId request.ID
}
func NewTestScheduler(t *testing.T, module request.Module) *TestScheduler {
return &TestScheduler{
t: t,
module: module,
allowance: make(map[request.Server]int),
expFail: make(map[request.Server]int),
sent: make(map[int][]requestWithID),
}
}
func (ts *TestScheduler) Run(testIndex int, exp ...any) {
expReqs := make([]requestWithID, len(exp)/2)
id := ts.lastId
for i := range expReqs {
id++
expReqs[i] = requestWithID{
sid: request.ServerAndID{Server: exp[i*2].(request.Server), ID: id},
request: exp[i*2+1].(request.Request),
}
}
if len(expReqs) == 0 {
expReqs = nil
}
ts.testIndex = testIndex
ts.module.Process(ts, ts.events)
ts.events = nil
for server, count := range ts.expFail {
delete(ts.expFail, server)
if count == 0 {
continue
}
ts.t.Errorf("Missing %d Server.Fail(s) from server %s in test case #%d", count, server.(string), testIndex)
}
if !reflect.DeepEqual(ts.sent[testIndex], expReqs) {
ts.t.Errorf("Wrong sent requests in test case #%d (expected %v, got %v)", testIndex, expReqs, ts.sent[testIndex])
}
}
func (ts *TestScheduler) CanSendTo() (cs []request.Server) {
for _, server := range ts.servers {
if ts.allowance[server] > 0 {
cs = append(cs, server)
}
}
return
}
func (ts *TestScheduler) Send(server request.Server, req request.Request) request.ID {
ts.lastId++
ts.sent[ts.testIndex] = append(ts.sent[ts.testIndex], requestWithID{
sid: request.ServerAndID{Server: server, ID: ts.lastId},
request: req,
})
ts.allowance[server]--
return ts.lastId
}
func (ts *TestScheduler) Fail(server request.Server, desc string) {
if ts.expFail[server] == 0 {
ts.t.Errorf("Unexpected Fail from server %s in test case #%d: %s", server.(string), ts.testIndex, desc)
return
}
ts.expFail[server]--
}
func (ts *TestScheduler) Request(testIndex, reqIndex int) requestWithID {
if len(ts.sent[testIndex]) < reqIndex {
ts.t.Errorf("Missing request from test case %d index %d", testIndex, reqIndex)
return requestWithID{}
}
return ts.sent[testIndex][reqIndex-1]
}
func (ts *TestScheduler) ServerEvent(evType *request.EventType, server request.Server, data any) {
ts.events = append(ts.events, request.Event{
Type: evType,
Server: server,
Data: data,
})
}
func (ts *TestScheduler) RequestEvent(evType *request.EventType, req requestWithID, resp request.Response) {
if req.request == nil {
return
}
ts.events = append(ts.events, request.Event{
Type: evType,
Server: req.sid.Server,
Data: request.RequestResponse{
ID: req.sid.ID,
Request: req.request,
Response: resp,
},
})
}
func (ts *TestScheduler) AddServer(server request.Server, allowance int) {
ts.servers = append(ts.servers, server)
ts.allowance[server] = allowance
ts.ServerEvent(request.EvRegistered, server, nil)
}
func (ts *TestScheduler) RemoveServer(server request.Server) {
ts.servers = append(ts.servers, server)
for i, s := range ts.servers {
if s == server {
copy(ts.servers[i:len(ts.servers)-1], ts.servers[i+1:])
ts.servers = ts.servers[:len(ts.servers)-1]
break
}
}
delete(ts.allowance, server)
ts.ServerEvent(request.EvUnregistered, server, nil)
}
func (ts *TestScheduler) AddAllowance(server request.Server, allowance int) {
ts.allowance[server] += allowance
}
func (ts *TestScheduler) ExpFail(server request.Server) {
ts.expFail[server]++
}
type TestCommitteeChain struct {
fsp, nsp uint64
init bool
}
func (t *TestCommitteeChain) CheckpointInit(bootstrap types.BootstrapData) error {
t.fsp, t.nsp, t.init = bootstrap.Header.SyncPeriod(), bootstrap.Header.SyncPeriod()+2, true
return nil
}
func (t *TestCommitteeChain) InsertUpdate(update *types.LightClientUpdate, nextCommittee *types.SerializedSyncCommittee) error {
period := update.AttestedHeader.Header.SyncPeriod()
if period < t.fsp || period > t.nsp || !t.init {
return light.ErrInvalidPeriod
}
if period == t.nsp {
t.nsp++
}
return nil
}
func (t *TestCommitteeChain) NextSyncPeriod() (uint64, bool) {
return t.nsp, t.init
}
func (tc *TestCommitteeChain) ExpInit(t *testing.T, ExpInit bool) {
if tc.init != ExpInit {
t.Errorf("Incorrect init flag (expected %v, got %v)", ExpInit, tc.init)
}
}
func (t *TestCommitteeChain) SetNextSyncPeriod(nsp uint64) {
t.init, t.nsp = true, nsp
}
func (tc *TestCommitteeChain) ExpNextSyncPeriod(t *testing.T, expNsp uint64) {
tc.ExpInit(t, true)
if tc.nsp != expNsp {
t.Errorf("Incorrect NextSyncPeriod (expected %d, got %d)", expNsp, tc.nsp)
}
}
type TestHeadTracker struct {
phead types.HeadInfo
validated []types.SignedHeader
}
func (ht *TestHeadTracker) ValidateHead(head types.SignedHeader) (bool, error) {
ht.validated = append(ht.validated, head)
return true, nil
}
// TODO add test case for finality
func (ht *TestHeadTracker) ValidateFinality(head types.FinalityUpdate) (bool, error) {
return true, nil
}
func (ht *TestHeadTracker) ExpValidated(t *testing.T, tci int, expHeads []types.SignedHeader) {
for i, expHead := range expHeads {
if i >= len(ht.validated) {
t.Errorf("Missing validated head in test case #%d index #%d (expected {slot %d blockRoot %x}, got none)", tci, i, expHead.Header.Slot, expHead.Header.Hash())
continue
}
if ht.validated[i] != expHead {
vhead := ht.validated[i].Header
t.Errorf("Wrong validated head in test case #%d index #%d (expected {slot %d blockRoot %x}, got {slot %d blockRoot %x})", tci, i, expHead.Header.Slot, expHead.Header.Hash(), vhead.Slot, vhead.Hash())
}
}
for i := len(expHeads); i < len(ht.validated); i++ {
vhead := ht.validated[i].Header
t.Errorf("Unexpected validated head in test case #%d index #%d (expected none, got {slot %d blockRoot %x})", tci, i, vhead.Slot, vhead.Hash())
}
ht.validated = nil
}
func (ht *TestHeadTracker) SetPrefetchHead(head types.HeadInfo) {
ht.phead = head
}
func (ht *TestHeadTracker) ExpPrefetch(t *testing.T, tci int, exp types.HeadInfo) {
if ht.phead != exp {
t.Errorf("Wrong prefetch head in test case #%d (expected {slot %d blockRoot %x}, got {slot %d blockRoot %x})", tci, exp.Slot, exp.BlockRoot, ht.phead.Slot, ht.phead.BlockRoot)
}
}

42
beacon/light/sync/types.go Normal file
View file

@ -0,0 +1,42 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package sync
import (
"github.com/ethereum/go-ethereum/beacon/light/request"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
)
var (
EvNewHead = &request.EventType{Name: "newHead"} // data: types.HeadInfo
EvNewSignedHead = &request.EventType{Name: "newSignedHead"} // data: types.SignedHeader
EvNewFinalityUpdate = &request.EventType{Name: "newFinalityUpdate"} // data: types.FinalityUpdate
)
type (
ReqUpdates struct {
FirstPeriod, Count uint64
}
RespUpdates struct {
Updates []*types.LightClientUpdate
Committees []*types.SerializedSyncCommittee
}
ReqHeader common.Hash
ReqCheckpointData common.Hash
ReqBeaconBlock common.Hash
)

299
beacon/light/sync/update_sync.go Normal file
View file

@ -0,0 +1,299 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package sync
import (
"sort"
"github.com/ethereum/go-ethereum/beacon/light"
"github.com/ethereum/go-ethereum/beacon/light/request"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
)
const maxUpdateRequest = 8 // maximum number of updates requested in a single request
type committeeChain interface {
CheckpointInit(bootstrap types.BootstrapData) error
InsertUpdate(update *types.LightClientUpdate, nextCommittee *types.SerializedSyncCommittee) error
NextSyncPeriod() (uint64, bool)
}
// CheckpointInit implements request.Module; it fetches the light client bootstrap
// data belonging to the given checkpoint hash and initializes the committee chain
// if successful.
type CheckpointInit struct {
chain committeeChain
checkpointHash common.Hash
locked request.ServerAndID
initialized bool
}
// NewCheckpointInit creates a new CheckpointInit.
func NewCheckpointInit(chain committeeChain, checkpointHash common.Hash) *CheckpointInit {
return &CheckpointInit{
chain: chain,
checkpointHash: checkpointHash,
}
}
// Process implements request.Module.
func (s *CheckpointInit) Process(requester request.Requester, events []request.Event) {
for _, event := range events {
if !event.IsRequestEvent() {
continue
}
sid, req, resp := event.RequestInfo()
if s.locked == sid {
s.locked = request.ServerAndID{}
}
if resp != nil {
if checkpoint := resp.(*types.BootstrapData); checkpoint.Header.Hash() == common.Hash(req.(ReqCheckpointData)) {
s.chain.CheckpointInit(*checkpoint)
s.initialized = true
return
}
requester.Fail(event.Server, "invalid checkpoint data")
}
}
// start a request if possible
if s.initialized || s.locked != (request.ServerAndID{}) {
return
}
cs := requester.CanSendTo()
if len(cs) == 0 {
return
}
server := cs[0]
id := requester.Send(server, ReqCheckpointData(s.checkpointHash))
s.locked = request.ServerAndID{Server: server, ID: id}
}
// ForwardUpdateSync implements request.Module; it fetches updates between the
// committee chain head and each server's announced head. Updates are fetched
// in batches and multiple batches can also be requested in parallel.
// Out of order responses are also handled; if a batch of updates cannot be added
// to the chain immediately because of a gap then the future updates are
// remembered until they can be processed.
type ForwardUpdateSync struct {
chain committeeChain
rangeLock rangeLock
lockedIDs map[request.ServerAndID]struct{}
processQueue []updateResponse
nextSyncPeriod map[request.Server]uint64
}
// NewForwardUpdateSync creates a new ForwardUpdateSync.
func NewForwardUpdateSync(chain committeeChain) *ForwardUpdateSync {
return &ForwardUpdateSync{
chain: chain,
rangeLock: make(rangeLock),
lockedIDs: make(map[request.ServerAndID]struct{}),
nextSyncPeriod: make(map[request.Server]uint64),
}
}
// rangeLock allows locking sections of an integer space, preventing the syncing
// mechanism from making requests again for sections where a not timed out request
// is already pending or where already fetched and unprocessed data is available.
type rangeLock map[uint64]int
// lock locks or unlocks the given section, depending on the sign of the add parameter.
func (r rangeLock) lock(first, count uint64, add int) {
for i := first; i < first+count; i++ {
if v := r[i] + add; v > 0 {
r[i] = v
} else {
delete(r, i)
}
}
}
// firstUnlocked returns the first unlocked section starting at or after start
// and not longer than maxCount.
func (r rangeLock) firstUnlocked(start, maxCount uint64) (first, count uint64) {
first = start
for {
if _, ok := r[first]; !ok {
break
}
first++
}
for {
count++
if count == maxCount {
break
}
if _, ok := r[first+count]; ok {
break
}
}
return
}
// lockRange locks the range belonging to the given update request, unless the
// same request has already been locked
func (s *ForwardUpdateSync) lockRange(sid request.ServerAndID, req ReqUpdates) {
if _, ok := s.lockedIDs[sid]; ok {
return
}
s.lockedIDs[sid] = struct{}{}
s.rangeLock.lock(req.FirstPeriod, req.Count, 1)
}
// unlockRange unlocks the range belonging to the given update request, unless
// same request has already been unlocked
func (s *ForwardUpdateSync) unlockRange(sid request.ServerAndID, req ReqUpdates) {
if _, ok := s.lockedIDs[sid]; !ok {
return
}
delete(s.lockedIDs, sid)
s.rangeLock.lock(req.FirstPeriod, req.Count, -1)
}
// verifyRange returns true if the number of updates and the individual update
// periods in the response match the requested section.
func (s *ForwardUpdateSync) verifyRange(request ReqUpdates, response RespUpdates) bool {
if uint64(len(response.Updates)) != request.Count || uint64(len(response.Committees)) != request.Count {
return false
}
for i, update := range response.Updates {
if update.AttestedHeader.Header.SyncPeriod() != request.FirstPeriod+uint64(i) {
return false
}
}
return true
}
// updateResponse is a response that has passed initial verification and has been
// queued for processing. Note that an update response cannot be processed until
// the previous updates have also been added to the chain.
type updateResponse struct {
sid request.ServerAndID
request ReqUpdates
response RespUpdates
}
// updateResponseList implements sort.Sort and sorts update request/response events by FirstPeriod.
type updateResponseList []updateResponse
func (u updateResponseList) Len() int { return len(u) }
func (u updateResponseList) Swap(i, j int) { u[i], u[j] = u[j], u[i] }
func (u updateResponseList) Less(i, j int) bool {
return u[i].request.FirstPeriod < u[j].request.FirstPeriod
}
// Process implements request.Module.
func (s *ForwardUpdateSync) Process(requester request.Requester, events []request.Event) {
for _, event := range events {
switch event.Type {
case request.EvResponse, request.EvFail, request.EvTimeout:
sid, rq, rs := event.RequestInfo()
req := rq.(ReqUpdates)
var queued bool
if event.Type == request.EvResponse {
resp := rs.(RespUpdates)
if s.verifyRange(req, resp) {
// there is a response with a valid format; put it in the process queue
s.processQueue = append(s.processQueue, updateResponse{sid: sid, request: req, response: resp})
s.lockRange(sid, req)
queued = true
} else {
requester.Fail(event.Server, "invalid update range")
}
}
if !queued {
s.unlockRange(sid, req)
}
case EvNewSignedHead:
signedHead := event.Data.(types.SignedHeader)
s.nextSyncPeriod[event.Server] = types.SyncPeriod(signedHead.SignatureSlot + 256)
case request.EvUnregistered:
delete(s.nextSyncPeriod, event.Server)
}
}
// try processing ordered list of available responses
sort.Sort(updateResponseList(s.processQueue))
for s.processQueue != nil {
u := s.processQueue[0]
if !s.processResponse(requester, u) {
break
}
s.unlockRange(u.sid, u.request)
s.processQueue = s.processQueue[1:]
if len(s.processQueue) == 0 {
s.processQueue = nil
}
}
// start new requests if possible
startPeriod, chainInit := s.chain.NextSyncPeriod()
if !chainInit {
return
}
for {
firstPeriod, maxCount := s.rangeLock.firstUnlocked(startPeriod, maxUpdateRequest)
var (
sendTo request.Server
bestCount uint64
)
for _, server := range requester.CanSendTo() {
nextPeriod := s.nextSyncPeriod[server]
if nextPeriod <= firstPeriod {
continue
}
count := maxCount
if nextPeriod < firstPeriod+maxCount {
count = nextPeriod - firstPeriod
}
if count > bestCount {
sendTo, bestCount = server, count
}
}
if sendTo == nil {
return
}
req := ReqUpdates{FirstPeriod: firstPeriod, Count: bestCount}
id := requester.Send(sendTo, req)
s.lockRange(request.ServerAndID{Server: sendTo, ID: id}, req)
}
}
// processResponse adds the fetched updates and committees to the committee chain.
// Returns true in case of full or partial success.
func (s *ForwardUpdateSync) processResponse(requester request.Requester, u updateResponse) (success bool) {
for i, update := range u.response.Updates {
if err := s.chain.InsertUpdate(update, u.response.Committees[i]); err != nil {
if err == light.ErrInvalidPeriod {
// there is a gap in the update periods; stop processing without
// failing and try again next time
return
}
if err == light.ErrInvalidUpdate || err == light.ErrWrongCommitteeRoot || err == light.ErrCannotReorg {
requester.Fail(u.sid.Server, "invalid update received")
} else {
log.Error("Unexpected InsertUpdate error", "error", err)
}
return
}
success = true
}
return
}

219
beacon/light/sync/update_sync_test.go Normal file
View file

@ -0,0 +1,219 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package sync
import (
"testing"
"github.com/ethereum/go-ethereum/beacon/light/request"
"github.com/ethereum/go-ethereum/beacon/types"
)
func TestCheckpointInit(t *testing.T) {
chain := &TestCommitteeChain{}
checkpoint := &types.BootstrapData{Header: types.Header{Slot: 0x2000*4 + 0x1000}} // period 4
checkpointHash := checkpoint.Header.Hash()
chkInit := NewCheckpointInit(chain, checkpointHash)
ts := NewTestScheduler(t, chkInit)
// add 2 servers
ts.AddServer(testServer1, 1)
ts.AddServer(testServer2, 1)
// expect bootstrap request to server 1
ts.Run(1, testServer1, ReqCheckpointData(checkpointHash))
// server 1 times out; expect request to server 2
ts.RequestEvent(request.EvTimeout, ts.Request(1, 1), nil)
ts.Run(2, testServer2, ReqCheckpointData(checkpointHash))
// invalid response from server 2; expect init state to still be false
ts.RequestEvent(request.EvResponse, ts.Request(2, 1), &types.BootstrapData{Header: types.Header{Slot: 123456}})
ts.ExpFail(testServer2)
ts.Run(3)
chain.ExpInit(t, false)
// server 1 fails (hard timeout)
ts.RequestEvent(request.EvFail, ts.Request(1, 1), nil)
ts.Run(4)
chain.ExpInit(t, false)
// server 3 is registered; expect bootstrap request to server 3
ts.AddServer(testServer3, 1)
ts.Run(5, testServer3, ReqCheckpointData(checkpointHash))
// valid response from server 3; expect chain to be initialized
ts.RequestEvent(request.EvResponse, ts.Request(5, 1), checkpoint)
ts.Run(6)
chain.ExpInit(t, true)
}
func TestUpdateSyncParallel(t *testing.T) {
chain := &TestCommitteeChain{}
chain.SetNextSyncPeriod(0)
updateSync := NewForwardUpdateSync(chain)
ts := NewTestScheduler(t, updateSync)
// add 2 servers, head at period 100; allow 3-3 parallel requests for each
ts.AddServer(testServer1, 3)
ts.ServerEvent(EvNewSignedHead, testServer1, types.SignedHeader{SignatureSlot: 0x2000*100 + 0x1000})
ts.AddServer(testServer2, 3)
ts.ServerEvent(EvNewSignedHead, testServer2, types.SignedHeader{SignatureSlot: 0x2000*100 + 0x1000})
// expect 6 requests to be sent
ts.Run(1,
testServer1, ReqUpdates{FirstPeriod: 0, Count: 8},
testServer1, ReqUpdates{FirstPeriod: 8, Count: 8},
testServer1, ReqUpdates{FirstPeriod: 16, Count: 8},
testServer2, ReqUpdates{FirstPeriod: 24, Count: 8},
testServer2, ReqUpdates{FirstPeriod: 32, Count: 8},
testServer2, ReqUpdates{FirstPeriod: 40, Count: 8})
// valid response to request 1; expect 8 periods synced and a new request started
ts.RequestEvent(request.EvResponse, ts.Request(1, 1), testRespUpdate(ts.Request(1, 1)))
ts.AddAllowance(testServer1, 1)
ts.Run(2, testServer1, ReqUpdates{FirstPeriod: 48, Count: 8})
chain.ExpNextSyncPeriod(t, 8)
// valid response to requests 4 and 5
ts.RequestEvent(request.EvResponse, ts.Request(1, 4), testRespUpdate(ts.Request(1, 4)))
ts.RequestEvent(request.EvResponse, ts.Request(1, 5), testRespUpdate(ts.Request(1, 5)))
ts.AddAllowance(testServer2, 2)
// expect 2 more requests but no sync progress (responses 4 and 5 cannot be added before 2 and 3)
ts.Run(3,
testServer2, ReqUpdates{FirstPeriod: 56, Count: 8},
testServer2, ReqUpdates{FirstPeriod: 64, Count: 8})
chain.ExpNextSyncPeriod(t, 8)
// soft timeout for requests 2 and 3 (server 1 is overloaded)
ts.RequestEvent(request.EvTimeout, ts.Request(1, 2), nil)
ts.RequestEvent(request.EvTimeout, ts.Request(1, 3), nil)
// no allowance, no more requests
ts.Run(4)
// valid response to requests 6 and 8 and 9
ts.RequestEvent(request.EvResponse, ts.Request(1, 6), testRespUpdate(ts.Request(1, 6)))
ts.RequestEvent(request.EvResponse, ts.Request(3, 1), testRespUpdate(ts.Request(3, 1)))
ts.RequestEvent(request.EvResponse, ts.Request(3, 2), testRespUpdate(ts.Request(3, 2)))
ts.AddAllowance(testServer2, 3)
// server 2 can now resend requests 2 and 3 (timed out by server 1) and also send a new one
ts.Run(5,
testServer2, ReqUpdates{FirstPeriod: 8, Count: 8},
testServer2, ReqUpdates{FirstPeriod: 16, Count: 8},
testServer2, ReqUpdates{FirstPeriod: 72, Count: 8})
// server 1 finally answers timed out request 2
ts.RequestEvent(request.EvResponse, ts.Request(1, 2), testRespUpdate(ts.Request(1, 2)))
ts.AddAllowance(testServer1, 1)
// expect sync progress and one new request
ts.Run(6, testServer1, ReqUpdates{FirstPeriod: 80, Count: 8})
chain.ExpNextSyncPeriod(t, 16)
// server 2 answers requests 11 and 12 (resends of requests 2 and 3)
ts.RequestEvent(request.EvResponse, ts.Request(5, 1), testRespUpdate(ts.Request(5, 1)))
ts.RequestEvent(request.EvResponse, ts.Request(5, 2), testRespUpdate(ts.Request(5, 2)))
ts.AddAllowance(testServer2, 2)
ts.Run(7,
testServer2, ReqUpdates{FirstPeriod: 88, Count: 8},
testServer2, ReqUpdates{FirstPeriod: 96, Count: 4})
// finally the gap is filled, update can process responses up to req6
chain.ExpNextSyncPeriod(t, 48)
// all remaining requests are answered
ts.RequestEvent(request.EvResponse, ts.Request(1, 3), testRespUpdate(ts.Request(1, 3)))
ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testRespUpdate(ts.Request(2, 1)))
ts.RequestEvent(request.EvResponse, ts.Request(5, 3), testRespUpdate(ts.Request(5, 3)))
ts.RequestEvent(request.EvResponse, ts.Request(6, 1), testRespUpdate(ts.Request(6, 1)))
ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testRespUpdate(ts.Request(7, 1)))
ts.RequestEvent(request.EvResponse, ts.Request(7, 2), testRespUpdate(ts.Request(7, 2)))
ts.Run(8)
// expect chain to be fully synced
chain.ExpNextSyncPeriod(t, 100)
}
func TestUpdateSyncDifferentHeads(t *testing.T) {
chain := &TestCommitteeChain{}
chain.SetNextSyncPeriod(10)
updateSync := NewForwardUpdateSync(chain)
ts := NewTestScheduler(t, updateSync)
// add 3 servers with different announced head periods
ts.AddServer(testServer1, 1)
ts.ServerEvent(EvNewSignedHead, testServer1, types.SignedHeader{SignatureSlot: 0x2000*15 + 0x1000})
ts.AddServer(testServer2, 1)
ts.ServerEvent(EvNewSignedHead, testServer2, types.SignedHeader{SignatureSlot: 0x2000*16 + 0x1000})
ts.AddServer(testServer3, 1)
ts.ServerEvent(EvNewSignedHead, testServer3, types.SignedHeader{SignatureSlot: 0x2000*17 + 0x1000})
// expect request to the best announced head
ts.Run(1, testServer3, ReqUpdates{FirstPeriod: 10, Count: 7})
// request times out, expect request to the next best head
ts.RequestEvent(request.EvTimeout, ts.Request(1, 1), nil)
ts.Run(2, testServer2, ReqUpdates{FirstPeriod: 10, Count: 6})
// request times out, expect request to the last available server
ts.RequestEvent(request.EvTimeout, ts.Request(2, 1), nil)
ts.Run(3, testServer1, ReqUpdates{FirstPeriod: 10, Count: 5})
// valid response to request 3, expect chain synced to period 15
ts.RequestEvent(request.EvResponse, ts.Request(3, 1), testRespUpdate(ts.Request(3, 1)))
ts.AddAllowance(testServer1, 1)
ts.Run(4)
chain.ExpNextSyncPeriod(t, 15)
// invalid response to request 1, server can only deliver updates up to period 15 despite announced head
truncated := ts.Request(1, 1)
truncated.request = ReqUpdates{FirstPeriod: 10, Count: 5}
ts.RequestEvent(request.EvResponse, ts.Request(1, 1), testRespUpdate(truncated))
ts.ExpFail(testServer3)
ts.Run(5)
// expect no progress of chain head
chain.ExpNextSyncPeriod(t, 15)
// valid response to request 2, expect chain synced to period 16
ts.RequestEvent(request.EvResponse, ts.Request(2, 1), testRespUpdate(ts.Request(2, 1)))
ts.AddAllowance(testServer2, 1)
ts.Run(6)
chain.ExpNextSyncPeriod(t, 16)
// a new server is registered with announced head period 17
ts.AddServer(testServer4, 1)
ts.ServerEvent(EvNewSignedHead, testServer4, types.SignedHeader{SignatureSlot: 0x2000*17 + 0x1000})
// expect request to sync one more period
ts.Run(7, testServer4, ReqUpdates{FirstPeriod: 16, Count: 1})
// valid response, expect chain synced to period 17
ts.RequestEvent(request.EvResponse, ts.Request(7, 1), testRespUpdate(ts.Request(7, 1)))
ts.AddAllowance(testServer4, 1)
ts.Run(8)
chain.ExpNextSyncPeriod(t, 17)
}
func testRespUpdate(request requestWithID) request.Response {
var resp RespUpdates
if request.request == nil {
return resp
}
req := request.request.(ReqUpdates)
resp.Updates = make([]*types.LightClientUpdate, int(req.Count))
resp.Committees = make([]*types.SerializedSyncCommittee, int(req.Count))
period := req.FirstPeriod
for i := range resp.Updates {
resp.Updates[i] = &types.LightClientUpdate{AttestedHeader: types.SignedHeader{Header: types.Header{Slot: 0x2000*period + 0x1000}}}
resp.Committees[i] = new(types.SerializedSyncCommittee)
period++
}
return resp
}

2
beacon/params/params.go
View file

@ -41,4 +41,6 @@ const (
StateIndexNextSyncCommittee = 55
StateIndexExecPayload = 56
StateIndexExecHead = 908
BodyIndexExecPayload = 25
)

56
beacon/types/light_sync.go
View file

@ -20,11 +20,20 @@ import (
"errors"
"fmt"
"github.com/ethereum/go-ethereum/beacon/engine"
"github.com/ethereum/go-ethereum/beacon/merkle"
"github.com/ethereum/go-ethereum/beacon/params"
"github.com/ethereum/go-ethereum/common"
"github.com/protolambda/zrnt/eth2/beacon/capella"
"github.com/protolambda/ztyp/tree"
)
// HeadInfo represents an unvalidated new head announcement.
type HeadInfo struct {
Slot uint64
BlockRoot common.Hash
}
// BootstrapData contains a sync committee where light sync can be started,
// together with a proof through a beacon header and corresponding state.
// Note: BootstrapData is fetched from a server based on a known checkpoint hash.
@ -134,3 +143,50 @@ func (u UpdateScore) BetterThan(w UpdateScore) bool {
}
return u.SignerCount > w.SignerCount
}
type HeaderWithExecProof struct {
Header
PayloadHeader *capella.ExecutionPayloadHeader
PayloadBranch merkle.Values
}
func (h *HeaderWithExecProof) Validate() error {
payloadRoot := merkle.Value(h.PayloadHeader.HashTreeRoot(tree.GetHashFn()))
return merkle.VerifyProof(h.BodyRoot, params.BodyIndexExecPayload, h.PayloadBranch, payloadRoot)
}
type FinalityUpdate struct {
Attested, Finalized HeaderWithExecProof
FinalityBranch merkle.Values
// Sync committee BLS signature aggregate
Signature SyncAggregate
// Slot in which the signature has been created (newer than Header.Slot,
// determines the signing sync committee)
SignatureSlot uint64
}
func (u *FinalityUpdate) SignedHeader() SignedHeader {
return SignedHeader{
Header: u.Attested.Header,
Signature: u.Signature,
SignatureSlot: u.SignatureSlot,
}
}
func (u *FinalityUpdate) Validate() error {
if err := u.Attested.Validate(); err != nil {
return err
}
if err := u.Finalized.Validate(); err != nil {
return err
}
return merkle.VerifyProof(u.Attested.StateRoot, params.StateIndexFinalBlock, u.FinalityBranch, merkle.Value(u.Finalized.Hash()))
}
// ChainHeadEvent returns an authenticated execution payload associated with the
// latest accepted head of the beacon chain, along with the hash of the latest
// finalized execution block.
type ChainHeadEvent struct {
HeadBlock *engine.ExecutableData
Finalized common.Hash
}

69
cmd/blsync/engine_api.go Normal file
View file

@ -0,0 +1,69 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"time"
"github.com/ethereum/go-ethereum/beacon/engine"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rpc"
)
func updateEngineApi(client *rpc.Client, headCh chan types.ChainHeadEvent) {
for event := range headCh {
if client == nil { // dry run, no engine API specified
log.Info("New execution block retrieved", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "finalized block hash", event.Finalized)
} else {
if status, err := callNewPayloadV2(client, event.HeadBlock); err == nil {
log.Info("Successful NewPayload", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "status", status)
} else {
log.Error("Failed NewPayload", "block number", event.HeadBlock.Number, "block hash", event.HeadBlock.BlockHash, "error", err)
}
if status, err := callForkchoiceUpdatedV1(client, event.HeadBlock.BlockHash, event.Finalized); err == nil {
log.Info("Successful ForkchoiceUpdated", "head", event.HeadBlock.BlockHash, "status", status)
} else {
log.Error("Failed ForkchoiceUpdated", "head", event.HeadBlock.BlockHash, "error", err)
}
}
}
}
func callNewPayloadV2(client *rpc.Client, execData *engine.ExecutableData) (string, error) {
var resp engine.PayloadStatusV1
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
err := client.CallContext(ctx, &resp, "engine_newPayloadV2", execData)
cancel()
return resp.Status, err
}
func callForkchoiceUpdatedV1(client *rpc.Client, headHash, finalizedHash common.Hash) (string, error) {
var resp engine.ForkChoiceResponse
update := engine.ForkchoiceStateV1{
HeadBlockHash: headHash,
SafeBlockHash: finalizedHash,
FinalizedBlockHash: finalizedHash,
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
err := client.CallContext(ctx, &resp, "engine_forkchoiceUpdatedV1", update, nil)
cancel()
return resp.PayloadStatus.Status, err
}

125
cmd/blsync/main.go Normal file
View file

@ -0,0 +1,125 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"context"
"fmt"
"io"
"os"
"github.com/ethereum/go-ethereum/beacon/blsync"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/rpc"
"github.com/mattn/go-colorable"
"github.com/mattn/go-isatty"
"github.com/urfave/cli/v2"
)
var (
verbosityFlag = &cli.IntFlag{
Name: "verbosity",
Usage: "Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail",
Value: 3,
Category: flags.LoggingCategory,
}
vmoduleFlag = &cli.StringFlag{
Name: "vmodule",
Usage: "Per-module verbosity: comma-separated list of <pattern>=<level> (e.g. eth/*=5,p2p=4)",
Value: "",
Hidden: true,
Category: flags.LoggingCategory,
}
)
func main() {
app := flags.NewApp("beacon light syncer tool")
app.Flags = []cli.Flag{
utils.BeaconApiFlag,
utils.BeaconApiHeaderFlag,
utils.BeaconThresholdFlag,
utils.BeaconNoFilterFlag,
utils.BeaconConfigFlag,
utils.BeaconGenesisRootFlag,
utils.BeaconGenesisTimeFlag,
utils.BeaconCheckpointFlag,
//TODO datadir for optional permanent database
utils.MainnetFlag,
utils.SepoliaFlag,
utils.GoerliFlag,
utils.BlsyncApiFlag,
utils.BlsyncJWTSecretFlag,
verbosityFlag,
vmoduleFlag,
}
app.Action = sync
if err := app.Run(os.Args); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func sync(ctx *cli.Context) error {
usecolor := (isatty.IsTerminal(os.Stderr.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd())) && os.Getenv("TERM") != "dumb"
output := io.Writer(os.Stderr)
if usecolor {
output = colorable.NewColorable(os.Stderr)
}
verbosity := log.FromLegacyLevel(ctx.Int(verbosityFlag.Name))
log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(output, verbosity, usecolor)))
headCh := make(chan types.ChainHeadEvent, 16)
client := blsync.NewClient(ctx)
sub := client.SubscribeChainHeadEvent(headCh)
go updateEngineApi(makeRPCClient(ctx), headCh)
client.Start()
// run until stopped
<-ctx.Done()
client.Stop()
sub.Unsubscribe()
close(headCh)
return nil
}
func makeRPCClient(ctx *cli.Context) *rpc.Client {
if !ctx.IsSet(utils.BlsyncApiFlag.Name) {
log.Warn("No engine API target specified, performing a dry run")
return nil
}
if !ctx.IsSet(utils.BlsyncJWTSecretFlag.Name) {
utils.Fatalf("JWT secret parameter missing") //TODO use default if datadir is specified
}
engineApiUrl, jwtFileName := ctx.String(utils.BlsyncApiFlag.Name), ctx.String(utils.BlsyncJWTSecretFlag.Name)
var jwtSecret [32]byte
if jwt, err := node.ObtainJWTSecret(jwtFileName); err == nil {
copy(jwtSecret[:], jwt)
} else {
utils.Fatalf("Error loading or generating JWT secret: %v", err)
}
auth := node.NewJWTAuth(jwtSecret)
cl, err := rpc.DialOptions(context.Background(), engineApiUrl, rpc.WithHTTPAuth(auth))
if err != nil {
utils.Fatalf("Could not create RPC client: %v", err)
}
return cl
}

8
cmd/clef/README.md
View file

@ -916,7 +916,7 @@ There are a couple of implementation for a UI. We'll try to keep this list up to
| Name | Repo | UI type| No external resources| Blocky support| Verifies permissions | Hash information | No secondary storage | Statically linked| Can modify parameters|
| ---- | ---- | -------| ---- | ---- | ---- |---- | ---- | ---- | ---- |
| QtSigner| https://github.com/holiman/qtsigner/| Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)|
| GtkSigner| https://github.com/holiman/gtksigner| Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: |
| Frame | https://github.com/floating/frame/commits/go-signer| Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: |
| Clef UI| https://github.com/ethereum/clef-ui| Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)|
| QtSigner| https://github.com/holiman/qtsigner/ | Python3/QT-based| :+1:| :+1:| :+1:| :+1:| :+1:| :x: | :+1: (partially)|
| GtkSigner| https://github.com/holiman/gtksigner | Python3/GTK-based| :+1:| :x:| :x:| :+1:| :+1:| :x: | :x: |
| Frame | https://github.com/floating/frame/commits/go-signer | Electron-based| :x:| :x:| :x:| :x:| ?| :x: | :x: |
| Clef UI| https://github.com/ethereum/clef-ui | Golang/QT-based| :+1:| :+1:| :x:| :+1:| :+1:| :x: | :+1: (approve tx only)|

2
cmd/devp2p/internal/ethtest/snap.go
View file

@ -648,7 +648,7 @@ The server should reject the request.`,
0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, 7, 8}},
},
nBytes: 5000,
expHashes: []common.Hash{common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")},
expHashes: []common.Hash{types.EmptyCodeHash},
},
{

12
cmd/devp2p/main.go
View file

@ -66,9 +66,15 @@ func commandHasFlag(ctx *cli.Context, flag cli.Flag) bool {
for _, name := range names {
set[name] = struct{}{}
}
for _, fn := range ctx.FlagNames() {
if _, ok := set[fn]; ok {
return true
for _, ctx := range ctx.Lineage() {
if ctx.Command != nil {
for _, f := range ctx.Command.Flags {
for _, name := range f.Names() {
if _, ok := set[name]; ok {
return true
}
}
}
}
}
return false

7
cmd/era/main.go
View file

@ -18,6 +18,7 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"math/big"
"os"
@ -182,7 +183,7 @@ func open(ctx *cli.Context, epoch uint64) (*era.Era, error) {
// that the accumulator matches the expected value.
func verify(ctx *cli.Context) error {
if ctx.Args().Len() != 1 {
return fmt.Errorf("missing accumulators file")
return errors.New("missing accumulators file")
}
roots, err := readHashes(ctx.Args().First())
@ -203,7 +204,7 @@ func verify(ctx *cli.Context) error {
}
if len(entries) != len(roots) {
return fmt.Errorf("number of era1 files should match the number of accumulator hashes")
return errors.New("number of era1 files should match the number of accumulator hashes")
}
// Verify each epoch matches the expected root.
@ -308,7 +309,7 @@ func checkAccumulator(e *era.Era) error {
func readHashes(f string) ([]common.Hash, error) {
b, err := os.ReadFile(f)
if err != nil {
return nil, fmt.Errorf("unable to open accumulators file")
return nil, errors.New("unable to open accumulators file")
}
s := strings.Split(string(b), "\n")
// Remove empty last element, if present.

2
cmd/evm/internal/t8ntool/block.go
View file

@ -242,7 +242,7 @@ func readInput(ctx *cli.Context) (*bbInput, error) {
if headerStr == stdinSelector || ommersStr == stdinSelector || txsStr == stdinSelector || cliqueStr == stdinSelector {
decoder := json.NewDecoder(os.Stdin)
if err := decoder.Decode(inputData); err != nil {
return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err))
return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err))
}
}
if cliqueStr != stdinSelector && cliqueStr != "" {

2
cmd/evm/internal/t8ntool/transaction.go
View file

@ -86,7 +86,7 @@ func Transaction(ctx *cli.Context) error {
if txStr == stdinSelector {
decoder := json.NewDecoder(os.Stdin)
if err := decoder.Decode(inputData); err != nil {
return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err))
return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err))
}
// Decode the body of already signed transactions
body = common.FromHex(inputData.TxRlp)

2
cmd/evm/internal/t8ntool/transition.go
View file

@ -135,7 +135,7 @@ func Transition(ctx *cli.Context) error {
if allocStr == stdinSelector || envStr == stdinSelector || txStr == stdinSelector {
decoder := json.NewDecoder(os.Stdin)
if err := decoder.Decode(inputData); err != nil {
return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err))
return NewError(ErrorJson, fmt.Errorf("failed unmarshalling stdin: %v", err))
}
}
if allocStr != stdinSelector {

2
cmd/evm/internal/t8ntool/tx_iterator.go
View file

@ -127,7 +127,7 @@ func loadTransactions(txStr string, inputData *input, env stEnv, chainConfig *pa
return newRlpTxIterator(body), nil
}
if err := json.Unmarshal(data, &txsWithKeys); err != nil {
return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshaling txs-file: %v", err))
return nil, NewError(ErrorJson, fmt.Errorf("failed unmarshalling txs-file: %v", err))
}
} else {
if len(inputData.TxRlp) > 0 {

2
cmd/evm/internal/t8ntool/utils.go
View file

@ -33,7 +33,7 @@ func readFile(path, desc string, dest interface{}) error {
defer inFile.Close()
decoder := json.NewDecoder(inFile)
if err := decoder.Decode(dest); err != nil {
return NewError(ErrorJson, fmt.Errorf("failed unmarshaling %s file: %v", desc, err))
return NewError(ErrorJson, fmt.Errorf("failed unmarshalling %s file: %v", desc, err))
}
return nil
}

26
cmd/geth/chaincmd.go
View file

@ -444,7 +444,7 @@ func importHistory(ctx *cli.Context) error {
return fmt.Errorf("no era1 files found in %s", dir)
}
if len(networks) > 1 {
return fmt.Errorf("multiple networks found, use a network flag to specify desired network")
return errors.New("multiple networks found, use a network flag to specify desired network")
}
network = networks[0]
}
@ -514,13 +514,10 @@ func importPreimages(ctx *cli.Context) error {
return nil
}
func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, ethdb.Database, common.Hash, error) {
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()
func parseDumpConfig(ctx *cli.Context, stack *node.Node, db ethdb.Database) (*state.DumpConfig, common.Hash, error) {
var header *types.Header
if ctx.NArg() > 1 {
return nil, nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg())
return nil, common.Hash{}, fmt.Errorf("expected 1 argument (number or hash), got %d", ctx.NArg())
}
if ctx.NArg() == 1 {
arg := ctx.Args().First()
@ -529,17 +526,17 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
if number := rawdb.ReadHeaderNumber(db, hash); number != nil {
header = rawdb.ReadHeader(db, hash, *number)
} else {
return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash)
return nil, common.Hash{}, fmt.Errorf("block %x not found", hash)
}
} else {
number, err := strconv.ParseUint(arg, 10, 64)
if err != nil {
return nil, nil, common.Hash{}, err
return nil, common.Hash{}, err
}
if hash := rawdb.ReadCanonicalHash(db, number); hash != (common.Hash{}) {
header = rawdb.ReadHeader(db, hash, number)
} else {
return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number)
return nil, common.Hash{}, fmt.Errorf("header for block %d not found", number)
}
}
} else {
@ -547,7 +544,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
header = rawdb.ReadHeadHeader(db)
}
if header == nil {
return nil, nil, common.Hash{}, errors.New("no head block found")
return nil, common.Hash{}, errors.New("no head block found")
}
startArg := common.FromHex(ctx.String(utils.StartKeyFlag.Name))
var start common.Hash
@ -559,7 +556,7 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
start = crypto.Keccak256Hash(startArg)
log.Info("Converting start-address to hash", "address", common.BytesToAddress(startArg), "hash", start.Hex())
default:
return nil, nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg)
return nil, common.Hash{}, fmt.Errorf("invalid start argument: %x. 20 or 32 hex-encoded bytes required", startArg)
}
var conf = &state.DumpConfig{
SkipCode: ctx.Bool(utils.ExcludeCodeFlag.Name),
@ -571,14 +568,17 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth
log.Info("State dump configured", "block", header.Number, "hash", header.Hash().Hex(),
"skipcode", conf.SkipCode, "skipstorage", conf.SkipStorage,
"start", hexutil.Encode(conf.Start), "limit", conf.Max)
return conf, db, header.Root, nil
return conf, header.Root, nil
}
func dump(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
conf, db, root, err := parseDumpConfig(ctx, stack)
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()
conf, root, err := parseDumpConfig(ctx, stack, db)
if err != nil {
return err
}

3
cmd/geth/config.go
View file

@ -31,6 +31,7 @@ import (
"github.com/ethereum/go-ethereum/accounts/keystore"
"github.com/ethereum/go-ethereum/accounts/scwallet"
"github.com/ethereum/go-ethereum/accounts/usbwallet"
"github.com/ethereum/go-ethereum/beacon/blsync"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@ -221,6 +222,8 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
}
catalyst.RegisterSimulatedBeaconAPIs(stack, simBeacon)
stack.RegisterLifecycle(simBeacon)
} else if ctx.IsSet(utils.BeaconApiFlag.Name) {
stack.RegisterLifecycle(catalyst.NewBlsync(blsync.NewClient(ctx), eth))
} else {
err := catalyst.Register(stack, eth)
if err != nil {

3
cmd/geth/consolecmd_test.go
View file

@ -71,7 +71,6 @@ func TestConsoleWelcome(t *testing.T) {
Welcome to the Geth JavaScript console!
instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
coinbase: {{.Etherbase}}
at block: 0 ({{niltime}})
datadir: {{.Datadir}}
modules: {{apis}}
@ -131,7 +130,6 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) {
attach.SetTemplateFunc("goarch", func() string { return runtime.GOARCH })
attach.SetTemplateFunc("gover", runtime.Version)
attach.SetTemplateFunc("gethver", func() string { return params.VersionWithCommit("", "") })
attach.SetTemplateFunc("etherbase", func() string { return geth.Etherbase })
attach.SetTemplateFunc("niltime", func() string {
return time.Unix(1548854791, 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)")
})
@ -144,7 +142,6 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) {
Welcome to the Geth JavaScript console!
instance: Geth/v{{gethver}}/{{goos}}-{{goarch}}/{{gover}}
coinbase: {{etherbase}}
at block: 0 ({{niltime}}){{if ipc}}
datadir: {{datadir}}{{end}}
modules: {{apis}}

34
cmd/geth/main.go
View file

@ -30,7 +30,6 @@ import (
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/console/prompt"
"github.com/ethereum/go-ethereum/eth"
"github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/ethclient"
"github.com/ethereum/go-ethereum/internal/debug"
@ -116,13 +115,14 @@ var (
utils.DiscoveryPortFlag,
utils.MaxPeersFlag,
utils.MaxPendingPeersFlag,
utils.MiningEnabledFlag,
utils.MiningEnabledFlag, // deprecated
utils.MinerGasLimitFlag,
utils.MinerGasPriceFlag,
utils.MinerEtherbaseFlag,
utils.MinerEtherbaseFlag, // deprecated
utils.MinerExtraDataFlag,
utils.MinerRecommitIntervalFlag,
utils.MinerNewPayloadTimeout,
utils.MinerPendingFeeRecipientFlag,
utils.MinerNewPayloadTimeoutFlag, // deprecated
utils.NATFlag,
utils.NoDiscoverFlag,
utils.DiscoveryV4Flag,
@ -146,6 +146,14 @@ var (
configFileFlag,
utils.LogDebugFlag,
utils.LogBacktraceAtFlag,
utils.BeaconApiFlag,
utils.BeaconApiHeaderFlag,
utils.BeaconThresholdFlag,
utils.BeaconNoFilterFlag,
utils.BeaconConfigFlag,
utils.BeaconGenesisRootFlag,
utils.BeaconGenesisTimeFlag,
utils.BeaconCheckpointFlag,
}, utils.NetworkFlags, utils.DatabaseFlags)
rpcFlags = []cli.Flag{
@ -421,24 +429,6 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon
}
}()
}
// Start auxiliary services if enabled
if ctx.Bool(utils.MiningEnabledFlag.Name) {
// Mining only makes sense if a full Ethereum node is running
if ctx.String(utils.SyncModeFlag.Name) == "light" {
utils.Fatalf("Light clients do not support mining")
}
ethBackend, ok := backend.(*eth.EthAPIBackend)
if !ok {
utils.Fatalf("Ethereum service not running")
}
// Set the gas price to the limits from the CLI and start mining
gasprice := flags.GlobalBig(ctx, utils.MinerGasPriceFlag.Name)
ethBackend.TxPool().SetGasTip(gasprice)
if err := ethBackend.StartMining(); err != nil {
utils.Fatalf("Failed to start mining: %v", err)
}
}
}
// unlockAccounts unlocks any account specifically requested.

7
cmd/geth/snapshot.go
View file

@ -36,7 +36,7 @@ import (
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/trie"
cli "github.com/urfave/cli/v2"
"github.com/urfave/cli/v2"
)
var (
@ -541,7 +541,10 @@ func dumpState(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
conf, db, root, err := parseDumpConfig(ctx, stack)
db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()
conf, root, err := parseDumpConfig(ctx, stack, db)
if err != nil {
return err
}

1
cmd/geth/testdata/clique.json vendored
View file

@ -8,6 +8,7 @@
"byzantiumBlock": 0,
"constantinopleBlock": 0,
"petersburgBlock": 0,
"terminalTotalDifficultyPassed": true,
"clique": {
"period": 5,
"epoch": 30000

2
cmd/geth/verkle.go
View file

@ -29,7 +29,7 @@ import (
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/log"
"github.com/gballet/go-verkle"
cli "github.com/urfave/cli/v2"
"github.com/urfave/cli/v2"
)
var (

2
cmd/utils/cmd.go
View file

@ -245,7 +245,7 @@ func readList(filename string) ([]string, error) {
// starting from genesis.
func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error {
if chain.CurrentSnapBlock().Number.BitLen() != 0 {
return fmt.Errorf("history import only supported when starting from genesis")
return errors.New("history import only supported when starting from genesis")
}
entries, err := era.ReadDir(dir, network)
if err != nil {

99
cmd/utils/flags.go
View file

@ -36,6 +36,7 @@ import (
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/accounts/keystore"
bparams "github.com/ethereum/go-ethereum/beacon/params"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/fdlimit"
"github.com/ethereum/go-ethereum/core"
@ -281,6 +282,58 @@ var (
Value: ethconfig.Defaults.TransactionHistory,
Category: flags.StateCategory,
}
// Beacon client light sync settings
BeaconApiFlag = &cli.StringSliceFlag{
Name: "beacon.api",
Usage: "Beacon node (CL) light client API URL. This flag can be given multiple times.",
Category: flags.BeaconCategory,
}
BeaconApiHeaderFlag = &cli.StringSliceFlag{
Name: "beacon.api.header",
Usage: "Pass custom HTTP header fields to the emote beacon node API in \"key:value\" format. This flag can be given multiple times.",
Category: flags.BeaconCategory,
}
BeaconThresholdFlag = &cli.IntFlag{
Name: "beacon.threshold",
Usage: "Beacon sync committee participation threshold",
Value: bparams.SyncCommitteeSupermajority,
Category: flags.BeaconCategory,
}
BeaconNoFilterFlag = &cli.BoolFlag{
Name: "beacon.nofilter",
Usage: "Disable future slot signature filter",
Category: flags.BeaconCategory,
}
BeaconConfigFlag = &cli.StringFlag{
Name: "beacon.config",
Usage: "Beacon chain config YAML file",
Category: flags.BeaconCategory,
}
BeaconGenesisRootFlag = &cli.StringFlag{
Name: "beacon.genesis.gvroot",
Usage: "Beacon chain genesis validators root",
Category: flags.BeaconCategory,
}
BeaconGenesisTimeFlag = &cli.Uint64Flag{
Name: "beacon.genesis.time",
Usage: "Beacon chain genesis time",
Category: flags.BeaconCategory,
}
BeaconCheckpointFlag = &cli.StringFlag{
Name: "beacon.checkpoint",
Usage: "Beacon chain weak subjectivity checkpoint block hash",
Category: flags.BeaconCategory,
}
BlsyncApiFlag = &cli.StringFlag{
Name: "blsync.engine.api",
Usage: "Target EL engine API URL",
Category: flags.BeaconCategory,
}
BlsyncJWTSecretFlag = &cli.StringFlag{
Name: "blsync.jwtsecret",
Usage: "Path to a JWT secret to use for target engine API endpoint",
Category: flags.BeaconCategory,
}
// Transaction pool settings
TxPoolLocalsFlag = &cli.StringFlag{
Name: "txpool.locals",
@ -425,11 +478,6 @@ var (
}
// Miner settings
MiningEnabledFlag = &cli.BoolFlag{
Name: "mine",
Usage: "Enable mining",
Category: flags.MinerCategory,
}
MinerGasLimitFlag = &cli.Uint64Flag{
Name: "miner.gaslimit",
Usage: "Target gas ceiling for mined blocks",
@ -442,11 +490,6 @@ var (
Value: ethconfig.Defaults.Miner.GasPrice,
Category: flags.MinerCategory,
}
MinerEtherbaseFlag = &cli.StringFlag{
Name: "miner.etherbase",
Usage: "0x prefixed public address for block mining rewards",
Category: flags.MinerCategory,
}
MinerExtraDataFlag = &cli.StringFlag{
Name: "miner.extradata",
Usage: "Block extra data set by the miner (default = client version)",
@ -458,10 +501,9 @@ var (
Value: ethconfig.Defaults.Miner.Recommit,
Category: flags.MinerCategory,
}
MinerNewPayloadTimeout = &cli.DurationFlag{
Name: "miner.newpayload-timeout",
Usage: "Specify the maximum time allowance for creating a new payload",
Value: ethconfig.Defaults.Miner.NewPayloadTimeout,
MinerPendingFeeRecipientFlag = &cli.StringFlag{
Name: "miner.pending.feeRecipient",
Usage: "0x prefixed public address for the pending block producer (not used for actual block production)",
Category: flags.MinerCategory,
}
@ -1268,19 +1310,23 @@ func MakeAddress(ks *keystore.KeyStore, account string) (accounts.Account, error
// setEtherbase retrieves the etherbase from the directly specified command line flags.
func setEtherbase(ctx *cli.Context, cfg *ethconfig.Config) {
if !ctx.IsSet(MinerEtherbaseFlag.Name) {
if ctx.IsSet(MinerEtherbaseFlag.Name) {
log.Warn("Option --miner.etherbase is deprecated as the etherbase is set by the consensus client post-merge")
return
}
addr := ctx.String(MinerEtherbaseFlag.Name)
if !ctx.IsSet(MinerPendingFeeRecipientFlag.Name) {
return
}
addr := ctx.String(MinerPendingFeeRecipientFlag.Name)
if strings.HasPrefix(addr, "0x") || strings.HasPrefix(addr, "0X") {
addr = addr[2:]
}
b, err := hex.DecodeString(addr)
if err != nil || len(b) != common.AddressLength {
Fatalf("-%s: invalid etherbase address %q", MinerEtherbaseFlag.Name, addr)
Fatalf("-%s: invalid pending block producer address %q", MinerPendingFeeRecipientFlag.Name, addr)
return
}
cfg.Miner.Etherbase = common.BytesToAddress(b)
cfg.Miner.PendingFeeRecipient = common.BytesToAddress(b)
}
// MakePasswordList reads password lines from the file specified by the global --password flag.
@ -1496,6 +1542,9 @@ func setTxPool(ctx *cli.Context, cfg *legacypool.Config) {
}
func setMiner(ctx *cli.Context, cfg *miner.Config) {
if ctx.Bool(MiningEnabledFlag.Name) {
log.Warn("The flag --mine is deprecated and will be removed")
}
if ctx.IsSet(MinerExtraDataFlag.Name) {
cfg.ExtraData = []byte(ctx.String(MinerExtraDataFlag.Name))
}
@ -1508,8 +1557,9 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) {
if ctx.IsSet(MinerRecommitIntervalFlag.Name) {
cfg.Recommit = ctx.Duration(MinerRecommitIntervalFlag.Name)
}
if ctx.IsSet(MinerNewPayloadTimeout.Name) {
cfg.NewPayloadTimeout = ctx.Duration(MinerNewPayloadTimeout.Name)
if ctx.IsSet(MinerNewPayloadTimeoutFlag.Name) {
log.Warn("The flag --miner.newpayload-timeout is deprecated and will be removed, please use --miner.recommit")
cfg.Recommit = ctx.Duration(MinerNewPayloadTimeoutFlag.Name)
}
}
@ -1668,6 +1718,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
if ctx.String(GCModeFlag.Name) == "archive" && cfg.TransactionHistory != 0 {
cfg.TransactionHistory = 0
log.Warn("Disabled transaction unindexing for archive node")
cfg.StateScheme = rawdb.HashScheme
log.Warn("Forcing hash state-scheme for archive mode")
}
if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) {
cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100
@ -1783,8 +1836,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
// Figure out the dev account address.
// setEtherbase has been called above, configuring the miner address from command line flags.
if cfg.Miner.Etherbase != (common.Address{}) {
developer = accounts.Account{Address: cfg.Miner.Etherbase}
if cfg.Miner.PendingFeeRecipient != (common.Address{}) {
developer = accounts.Account{Address: cfg.Miner.PendingFeeRecipient}
} else if accs := ks.Accounts(); len(accs) > 0 {
developer = ks.Accounts()[0]
} else {
@ -1795,7 +1848,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
}
// Make sure the address is configured as fee recipient, otherwise
// the miner will fail to start.
cfg.Miner.Etherbase = developer.Address
cfg.Miner.PendingFeeRecipient = developer.Address
if err := ks.Unlock(developer, passphrase); err != nil {
Fatalf("Failed to unlock developer account: %v", err)

20
cmd/utils/flags_legacy.go
View file

@ -47,6 +47,9 @@ var DeprecatedFlags = []cli.Flag{
LightNoSyncServeFlag,
LogBacktraceAtFlag,
LogDebugFlag,
MinerNewPayloadTimeoutFlag,
MinerEtherbaseFlag,
MiningEnabledFlag,
}
var (
@ -132,6 +135,23 @@ var (
Usage: "Prepends log messages with call-site location (deprecated)",
Category: flags.DeprecatedCategory,
}
// Deprecated February 2024
MinerNewPayloadTimeoutFlag = &cli.DurationFlag{
Name: "miner.newpayload-timeout",
Usage: "Specify the maximum time allowance for creating a new payload",
Value: ethconfig.Defaults.Miner.Recommit,
Category: flags.MinerCategory,
}
MinerEtherbaseFlag = &cli.StringFlag{
Name: "miner.etherbase",
Usage: "0x prefixed public address for block mining rewards",
Category: flags.MinerCategory,
}
MiningEnabledFlag = &cli.BoolFlag{
Name: "mine",
Usage: "Enable mining",
Category: flags.MinerCategory,
}
)
// showDeprecated displays deprecated flags that will be soon removed from the codebase.

2
consensus/clique/clique.go
View file

@ -30,7 +30,7 @@ import (
"github.com/ethereum/go-ethereum/accounts"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
lru "github.com/ethereum/go-ethereum/common/lru"
"github.com/ethereum/go-ethereum/common/lru"
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/consensus/misc"
"github.com/ethereum/go-ethereum/consensus/misc/eip1559"

8
consensus/consensus.go
View file

@ -119,11 +119,3 @@ type Engine interface {
// Close terminates any background threads maintained by the consensus engine.
Close() error
}
// PoW is a consensus engine based on proof-of-work.
type PoW interface {
Engine
// Hashrate returns the current mining hashrate of a PoW consensus engine.
Hashrate() float64
}

110
consensus/merger.go
View file

@ -1,110 +0,0 @@
// Copyright 2021 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package consensus
import (
"fmt"
"sync"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/rlp"
)
// transitionStatus describes the status of eth1/2 transition. This switch
// between modes is a one-way action which is triggered by corresponding
// consensus-layer message.
type transitionStatus struct {
LeftPoW bool // The flag is set when the first NewHead message received
EnteredPoS bool // The flag is set when the first FinalisedBlock message received
}
// Merger is an internal help structure used to track the eth1/2 transition status.
// It's a common structure can be used in both full node and light client.
type Merger struct {
db ethdb.KeyValueStore
status transitionStatus
mu sync.RWMutex
}
// NewMerger creates a new Merger which stores its transition status in the provided db.
func NewMerger(db ethdb.KeyValueStore) *Merger {
var status transitionStatus
blob := rawdb.ReadTransitionStatus(db)
if len(blob) != 0 {
if err := rlp.DecodeBytes(blob, &status); err != nil {
log.Crit("Failed to decode the transition status", "err", err)
}
}
return &Merger{
db: db,
status: status,
}
}
// ReachTTD is called whenever the first NewHead message received
// from the consensus-layer.
func (m *Merger) ReachTTD() {
m.mu.Lock()
defer m.mu.Unlock()
if m.status.LeftPoW {
return
}
m.status = transitionStatus{LeftPoW: true}
blob, err := rlp.EncodeToBytes(m.status)
if err != nil {
panic(fmt.Sprintf("Failed to encode the transition status: %v", err))
}
rawdb.WriteTransitionStatus(m.db, blob)
log.Info("Left PoW stage")
}
// FinalizePoS is called whenever the first FinalisedBlock message received
// from the consensus-layer.
func (m *Merger) FinalizePoS() {
m.mu.Lock()
defer m.mu.Unlock()
if m.status.EnteredPoS {
return
}
m.status = transitionStatus{LeftPoW: true, EnteredPoS: true}
blob, err := rlp.EncodeToBytes(m.status)
if err != nil {
panic(fmt.Sprintf("Failed to encode the transition status: %v", err))
}
rawdb.WriteTransitionStatus(m.db, blob)
log.Info("Entered PoS stage")
}
// TDDReached reports whether the chain has left the PoW stage.
func (m *Merger) TDDReached() bool {
m.mu.RLock()
defer m.mu.RUnlock()
return m.status.LeftPoW
}
// PoSFinalized reports whether the chain has entered the PoS stage.
func (m *Merger) PoSFinalized() bool {
m.mu.RLock()
defer m.mu.RUnlock()
return m.status.EnteredPoS
}

3
console/console.go
View file

@ -325,9 +325,6 @@ func (c *Console) Welcome() {
// Print some generic Geth metadata
if res, err := c.jsre.Run(`
var message = "instance: " + web3.version.node + "\n";
try {
message += "coinbase: " + eth.coinbase + "\n";
} catch (err) {}
message += "at block: " + eth.blockNumber + " (" + new Date(1000 * eth.getBlock(eth.blockNumber).timestamp) + ")\n";
try {
message += " datadir: " + admin.datadir + "\n";

13
console/console_test.go
View file

@ -96,7 +96,7 @@ func newTester(t *testing.T, confOverride func(*ethconfig.Config)) *tester {
ethConf := &ethconfig.Config{
Genesis: core.DeveloperGenesisBlock(11_500_000, nil),
Miner: miner.Config{
Etherbase: common.HexToAddress(testAddress),
PendingFeeRecipient: common.HexToAddress(testAddress),
},
}
if confOverride != nil {
@ -152,8 +152,7 @@ func (env *tester) Close(t *testing.T) {
}
// Tests that the node lists the correct welcome message, notably that it contains
// the instance name, coinbase account, block number, data directory and supported
// console modules.
// the instance name, block number, data directory and supported console modules.
func TestWelcome(t *testing.T) {
tester := newTester(t, nil)
defer tester.Close(t)
@ -167,14 +166,14 @@ func TestWelcome(t *testing.T) {
if want := fmt.Sprintf("instance: %s", testInstance); !strings.Contains(output, want) {
t.Fatalf("console output missing instance: have\n%s\nwant also %s", output, want)
}
if want := fmt.Sprintf("coinbase: %s", testAddress); !strings.Contains(output, want) {
t.Fatalf("console output missing coinbase: have\n%s\nwant also %s", output, want)
}
if want := "at block: 0"; !strings.Contains(output, want) {
t.Fatalf("console output missing sync status: have\n%s\nwant also %s", output, want)
}
if want := fmt.Sprintf("datadir: %s", tester.workspace); !strings.Contains(output, want) {
t.Fatalf("console output missing coinbase: have\n%s\nwant also %s", output, want)
t.Fatalf("console output missing datadir: have\n%s\nwant also %s", output, want)
}
if want := "modules: "; !strings.Contains(output, want) {
t.Fatalf("console output missing modules: have\n%s\nwant also %s", output, want)
}
}

6
core/block_validator_test.go
View file

@ -94,7 +94,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) {
preBlocks []*types.Block
postBlocks []*types.Block
engine consensus.Engine
merger = consensus.NewMerger(rawdb.NewMemoryDatabase())
)
if isClique {
var (
@ -186,11 +185,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) {
}
chain.InsertChain(preBlocks[i : i+1])
}
// Make the transition
merger.ReachTTD()
merger.FinalizePoS()
// Verify the blocks after the merging
for i := 0; i < len(postBlocks); i++ {
_, results := engine.VerifyHeaders(chain, []*types.Header{postHeaders[i]})

108
core/blockchain.go
View file

@ -50,7 +50,6 @@ import (
"github.com/ethereum/go-ethereum/triedb"
"github.com/ethereum/go-ethereum/triedb/hashdb"
"github.com/ethereum/go-ethereum/triedb/pathdb"
"golang.org/x/exp/slices"
)
var (
@ -97,13 +96,11 @@ var (
)
const (
bodyCacheLimit = 256
blockCacheLimit = 256
receiptsCacheLimit = 32
txLookupCacheLimit = 1024
maxFutureBlocks = 256
maxTimeFutureBlocks = 30
TriesInMemory = 128
bodyCacheLimit = 256
blockCacheLimit = 256
receiptsCacheLimit = 32
txLookupCacheLimit = 1024
TriesInMemory = 128
// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
//
@ -245,9 +242,6 @@ type BlockChain struct {
blockCache *lru.Cache[common.Hash, *types.Block]
txLookupCache *lru.Cache[common.Hash, txLookup]
// future blocks are blocks added for later processing
futureBlocks *lru.Cache[common.Hash, *types.Block]
wg sync.WaitGroup
quit chan struct{} // shutdown signal, closed in Stop.
stopping atomic.Bool // false if chain is running, true when stopped
@ -299,7 +293,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
receiptsCache: lru.NewCache[common.Hash, []*types.Receipt](receiptsCacheLimit),
blockCache: lru.NewCache[common.Hash, *types.Block](blockCacheLimit),
txLookupCache: lru.NewCache[common.Hash, txLookup](txLookupCacheLimit),
futureBlocks: lru.NewCache[common.Hash, *types.Block](maxFutureBlocks),
engine: engine,
vmConfig: vmConfig,
}
@ -449,11 +442,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis
}
bc.snaps, _ = snapshot.New(snapconfig, bc.db, bc.triedb, head.Root)
}
// Start future block processor.
bc.wg.Add(1)
go bc.updateFutureBlocks()
// Rewind the chain in case of an incompatible config upgrade.
if compat, ok := genesisErr.(*params.ConfigCompatError); ok {
log.Warn("Rewinding chain to upgrade configuration", "err", compat)
@ -794,7 +782,6 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha
bc.receiptsCache.Purge()
bc.blockCache.Purge()
bc.txLookupCache.Purge()
bc.futureBlocks.Purge()
// Clear safe block, finalized block if needed
if safe := bc.CurrentSafeBlock(); safe != nil && head < safe.Number.Uint64() {
@ -1048,24 +1035,6 @@ func (bc *BlockChain) insertStopped() bool {
return bc.procInterrupt.Load()
}
func (bc *BlockChain) procFutureBlocks() {
blocks := make([]*types.Block, 0, bc.futureBlocks.Len())
for _, hash := range bc.futureBlocks.Keys() {
if block, exist := bc.futureBlocks.Peek(hash); exist {
blocks = append(blocks, block)
}
}
if len(blocks) > 0 {
slices.SortFunc(blocks, func(a, b *types.Block) int {
return a.Number().Cmp(b.Number())
})
// Insert one by one as chain insertion needs contiguous ancestry between blocks
for i := range blocks {
bc.InsertChain(blocks[i : i+1])
}
}
}
// WriteStatus status of write
type WriteStatus byte
@ -1466,8 +1435,6 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types
if status == CanonStatTy {
bc.writeHeadBlock(block)
}
bc.futureBlocks.Remove(block.Hash())
if status == CanonStatTy {
bc.chainFeed.Send(ChainEvent{Block: block, Hash: block.Hash(), Logs: logs})
if len(logs) > 0 {
@ -1487,25 +1454,6 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types
return status, nil
}
// addFutureBlock checks if the block is within the max allowed window to get
// accepted for future processing, and returns an error if the block is too far
// ahead and was not added.
//
// TODO after the transition, the future block shouldn't be kept. Because
// it's not checked in the Geth side anymore.
func (bc *BlockChain) addFutureBlock(block *types.Block) error {
max := uint64(time.Now().Unix() + maxTimeFutureBlocks)
if block.Time() > max {
return fmt.Errorf("future block timestamp %v > allowed %v", block.Time(), max)
}
if block.Difficulty().Cmp(common.Big0) == 0 {
// Never add PoS blocks into the future queue
return nil
}
bc.futureBlocks.Add(block.Hash(), block)
return nil
}
// InsertChain attempts to insert the given batch of blocks in to the canonical
// chain or, otherwise, create a fork. If an error is returned it will return
// the index number of the failing block as well an error describing what went
@ -1643,26 +1591,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
_, err := bc.recoverAncestors(block)
return it.index, err
}
// First block is future, shove it (and all children) to the future queue (unknown ancestor)
case errors.Is(err, consensus.ErrFutureBlock) || (errors.Is(err, consensus.ErrUnknownAncestor) && bc.futureBlocks.Contains(it.first().ParentHash())):
for block != nil && (it.index == 0 || errors.Is(err, consensus.ErrUnknownAncestor)) {
log.Debug("Future block, postponing import", "number", block.Number(), "hash", block.Hash())
if err := bc.addFutureBlock(block); err != nil {
return it.index, err
}
block, err = it.next()
}
stats.queued += it.processed()
stats.ignored += it.remaining()
// If there are any still remaining, mark as ignored
return it.index, err
// Some other error(except ErrKnownBlock) occurred, abort.
// ErrKnownBlock is allowed here since some known blocks
// still need re-execution to generate snapshots that are missing
case err != nil && !errors.Is(err, ErrKnownBlock):
bc.futureBlocks.Remove(block.Hash())
stats.ignored += len(it.chain)
bc.reportBlock(block, nil, err)
return it.index, err
@ -1867,23 +1799,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error)
"root", block.Root())
}
}
// Any blocks remaining here? The only ones we care about are the future ones
if block != nil && errors.Is(err, consensus.ErrFutureBlock) {
if err := bc.addFutureBlock(block); err != nil {
return it.index, err
}
block, err = it.next()
for ; block != nil && errors.Is(err, consensus.ErrUnknownAncestor); block, err = it.next() {
if err := bc.addFutureBlock(block); err != nil {
return it.index, err
}
stats.queued++
}
}
stats.ignored += it.remaining()
return it.index, err
}
@ -2334,20 +2250,6 @@ func (bc *BlockChain) SetCanonical(head *types.Block) (common.Hash, error) {
return head.Hash(), nil
}
func (bc *BlockChain) updateFutureBlocks() {
futureTimer := time.NewTicker(5 * time.Second)
defer futureTimer.Stop()
defer bc.wg.Done()
for {
select {
case <-futureTimer.C:
bc.procFutureBlocks()
case <-bc.quit:
return
}
}
}
// skipBlock returns 'true', if the block being imported can be skipped over, meaning
// that the block does not need to be processed but can be considered already fully 'done'.
func (bc *BlockChain) skipBlock(err error, it *insertIterator) bool {

5
core/blockchain_insert.go
View file

@ -179,8 +179,3 @@ func (it *insertIterator) first() *types.Block {
func (it *insertIterator) remaining() int {
return len(it.chain) - it.index
}
// processed returns the number of processed blocks.
func (it *insertIterator) processed() int {
return it.index + 1
}

10
core/blockchain_repair_test.go
View file

@ -1757,7 +1757,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme string) {
// It's hard to follow the test case, visualize the input
//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
// fmt.Println(tt.dump(true))
// Create a temporary persistent database
@ -1830,10 +1830,14 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
}
// Force run a freeze cycle
type freezer interface {
Freeze(threshold uint64) error
Freeze() error
Ancients() (uint64, error)
}
db.(freezer).Freeze(tt.freezeThreshold)
if tt.freezeThreshold < uint64(tt.canonicalBlocks) {
final := uint64(tt.canonicalBlocks) - tt.freezeThreshold
chain.SetFinalized(canonblocks[int(final)-1].Header())
}
db.(freezer).Freeze()
// Set the simulated pivot block
if tt.pivotBlock != nil {

8
core/blockchain_sethead_test.go
View file

@ -2044,10 +2044,14 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
// Force run a freeze cycle
type freezer interface {
Freeze(threshold uint64) error
Freeze() error
Ancients() (uint64, error)
}
db.(freezer).Freeze(tt.freezeThreshold)
if tt.freezeThreshold < uint64(tt.canonicalBlocks) {
final := uint64(tt.canonicalBlocks) - tt.freezeThreshold
chain.SetFinalized(canonblocks[int(final)-1].Header())
}
db.(freezer).Freeze()
// Set the simulated pivot block
if tt.pivotBlock != nil {

5
core/blockchain_test.go
View file

@ -2129,7 +2129,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon
// Generate a canonical chain to act as the main dataset
chainConfig := *params.TestChainConfig
var (
merger = consensus.NewMerger(rawdb.NewMemoryDatabase())
engine = beacon.New(ethash.NewFaker())
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
addr = crypto.PubkeyToAddress(key.PublicKey)
@ -2153,8 +2152,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon
// Activate the transition since genesis if required
if mergePoint == 0 {
mergeBlock = 0
merger.ReachTTD()
merger.FinalizePoS()
// Set the terminal total difficulty in the config
gspec.Config.TerminalTotalDifficulty = big.NewInt(0)
@ -2189,8 +2186,6 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon
// Activate the transition in the middle of the chain
if mergePoint == 1 {
merger.ReachTTD()
merger.FinalizePoS()
// Set the terminal total difficulty in the config
ttd := big.NewInt(int64(len(blocks)))
ttd.Mul(ttd, params.GenesisDifficulty)

8
core/rawdb/accessors_trie.go
View file

@ -315,7 +315,7 @@ func ReadStateScheme(db ethdb.Reader) string {
// the stored state.
//
// - If the provided scheme is none, use the scheme consistent with persistent
// state, or fallback to hash-based scheme if state is empty.
// state, or fallback to path-based scheme if state is empty.
//
// - If the provided scheme is hash, use hash-based scheme or error out if not
// compatible with persistent state scheme.
@ -329,10 +329,8 @@ func ParseStateScheme(provided string, disk ethdb.Database) (string, error) {
stored := ReadStateScheme(disk)
if provided == "" {
if stored == "" {
// use default scheme for empty database, flip it when
// path mode is chosen as default
log.Info("State schema set to default", "scheme", "hash")
return HashScheme, nil
log.Info("State schema set to default", "scheme", "path")
return PathScheme, nil // use default scheme for empty database
}
log.Info("State scheme set to already existing", "scheme", stored)
return stored, nil // reuse scheme of persistent scheme

113
core/rawdb/chain_freezer.go
View file

@ -17,9 +17,9 @@
package rawdb
import (
"errors"
"fmt"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
@ -43,8 +43,6 @@ const (
// The background thread will keep moving ancient chain segments from key-value
// database to flat files for saving space on live database.
type chainFreezer struct {
threshold atomic.Uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests)
*Freezer
quit chan struct{}
wg sync.WaitGroup
@ -57,13 +55,11 @@ func newChainFreezer(datadir string, namespace string, readonly bool) (*chainFre
if err != nil {
return nil, err
}
cf := chainFreezer{
return &chainFreezer{
Freezer: freezer,
quit: make(chan struct{}),
trigger: make(chan chan struct{}),
}
cf.threshold.Store(params.FullImmutabilityThreshold)
return &cf, nil
}, nil
}
// Close closes the chain freezer instance and terminates the background thread.
@ -77,6 +73,57 @@ func (f *chainFreezer) Close() error {
return f.Freezer.Close()
}
// readHeadNumber returns the number of chain head block. 0 is returned if the
// block is unknown or not available yet.
func (f *chainFreezer) readHeadNumber(db ethdb.KeyValueReader) uint64 {
hash := ReadHeadBlockHash(db)
if hash == (common.Hash{}) {
log.Error("Head block is not reachable")
return 0
}
number := ReadHeaderNumber(db, hash)
if number == nil {
log.Error("Number of head block is missing")
return 0
}
return *number
}
// readFinalizedNumber returns the number of finalized block. 0 is returned
// if the block is unknown or not available yet.
func (f *chainFreezer) readFinalizedNumber(db ethdb.KeyValueReader) uint64 {
hash := ReadFinalizedBlockHash(db)
if hash == (common.Hash{}) {
return 0
}
number := ReadHeaderNumber(db, hash)
if number == nil {
log.Error("Number of finalized block is missing")
return 0
}
return *number
}
// freezeThreshold returns the threshold for chain freezing. It's determined
// by formula: max(finality, HEAD-params.FullImmutabilityThreshold).
func (f *chainFreezer) freezeThreshold(db ethdb.KeyValueReader) (uint64, error) {
var (
head = f.readHeadNumber(db)
final = f.readFinalizedNumber(db)
headLimit uint64
)
if head > params.FullImmutabilityThreshold {
headLimit = head - params.FullImmutabilityThreshold
}
if final == 0 && headLimit == 0 {
return 0, errors.New("freezing threshold is not available")
}
if final > headLimit {
return final, nil
}
return headLimit, nil
}
// freeze is a background thread that periodically checks the blockchain for any
// import progress and moves ancient data from the fast database into the freezer.
//
@ -114,60 +161,39 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
return
}
}
// Retrieve the freezing threshold.
hash := ReadHeadBlockHash(nfdb)
if hash == (common.Hash{}) {
log.Debug("Current full block hash unavailable") // new chain, empty database
threshold, err := f.freezeThreshold(nfdb)
if err != nil {
backoff = true
log.Debug("Current full block not old enough to freeze", "err", err)
continue
}
number := ReadHeaderNumber(nfdb, hash)
threshold := f.threshold.Load()
frozen := f.frozen.Load()
switch {
case number == nil:
log.Error("Current full block number unavailable", "hash", hash)
backoff = true
continue
case *number < threshold:
log.Debug("Current full block not old enough to freeze", "number", *number, "hash", hash, "delay", threshold)
backoff = true
continue
case *number-threshold <= frozen:
log.Debug("Ancient blocks frozen already", "number", *number, "hash", hash, "frozen", frozen)
// Short circuit if the blocks below threshold are already frozen.
if frozen != 0 && frozen-1 >= threshold {
backoff = true
log.Debug("Ancient blocks frozen already", "threshold", threshold, "frozen", frozen)
continue
}
head := ReadHeader(nfdb, hash, *number)
if head == nil {
log.Error("Current full block unavailable", "number", *number, "hash", hash)
backoff = true
continue
}
// Seems we have data ready to be frozen, process in usable batches
var (
start = time.Now()
first, _ = f.Ancients()
limit = *number - threshold
start = time.Now()
first = frozen // the first block to freeze
last = threshold // the last block to freeze
)
if limit-first > freezerBatchLimit {
limit = first + freezerBatchLimit
if last-first+1 > freezerBatchLimit {
last = freezerBatchLimit + first - 1
}
ancients, err := f.freezeRange(nfdb, first, limit)
ancients, err := f.freezeRange(nfdb, first, last)
if err != nil {
log.Error("Error in block freeze operation", "err", err)
backoff = true
continue
}
// Batch of blocks have been frozen, flush them before wiping from leveldb
if err := f.Sync(); err != nil {
log.Crit("Failed to flush frozen tables", "err", err)
}
// Wipe out all data from the active database
batch := db.NewBatch()
for i := 0; i < len(ancients); i++ {
@ -250,8 +276,11 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
}
}
// freezeRange moves a batch of chain segments from the fast database to the freezer.
// The parameters (number, limit) specify the relevant block range, both of which
// are included.
func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes []common.Hash, err error) {
hashes = make([]common.Hash, 0, limit-number)
hashes = make([]common.Hash, 0, limit-number+1)
_, err = f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
for ; number <= limit; number++ {
@ -293,11 +322,9 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
if err := op.AppendRaw(ChainFreezerDifficultyTable, number, td); err != nil {
return fmt.Errorf("can't write td to Freezer: %v", err)
}
hashes = append(hashes, hash)
}
return nil
})
return hashes, err
}

8
core/rawdb/database.go
View file

@ -66,16 +66,10 @@ func (frdb *freezerdb) Close() error {
// Freeze is a helper method used for external testing to trigger and block until
// a freeze cycle completes, without having to sleep for a minute to trigger the
// automatic background run.
func (frdb *freezerdb) Freeze(threshold uint64) error {
func (frdb *freezerdb) Freeze() error {
if frdb.AncientStore.(*chainFreezer).readonly {
return errReadOnly
}
// Set the freezer threshold to a temporary value
defer func(old uint64) {
frdb.AncientStore.(*chainFreezer).threshold.Store(old)
}(frdb.AncientStore.(*chainFreezer).threshold.Load())
frdb.AncientStore.(*chainFreezer).threshold.Store(threshold)
// Trigger a freeze cycle and block until it's done
trigger := make(chan struct{}, 1)
frdb.AncientStore.(*chainFreezer).trigger <- trigger

30
core/rawdb/schema.go
View file

@ -113,8 +113,8 @@ var (
skeletonHeaderPrefix = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header
// Path-based storage scheme of merkle patricia trie.
trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node
trieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node
TrieNodeAccountPrefix = []byte("A") // TrieNodeAccountPrefix + hexPath -> trie node
TrieNodeStoragePrefix = []byte("O") // TrieNodeStoragePrefix + accountHash + hexPath -> trie node
stateIDPrefix = []byte("L") // stateIDPrefix + state root -> state id
PreimagePrefix = []byte("secure-key-") // PreimagePrefix + hash -> preimage
@ -265,15 +265,15 @@ func stateIDKey(root common.Hash) []byte {
return append(stateIDPrefix, root.Bytes()...)
}
// accountTrieNodeKey = trieNodeAccountPrefix + nodePath.
// accountTrieNodeKey = TrieNodeAccountPrefix + nodePath.
func accountTrieNodeKey(path []byte) []byte {
return append(trieNodeAccountPrefix, path...)
return append(TrieNodeAccountPrefix, path...)
}
// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath.
// storageTrieNodeKey = TrieNodeStoragePrefix + accountHash + nodePath.
func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte {
buf := make([]byte, len(trieNodeStoragePrefix)+common.HashLength+len(path))
n := copy(buf, trieNodeStoragePrefix)
buf := make([]byte, len(TrieNodeStoragePrefix)+common.HashLength+len(path))
n := copy(buf, TrieNodeStoragePrefix)
n += copy(buf[n:], accountHash.Bytes())
copy(buf[n:], path)
return buf
@ -294,16 +294,16 @@ func IsLegacyTrieNode(key []byte, val []byte) bool {
// account trie node in path-based state scheme, and returns the resolved
// node path if so.
func ResolveAccountTrieNodeKey(key []byte) (bool, []byte) {
if !bytes.HasPrefix(key, trieNodeAccountPrefix) {
if !bytes.HasPrefix(key, TrieNodeAccountPrefix) {
return false, nil
}
// The remaining key should only consist a hex node path
// whose length is in the range 0 to 64 (64 is excluded
// since leaves are always wrapped with shortNode).
if len(key) >= len(trieNodeAccountPrefix)+common.HashLength*2 {
if len(key) >= len(TrieNodeAccountPrefix)+common.HashLength*2 {
return false, nil
}
return true, key[len(trieNodeAccountPrefix):]
return true, key[len(TrieNodeAccountPrefix):]
}
// IsAccountTrieNode reports whether a provided database entry is an account
@ -317,20 +317,20 @@ func IsAccountTrieNode(key []byte) bool {
// trie node in path-based state scheme, and returns the resolved account hash
// and node path if so.
func ResolveStorageTrieNode(key []byte) (bool, common.Hash, []byte) {
if !bytes.HasPrefix(key, trieNodeStoragePrefix) {
if !bytes.HasPrefix(key, TrieNodeStoragePrefix) {
return false, common.Hash{}, nil
}
// The remaining key consists of 2 parts:
// - 32 bytes account hash
// - hex node path whose length is in the range 0 to 64
if len(key) < len(trieNodeStoragePrefix)+common.HashLength {
if len(key) < len(TrieNodeStoragePrefix)+common.HashLength {
return false, common.Hash{}, nil
}
if len(key) >= len(trieNodeStoragePrefix)+common.HashLength+common.HashLength*2 {
if len(key) >= len(TrieNodeStoragePrefix)+common.HashLength+common.HashLength*2 {
return false, common.Hash{}, nil
}
accountHash := common.BytesToHash(key[len(trieNodeStoragePrefix) : len(trieNodeStoragePrefix)+common.HashLength])
return true, accountHash, key[len(trieNodeStoragePrefix)+common.HashLength:]
accountHash := common.BytesToHash(key[len(TrieNodeStoragePrefix) : len(TrieNodeStoragePrefix)+common.HashLength])
return true, accountHash, key[len(TrieNodeStoragePrefix)+common.HashLength:]
}
// IsStorageTrieNode reports whether a provided database entry is a storage

1
core/state/metrics.go
View file

@ -33,5 +33,4 @@ var (
slotDeletionTimer = metrics.NewRegisteredResettingTimer("state/delete/storage/timer", nil)
slotDeletionCount = metrics.NewRegisteredMeter("state/delete/storage/slot", nil)
slotDeletionSize = metrics.NewRegisteredMeter("state/delete/storage/size", nil)
slotDeletionSkip = metrics.NewRegisteredGauge("state/delete/storage/skip", nil)
)

87
core/state/statedb.go
View file

@ -36,12 +36,6 @@ import (
"github.com/holiman/uint256"
)
const (
// storageDeleteLimit denotes the highest permissible memory allocation
// employed for contract storage deletion.
storageDeleteLimit = 512 * 1024 * 1024
)
type revision struct {
id int
journalIndex int
@ -949,10 +943,10 @@ func (s *StateDB) clearJournalAndRefund() {
// of a specific account. It leverages the associated state snapshot for fast
// storage iteration and constructs trie node deletion markers by creating
// stack trie with iterated slots.
func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) {
func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) {
iter, err := s.snaps.StorageIterator(s.originalRoot, addrHash, common.Hash{})
if err != nil {
return false, 0, nil, nil, err
return 0, nil, nil, err
}
defer iter.Release()
@ -968,40 +962,37 @@ func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (boo
})
stack := trie.NewStackTrie(options)
for iter.Next() {
if size > storageDeleteLimit {
return true, size, nil, nil, nil
}
slot := common.CopyBytes(iter.Slot())
if err := iter.Error(); err != nil { // error might occur after Slot function
return false, 0, nil, nil, err
return 0, nil, nil, err
}
size += common.StorageSize(common.HashLength + len(slot))
slots[iter.Hash()] = slot
if err := stack.Update(iter.Hash().Bytes(), slot); err != nil {
return false, 0, nil, nil, err
return 0, nil, nil, err
}
}
if err := iter.Error(); err != nil { // error might occur during iteration
return false, 0, nil, nil, err
return 0, nil, nil, err
}
if stack.Hash() != root {
return false, 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash())
return 0, nil, nil, fmt.Errorf("snapshot is not matched, exp %x, got %x", root, stack.Hash())
}
return false, size, slots, nodes, nil
return size, slots, nodes, nil
}
// slowDeleteStorage serves as a less-efficient alternative to "fastDeleteStorage,"
// employed when the associated state snapshot is not available. It iterates the
// storage slots along with all internal trie nodes via trie directly.
func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) {
func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) {
tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie)
if err != nil {
return false, 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err)
return 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err)
}
it, err := tr.NodeIterator(nil)
if err != nil {
return false, 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err)
return 0, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err)
}
var (
size common.StorageSize
@ -1009,9 +1000,6 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r
slots = make(map[common.Hash][]byte)
)
for it.Next(true) {
if size > storageDeleteLimit {
return true, size, nil, nil, nil
}
if it.Leaf() {
slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob())
size += common.StorageSize(common.HashLength + len(it.LeafBlob()))
@ -1024,9 +1012,9 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r
nodes.AddNode(it.Path(), trienode.NewDeleted())
}
if err := it.Error(); err != nil {
return false, 0, nil, nil, err
return 0, nil, nil, err
}
return false, size, slots, nodes, nil
return size, slots, nodes, nil
}
// deleteStorage is designed to delete the storage trie of a designated account.
@ -1034,31 +1022,27 @@ func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, r
// potentially leading to an out-of-memory panic. The function will make an attempt
// to utilize an efficient strategy if the associated state snapshot is reachable;
// otherwise, it will resort to a less-efficient approach.
func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, map[common.Hash][]byte, *trienode.NodeSet, error) {
func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (map[common.Hash][]byte, *trienode.NodeSet, error) {
var (
start = time.Now()
err error
aborted bool
size common.StorageSize
slots map[common.Hash][]byte
nodes *trienode.NodeSet
start = time.Now()
err error
size common.StorageSize
slots map[common.Hash][]byte
nodes *trienode.NodeSet
)
// The fast approach can be failed if the snapshot is not fully
// generated, or it's internally corrupted. Fallback to the slow
// one just in case.
if s.snap != nil {
aborted, size, slots, nodes, err = s.fastDeleteStorage(addrHash, root)
size, slots, nodes, err = s.fastDeleteStorage(addrHash, root)
}
if s.snap == nil || err != nil {
aborted, size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root)
size, slots, nodes, err = s.slowDeleteStorage(addr, addrHash, root)
}
if err != nil {
return false, nil, nil, err
return nil, nil, err
}
if metrics.EnabledExpensive {
if aborted {
slotDeletionSkip.Inc(1)
}
n := int64(len(slots))
slotDeletionMaxCount.UpdateIfGt(int64(len(slots)))
@ -1068,7 +1052,7 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root
slotDeletionCount.Mark(n)
slotDeletionSize.Mark(int64(size))
}
return aborted, slots, nodes, nil
return slots, nodes, nil
}
// handleDestruction processes all destruction markers and deletes the account
@ -1095,13 +1079,12 @@ func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root
//
// In case (d), **original** account along with its storages should be deleted,
// with their values be tracked as original value.
func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.Address]struct{}, error) {
func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) error {
// Short circuit if geth is running with hash mode. This procedure can consume
// considerable time and storage deletion isn't supported in hash mode, thus
// preemptively avoiding unnecessary expenses.
incomplete := make(map[common.Address]struct{})
if s.db.TrieDB().Scheme() == rawdb.HashScheme {
return incomplete, nil
return nil
}
for addr, prev := range s.stateObjectsDestruct {
// The original account was non-existing, and it's marked as destructed
@ -1124,18 +1107,9 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A
continue
}
// Remove storage slots belong to the account.
aborted, slots, set, err := s.deleteStorage(addr, addrHash, prev.Root)
slots, set, err := s.deleteStorage(addr, addrHash, prev.Root)
if err != nil {
return nil, fmt.Errorf("failed to delete storage, err: %w", err)
}
// The storage is too huge to handle, skip it but mark as incomplete.
// For case (d), the account is resurrected might with a few slots
// created. In this case, wipe the entire storage state diff because
// of aborted deletion.
if aborted {
incomplete[addr] = struct{}{}
delete(s.storagesOrigin, addr)
continue
return fmt.Errorf("failed to delete storage, err: %w", err)
}
if s.storagesOrigin[addr] == nil {
s.storagesOrigin[addr] = slots
@ -1147,10 +1121,10 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.A
}
}
if err := nodes.Merge(set); err != nil {
return nil, err
return err
}
}
return incomplete, nil
return nil
}
// Commit writes the state to the underlying in-memory trie database.
@ -1179,8 +1153,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er
codeWriter = s.db.DiskDB().NewBatch()
)
// Handle all state deletions first
incomplete, err := s.handleDestruction(nodes)
if err != nil {
if err := s.handleDestruction(nodes); err != nil {
return common.Hash{}, err
}
// Handle all state updates afterwards
@ -1276,7 +1249,7 @@ func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, er
}
if root != origin {
start := time.Now()
set := triestate.New(s.accountsOrigin, s.storagesOrigin, incomplete)
set := triestate.New(s.accountsOrigin, s.storagesOrigin)
if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil {
return common.Hash{}, err
}

4
core/state/statedb_test.go
View file

@ -1161,12 +1161,12 @@ func TestDeleteStorage(t *testing.T) {
obj := fastState.getOrNewStateObject(addr)
storageRoot := obj.data.Root
_, _, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
_, fastNodes, err := fastState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
if err != nil {
t.Fatal(err)
}
_, _, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
_, slowNodes, err := slowState.deleteStorage(addr, crypto.Keccak256Hash(addr[:]), storageRoot)
if err != nil {
t.Fatal(err)
}

2
core/state/sync.go
View file

@ -24,7 +24,7 @@ import (
"github.com/ethereum/go-ethereum/trie"
)
// NewStateSync create a new state trie download scheduler.
// NewStateSync creates a new state trie download scheduler.
func NewStateSync(root common.Hash, database ethdb.KeyValueReader, onLeaf func(keys [][]byte, leaf []byte) error, scheme string) *trie.Sync {
// Register the storage slot callback if the external callback is specified.
var onSlot func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error

8
core/state_transition.go
View file

@ -67,7 +67,7 @@ func (result *ExecutionResult) Revert() []byte {
}
// IntrinsicGas computes the 'intrinsic gas' for a message with the given data.
func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028 bool, isEIP3860 bool) (uint64, error) {
func IntrinsicGas(data []byte, accessList types.AccessList, isContractCreation bool, isHomestead, isEIP2028, isEIP3860 bool) (uint64, error) {
// Set the starting gas for the raw transaction
var gas uint64
if isContractCreation && isHomestead {
@ -234,7 +234,7 @@ func (st *StateTransition) to() common.Address {
func (st *StateTransition) buyGas() error {
mgval := new(big.Int).SetUint64(st.msg.GasLimit)
mgval = mgval.Mul(mgval, st.msg.GasPrice)
mgval.Mul(mgval, st.msg.GasPrice)
balanceCheck := new(big.Int).Set(mgval)
if st.msg.GasFeeCap != nil {
balanceCheck.SetUint64(st.msg.GasLimit)
@ -263,7 +263,7 @@ func (st *StateTransition) buyGas() error {
if err := st.gp.SubGas(st.msg.GasLimit); err != nil {
return err
}
st.gasRemaining += st.msg.GasLimit
st.gasRemaining = st.msg.GasLimit
st.initialGas = st.msg.GasLimit
mgvalU256, _ := uint256.FromBig(mgval)
@ -477,7 +477,7 @@ func (st *StateTransition) refundGas(refundQuotient uint64) uint64 {
// Return ETH for remaining gas, exchanged at the original rate.
remaining := uint256.NewInt(st.gasRemaining)
remaining = remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice))
remaining.Mul(remaining, uint256.MustFromBig(st.msg.GasPrice))
st.state.AddBalance(st.msg.From, remaining)
// Also return remaining gas to the block gas counter so it is

52
core/txpool/blobpool/blobpool.go
View file

@ -360,7 +360,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres
}
}
// Initialize the state with head block, or fallback to empty one in
// case the head state is not available(might occur when node is not
// case the head state is not available (might occur when node is not
// fully synced).
state, err := p.chain.StateAt(head.Root)
if err != nil {
@ -371,7 +371,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres
}
p.head, p.state = head, state
// Index all transactions on disk and delete anything inprocessable
// Index all transactions on disk and delete anything unprocessable
var fails []uint64
index := func(id uint64, size uint32, blob []byte) {
if p.parseTransaction(id, size, blob) != nil {
@ -402,7 +402,7 @@ func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.Addres
}
var (
basefee = uint256.MustFromBig(eip1559.CalcBaseFee(p.chain.Config(), p.head))
blobfee = uint256.MustFromBig(big.NewInt(params.BlobTxMinBlobGasprice))
blobfee = uint256.NewInt(params.BlobTxMinBlobGasprice)
)
if p.head.ExcessBlobGas != nil {
blobfee = uint256.MustFromBig(eip4844.CalcBlobFee(*p.head.ExcessBlobGas))
@ -540,7 +540,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
}
delete(p.index, addr)
delete(p.spent, addr)
if inclusions != nil { // only during reorgs will the heap will be initialized
if inclusions != nil { // only during reorgs will the heap be initialized
heap.Remove(p.evict, p.evict.index[addr])
}
p.reserve(addr, false)
@ -693,7 +693,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6
if len(txs) == 0 {
delete(p.index, addr)
delete(p.spent, addr)
if inclusions != nil { // only during reorgs will the heap will be initialized
if inclusions != nil { // only during reorgs will the heap be initialized
heap.Remove(p.evict, p.evict.index[addr])
}
p.reserve(addr, false)
@ -809,7 +809,7 @@ func (p *BlobPool) Reset(oldHead, newHead *types.Header) {
}
}
// Recheck the account's pooled transactions to drop included and
// invalidated one
// invalidated ones
p.recheck(addr, inclusions)
}
if len(adds) > 0 {
@ -1226,7 +1226,7 @@ func (p *BlobPool) Add(txs []*types.Transaction, local bool, sync bool) []error
// consensus validity and pool restrictions).
func (p *BlobPool) add(tx *types.Transaction) (err error) {
// The blob pool blocks on adding a transaction. This is because blob txs are
// only even pulled form the network, so this method will act as the overload
// only even pulled from the network, so this method will act as the overload
// protection for fetches.
waitStart := time.Now()
p.lock.Lock()
@ -1446,7 +1446,12 @@ func (p *BlobPool) drop() {
//
// The transactions can also be pre-filtered by the dynamic fee components to
// reduce allocations and load on downstream subsystems.
func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction {
func (p *BlobPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction {
// If only plain transactions are requested, this pool is unsuitable as it
// contains none, don't even bother.
if filter.OnlyPlainTxs {
return nil
}
// Track the amount of time waiting to retrieve the list of pending blob txs
// from the pool and the amount of time actually spent on assembling the data.
// The latter will be pretty much moot, but we've kept it to have symmetric
@ -1456,29 +1461,30 @@ func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *u
pendwaitHist.Update(time.Since(pendStart).Nanoseconds())
defer p.lock.RUnlock()
defer func(start time.Time) {
pendtimeHist.Update(time.Since(start).Nanoseconds())
}(time.Now())
execStart := time.Now()
defer func() {
pendtimeHist.Update(time.Since(execStart).Nanoseconds())
}()
pending := make(map[common.Address][]*txpool.LazyTransaction)
pending := make(map[common.Address][]*txpool.LazyTransaction, len(p.index))
for addr, txs := range p.index {
var lazies []*txpool.LazyTransaction
lazies := make([]*txpool.LazyTransaction, 0, len(txs))
for _, tx := range txs {
// If transaction filtering was requested, discard badly priced ones
if minTip != nil && baseFee != nil {
if tx.execFeeCap.Lt(baseFee) {
if filter.MinTip != nil && filter.BaseFee != nil {
if tx.execFeeCap.Lt(filter.BaseFee) {
break // basefee too low, cannot be included, discard rest of txs from the account
}
tip := new(uint256.Int).Sub(tx.execFeeCap, baseFee)
tip := new(uint256.Int).Sub(tx.execFeeCap, filter.BaseFee)
if tip.Gt(tx.execTipCap) {
tip = tx.execTipCap
}
if tip.Lt(minTip) {
if tip.Lt(filter.MinTip) {
break // allowed or remaining tip too low, cannot be included, discard rest of txs from the account
}
}
if blobFee != nil {
if tx.blobFeeCap.Lt(blobFee) {
if filter.BlobFee != nil {
if tx.blobFeeCap.Lt(filter.BlobFee) {
break // blobfee too low, cannot be included, discard rest of txs from the account
}
}
@ -1486,9 +1492,9 @@ func (p *BlobPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *u
lazies = append(lazies, &txpool.LazyTransaction{
Pool: p,
Hash: tx.hash,
Time: time.Now(), // TODO(karalabe): Maybe save these and use that?
GasFeeCap: tx.execFeeCap.ToBig(),
GasTipCap: tx.execTipCap.ToBig(),
Time: execStart, // TODO(karalabe): Maybe save these and use that?
GasFeeCap: tx.execFeeCap,
GasTipCap: tx.execTipCap,
Gas: tx.execGas,
BlobGas: tx.blobGas,
})
@ -1548,7 +1554,7 @@ func (p *BlobPool) updateStorageMetrics() {
}
// updateLimboMetrics retrieves a bunch of stats from the limbo store and pushes
// // them out as metrics.
// them out as metrics.
func (p *BlobPool) updateLimboMetrics() {
stats := p.limbo.store.Infos()

84
core/txpool/blobpool/blobpool_test.go
View file

@ -185,7 +185,7 @@ func makeTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64,
return types.MustSignNewTx(key, types.LatestSigner(testChainConfig), blobtx)
}
// makeUnsignedTx is a utility method to construct a random blob tranasaction
// makeUnsignedTx is a utility method to construct a random blob transaction
// without signing it.
func makeUnsignedTx(nonce uint64, gasTipCap uint64, gasFeeCap uint64, blobFeeCap uint64) *types.BlobTx {
return &types.BlobTx{
@ -391,7 +391,7 @@ func TestOpenDrops(t *testing.T) {
id, _ := store.Put(blob)
filled[id] = struct{}{}
}
// Insert a sequence of transactions with partially passed nonces to veirfy
// Insert a sequence of transactions with partially passed nonces to verify
// that the included part of the set will get dropped (case 4).
var (
overlapper, _ = crypto.GenerateKey()
@ -1228,6 +1228,24 @@ func TestAdd(t *testing.T) {
},
},
},
// Blob transactions that don't meet the min blob gas price should be rejected
{
seeds: map[string]seed{
"alice": {balance: 10000000},
},
adds: []addtx{
{ // New account, no previous txs, nonce 0, but blob fee cap too low
from: "alice",
tx: makeUnsignedTx(0, 1, 1, 0),
err: txpool.ErrUnderpriced,
},
{ // Same as above but blob fee cap equals minimum, should be accepted
from: "alice",
tx: makeUnsignedTx(0, 1, 1, params.BlobTxMinBlobGasprice),
err: nil,
},
},
},
}
for i, tt := range tests {
// Create a temporary folder for the persistent backend
@ -1288,3 +1306,65 @@ func TestAdd(t *testing.T) {
pool.Close()
}
}
// Benchmarks the time it takes to assemble the lazy pending transaction list
// from the pool contents.
func BenchmarkPoolPending100Mb(b *testing.B) { benchmarkPoolPending(b, 100_000_000) }
func BenchmarkPoolPending1GB(b *testing.B) { benchmarkPoolPending(b, 1_000_000_000) }
func BenchmarkPoolPending10GB(b *testing.B) { benchmarkPoolPending(b, 10_000_000_000) }
func benchmarkPoolPending(b *testing.B, datacap uint64) {
// Calculate the maximum number of transaction that would fit into the pool
// and generate a set of random accounts to seed them with.
capacity := datacap / params.BlobTxBlobGasPerBlob
var (
basefee = uint64(1050)
blobfee = uint64(105)
signer = types.LatestSigner(testChainConfig)
statedb, _ = state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewDatabase(memorydb.New())), nil)
chain = &testBlockChain{
config: testChainConfig,
basefee: uint256.NewInt(basefee),
blobfee: uint256.NewInt(blobfee),
statedb: statedb,
}
pool = New(Config{Datadir: ""}, chain)
)
if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil {
b.Fatalf("failed to create blob pool: %v", err)
}
// Fill the pool up with one random transaction from each account with the
// same price and everything to maximize the worst case scenario
for i := 0; i < int(capacity); i++ {
blobtx := makeUnsignedTx(0, 10, basefee+10, blobfee)
blobtx.R = uint256.NewInt(1)
blobtx.S = uint256.NewInt(uint64(100 + i))
blobtx.V = uint256.NewInt(0)
tx := types.NewTx(blobtx)
addr, err := types.Sender(signer, tx)
if err != nil {
b.Fatal(err)
}
statedb.AddBalance(addr, uint256.NewInt(1_000_000_000))
pool.add(tx)
}
statedb.Commit(0, true)
defer pool.Close()
// Benchmark assembling the pending
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
p := pool.Pending(txpool.PendingFilter{
MinTip: uint256.NewInt(1),
BaseFee: chain.basefee,
BlobFee: chain.blobfee,
})
if len(p) != int(capacity) {
b.Fatalf("have %d want %d", len(p), capacity)
}
}
}

4
core/txpool/blobpool/config.go
View file

@ -30,8 +30,8 @@ type Config struct {
// DefaultConfig contains the default configurations for the transaction pool.
var DefaultConfig = Config{
Datadir: "blobpool",
Datacap: 10 * 1024 * 1024 * 1024,
PriceBump: 100, // either have patience or be aggressive, no mushy ground
Datacap: 10 * 1024 * 1024 * 1024 / 4, // TODO(karalabe): /4 handicap for rollout, gradually bump back up to 10GB
PriceBump: 100, // either have patience or be aggressive, no mushy ground
}
// sanitize checks the provided user configurations and changes anything that's

2
core/txpool/blobpool/evictheap.go
View file

@ -30,7 +30,7 @@ import (
// transaction from each account to determine which account to evict from.
//
// The heap internally tracks a slice of cheapest transactions from each account
// and a mapping from addresses to indices for direct removals/udates.
// and a mapping from addresses to indices for direct removals/updates.
//
// The goal of the heap is to decide which account has the worst bottleneck to
// evict transactions from.

2
core/txpool/blobpool/priority_test.go
View file

@ -64,7 +64,7 @@ func BenchmarkDynamicFeeJumpCalculation(b *testing.B) {
// Benchmarks how many priority recalculations can be done.
func BenchmarkPriorityCalculation(b *testing.B) {
// The basefee and blob fee is constant for all transactions across a block,
// so we can assume theit absolute jump counts can be pre-computed.
// so we can assume their absolute jump counts can be pre-computed.
basefee := uint256.NewInt(17_200_000_000) // 17.2 Gwei is the 22.03.2023 zero-emission basefee, random number
blobfee := uint256.NewInt(123_456_789_000) // Completely random, no idea what this will be

6
core/txpool/errors.go
View file

@ -54,4 +54,10 @@ var (
// ErrFutureReplacePending is returned if a future transaction replaces a pending
// one. Future transactions should only be able to replace other future transactions.
ErrFutureReplacePending = errors.New("future transaction tries to replace pending")
// ErrAlreadyReserved is returned if the sender address has a pending transaction
// in a different subpool. For example, this error is returned in response to any
// input transaction of non-blob type when a blob transaction from this sender
// remains pending (and vice-versa).
ErrAlreadyReserved = errors.New("address already reserved")
)

7
core/txpool/legacypool/journal.go
View file

@ -164,7 +164,12 @@ func (journal *journal) rotate(all map[common.Address]types.Transactions) error
return err
}
journal.writer = sink
log.Info("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all))
logger := log.Info
if len(all) == 0 {
logger = log.Debug
}
logger("Regenerated local transaction journal", "transactions", journaled, "accounts", len(all))
return nil
}

22
core/txpool/legacypool/legacypool.go
View file

@ -296,7 +296,7 @@ func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserve txpool.A
pool.gasTip.Store(uint256.NewInt(gasTip))
// Initialize the state with head block, or fallback to empty one in
// case the head state is not available(might occur when node is not
// case the head state is not available (might occur when node is not
// fully synced).
statedb, err := pool.chain.StateAt(head.Root)
if err != nil {
@ -522,7 +522,12 @@ func (pool *LegacyPool) ContentFrom(addr common.Address) ([]*types.Transaction,
//
// The transactions can also be pre-filtered by the dynamic fee components to
// reduce allocations and load on downstream subsystems.
func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*txpool.LazyTransaction {
func (pool *LegacyPool) Pending(filter txpool.PendingFilter) map[common.Address][]*txpool.LazyTransaction {
// If only blob transactions are requested, this pool is unsuitable as it
// contains none, don't even bother.
if filter.OnlyBlobTxs {
return nil
}
pool.mu.Lock()
defer pool.mu.Unlock()
@ -531,13 +536,12 @@ func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobF
minTipBig *big.Int
baseFeeBig *big.Int
)
if minTip != nil {
minTipBig = minTip.ToBig()
if filter.MinTip != nil {
minTipBig = filter.MinTip.ToBig()
}
if baseFee != nil {
baseFeeBig = baseFee.ToBig()
if filter.BaseFee != nil {
baseFeeBig = filter.BaseFee.ToBig()
}
pending := make(map[common.Address][]*txpool.LazyTransaction, len(pool.pending))
for addr, list := range pool.pending {
txs := list.Flatten()
@ -559,8 +563,8 @@ func (pool *LegacyPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobF
Hash: txs[i].Hash(),
Tx: txs[i],
Time: txs[i].Time(),
GasFeeCap: txs[i].GasFeeCap(),
GasTipCap: txs[i].GasTipCap(),
GasFeeCap: uint256.MustFromBig(txs[i].GasFeeCap()),
GasTipCap: uint256.MustFromBig(txs[i].GasTipCap()),
Gas: txs[i].Gas(),
BlobGas: txs[i].BlobGas(),
}

2
core/txpool/legacypool/list.go
View file

@ -278,7 +278,7 @@ type list struct {
totalcost *uint256.Int // Total cost of all transactions in the list
}
// newList create a new transaction list for maintaining nonce-indexable fast,
// newList creates a new transaction list for maintaining nonce-indexable fast,
// gapped, sortable transaction lists.
func newList(strict bool) *list {
return &list{

23
core/txpool/subpool.go
View file

@ -35,9 +35,9 @@ type LazyTransaction struct {
Hash common.Hash // Transaction hash to pull up if needed
Tx *types.Transaction // Transaction if already resolved
Time time.Time // Time when the transaction was first seen
GasFeeCap *big.Int // Maximum fee per gas the transaction may consume
GasTipCap *big.Int // Maximum miner tip per gas the transaction can pay
Time time.Time // Time when the transaction was first seen
GasFeeCap *uint256.Int // Maximum fee per gas the transaction may consume
GasTipCap *uint256.Int // Maximum miner tip per gas the transaction can pay
Gas uint64 // Amount of gas required by the transaction
BlobGas uint64 // Amount of blob gas required by the transaction
@ -70,6 +70,21 @@ type LazyResolver interface {
// may request (and relinquish) exclusive access to certain addresses.
type AddressReserver func(addr common.Address, reserve bool) error
// PendingFilter is a collection of filter rules to allow retrieving a subset
// of transactions for announcement or mining.
//
// Note, the entries here are not arbitrary useful filters, rather each one has
// a very specific call site in mind and each one can be evaluated very cheaply
// by the pool implementations. Only add new ones that satisfy those constraints.
type PendingFilter struct {
MinTip *uint256.Int // Minimum miner tip required to include a transaction
BaseFee *uint256.Int // Minimum 1559 basefee needed to include a transaction
BlobFee *uint256.Int // Minimum 4844 blobfee needed to include a blob transaction
OnlyPlainTxs bool // Return only plain EVM transactions (peer-join announces, block space filling)
OnlyBlobTxs bool // Return only blob transactions (block blob-space filling)
}
// SubPool represents a specialized transaction pool that lives on its own (e.g.
// blob pool). Since independent of how many specialized pools we have, they do
// need to be updated in lockstep and assemble into one coherent view for block
@ -118,7 +133,7 @@ type SubPool interface {
//
// The transactions can also be pre-filtered by the dynamic fee components to
// reduce allocations and load on downstream subsystems.
Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*LazyTransaction
Pending(filter PendingFilter) map[common.Address][]*LazyTransaction
// SubscribeTransactions subscribes to new transaction events. The subscriber
// can decide whether to receive notifications only for newly seen transactions

7
core/txpool/txpool.go
View file

@ -28,7 +28,6 @@ import (
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/holiman/uint256"
)
// TxStatus is the current status of a transaction as seen by the pool.
@ -123,7 +122,7 @@ func (p *TxPool) reserver(id int, subpool SubPool) AddressReserver {
log.Error("pool attempted to reserve already-owned address", "address", addr)
return nil // Ignore fault to give the pool a chance to recover while the bug gets fixed
}
return errors.New("address already reserved")
return ErrAlreadyReserved
}
p.reservations[addr] = subpool
if metrics.Enabled {
@ -357,10 +356,10 @@ func (p *TxPool) Add(txs []*types.Transaction, local bool, sync bool) []error {
//
// The transactions can also be pre-filtered by the dynamic fee components to
// reduce allocations and load on downstream subsystems.
func (p *TxPool) Pending(minTip *uint256.Int, baseFee *uint256.Int, blobFee *uint256.Int) map[common.Address][]*LazyTransaction {
func (p *TxPool) Pending(filter PendingFilter) map[common.Address][]*LazyTransaction {
txs := make(map[common.Address][]*LazyTransaction)
for _, subpool := range p.subpools {
for addr, set := range subpool.Pending(minTip, baseFee, blobFee) {
for addr, set := range subpool.Pending(filter) {
txs[addr] = set
}
}

24
core/txpool/validation.go
View file

@ -18,6 +18,7 @@ package txpool
import (
"crypto/sha256"
"errors"
"fmt"
"math/big"
@ -30,6 +31,12 @@ import (
"github.com/ethereum/go-ethereum/params"
)
var (
// blobTxMinBlobGasPrice is the big.Int version of the configured protocol
// parameter to avoid constucting a new big integer for every transaction.
blobTxMinBlobGasPrice = big.NewInt(params.BlobTxMinBlobGasprice)
)
// ValidationOptions define certain differences between transaction validation
// across the different pools without having to duplicate those checks.
type ValidationOptions struct {
@ -101,28 +108,31 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types
return err
}
if tx.Gas() < intrGas {
return fmt.Errorf("%w: needed %v, allowed %v", core.ErrIntrinsicGas, intrGas, tx.Gas())
return fmt.Errorf("%w: gas %v, minimum needed %v", core.ErrIntrinsicGas, tx.Gas(), intrGas)
}
// Ensure the gasprice is high enough to cover the requirement of the calling
// pool and/or block producer
// Ensure the gasprice is high enough to cover the requirement of the calling pool
if tx.GasTipCapIntCmp(opts.MinTip) < 0 {
return fmt.Errorf("%w: tip needed %v, tip permitted %v", ErrUnderpriced, opts.MinTip, tx.GasTipCap())
return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrUnderpriced, tx.GasTipCap(), opts.MinTip)
}
// Ensure blob transactions have valid commitments
if tx.Type() == types.BlobTxType {
// Ensure the blob fee cap satisfies the minimum blob gas price
if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 {
return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrUnderpriced, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice)
}
sidecar := tx.BlobTxSidecar()
if sidecar == nil {
return fmt.Errorf("missing sidecar in blob transaction")
return errors.New("missing sidecar in blob transaction")
}
// Ensure the number of items in the blob transaction and various side
// data match up before doing any expensive validations
hashes := tx.BlobHashes()
if len(hashes) == 0 {
return fmt.Errorf("blobless blob transaction")
return errors.New("blobless blob transaction")
}
if len(hashes) > params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob {
return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob)
}
// Ensure commitments, proofs and hashes are valid
if err := validateBlobSidecar(hashes, sidecar); err != nil {
return err
}

2
core/types/account.go
View file

@ -52,7 +52,7 @@ type accountMarshaling struct {
}
// storageJSON represents a 256 bit byte array, but allows less than 256 bits when
// unmarshaling from hex.
// unmarshalling from hex.
type storageJSON common.Hash
func (h *storageJSON) UnmarshalText(text []byte) error {

2
core/types/block_test.go
View file

@ -196,7 +196,7 @@ func TestEIP2718BlockEncoding(t *testing.T) {
func TestUncleHash(t *testing.T) {
uncles := make([]*Header, 0)
h := CalcUncleHash(uncles)
exp := common.HexToHash("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347")
exp := EmptyUncleHash
if h != exp {
t.Fatalf("empty uncle hash is wrong, got %x != %x", h, exp)
}

4
core/types/receipt_test.go
View file

@ -343,7 +343,7 @@ func TestReceiptJSON(t *testing.T) {
r := Receipt{}
err = r.UnmarshalJSON(b)
if err != nil {
t.Fatal("error unmarshaling receipt from json:", err)
t.Fatal("error unmarshalling receipt from json:", err)
}
}
}
@ -360,7 +360,7 @@ func TestEffectiveGasPriceNotRequired(t *testing.T) {
r2 := Receipt{}
err = r2.UnmarshalJSON(b)
if err != nil {
t.Fatal("error unmarshaling receipt from json:", err)
t.Fatal("error unmarshalling receipt from json:", err)
}
}

8
core/types/transaction_signing.go
View file

@ -107,13 +107,7 @@ func SignTx(tx *Transaction, s Signer, prv *ecdsa.PrivateKey) (*Transaction, err
// SignNewTx creates a transaction and signs it.
func SignNewTx(prv *ecdsa.PrivateKey, s Signer, txdata TxData) (*Transaction, error) {
tx := NewTx(txdata)
h := s.Hash(tx)
sig, err := crypto.Sign(h[:], prv)
if err != nil {
return nil, err
}
return tx.WithSignature(s, sig)
return SignTx(NewTx(txdata), s, prv)
}
// MustSignNewTx creates a transaction and signs it.

1
core/vm/errors.go
View file

@ -29,7 +29,6 @@ var (
ErrInsufficientBalance = errors.New("insufficient balance for transfer")
ErrContractAddressCollision = errors.New("contract address collision")
ErrExecutionReverted = errors.New("execution reverted")
ErrMaxInitCodeSizeExceeded = errors.New("max initcode size exceeded")
ErrMaxCodeSizeExceeded = errors.New("max code size exceeded")
ErrInvalidJump = errors.New("invalid jump destination")
ErrWriteProtection = errors.New("write protection")

8
core/vm/instructions.go
View file

@ -17,6 +17,8 @@
package vm
import (
"math"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
@ -303,7 +305,7 @@ func opCallDataCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext
)
dataOffset64, overflow := dataOffset.Uint64WithOverflow()
if overflow {
dataOffset64 = 0xffffffffffffffff
dataOffset64 = math.MaxUint64
}
// These values are checked for overflow during gas cost calculation
memOffset64 := memOffset.Uint64()
@ -359,7 +361,7 @@ func opCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([
)
uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow()
if overflow {
uint64CodeOffset = 0xffffffffffffffff
uint64CodeOffset = math.MaxUint64
}
codeCopy := getData(scope.Contract.Code, uint64CodeOffset, length.Uint64())
scope.Memory.Set(memOffset.Uint64(), length.Uint64(), codeCopy)
@ -377,7 +379,7 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext)
)
uint64CodeOffset, overflow := codeOffset.Uint64WithOverflow()
if overflow {
uint64CodeOffset = 0xffffffffffffffff
uint64CodeOffset = math.MaxUint64
}
addr := common.Address(a.Bytes20())
codeCopy := getData(interpreter.evm.StateDB.GetCode(addr), uint64CodeOffset, length.Uint64())

7
core/vm/operations_acl.go
View file

@ -187,7 +187,12 @@ func makeCallVariantGasCallEIP2929(oldCalculator gasFunc) gasFunc {
// outside of this function, as part of the dynamic gas, and that will make it
// also become correctly reported to tracers.
contract.Gas += coldCost
return gas + coldCost, nil
var overflow bool
if gas, overflow = math.SafeAdd(gas, coldCost); overflow {
return 0, ErrGasUintOverflow
}
return gas, nil
}
}

2
crypto/kzg4844/kzg4844.go
View file

@ -85,7 +85,7 @@ type Claim [32]byte
var useCKZG atomic.Bool
// UseCKZG can be called to switch the default Go implementation of KZG to the C
// library if fo some reason the user wishes to do so (e.g. consensus bug in one
// library if for some reason the user wishes to do so (e.g. consensus bug in one
// or the other).
func UseCKZG(use bool) error {
if use && !ckzgAvailable {

2
crypto/secp256k1/libsecp256k1/include/secp256k1.h
View file

@ -357,7 +357,7 @@ SECP256K1_API int secp256k1_ecdsa_signature_serialize_compact(
/** Verify an ECDSA signature.
*
* Returns: 1: correct signature
* 0: incorrect or unparseable signature
* 0: incorrect or unparsable signature
* Args: ctx: a secp256k1 context object, initialized for verification.
* In: sig: the signature being verified (cannot be NULL)
* msg32: the 32-byte message hash being verified (cannot be NULL)

2
crypto/secp256k1/libsecp256k1/sage/group_prover.sage
View file

@ -17,7 +17,7 @@
# - A constraint describing the requirements of the law, called "require"
# * Implementations are transliterated into functions that operate as well on
# algebraic input points, and are called once per combination of branches
# exectured. Each execution returns:
# executed. Each execution returns:
# - A constraint describing the assumptions this implementation requires
# (such as Z1=1), called "assumeFormula"
# - A constraint describing the assumptions this specific branch requires,

52
eth/api.go
View file

@ -1,52 +0,0 @@
// Copyright 2015 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package eth
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
)
// EthereumAPI provides an API to access Ethereum full node-related information.
type EthereumAPI struct {
e *Ethereum
}
// NewEthereumAPI creates a new Ethereum protocol API for full nodes.
func NewEthereumAPI(e *Ethereum) *EthereumAPI {
return &EthereumAPI{e}
}
// Etherbase is the address that mining rewards will be sent to.
func (api *EthereumAPI) Etherbase() (common.Address, error) {
return api.e.Etherbase()
}
// Coinbase is the address that mining rewards will be sent to (alias for Etherbase).
func (api *EthereumAPI) Coinbase() (common.Address, error) {
return api.Etherbase()
}
// Hashrate returns the POW hashrate.
func (api *EthereumAPI) Hashrate() hexutil.Uint64 {
return hexutil.Uint64(api.e.Miner().Hashrate())
}
// Mining returns an indication if this node is currently mining.
func (api *EthereumAPI) Mining() bool {
return api.e.IsMining()
}

25
eth/api_backend.go
View file

@ -37,7 +37,6 @@ import (
"github.com/ethereum/go-ethereum/eth/tracers"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/miner"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rpc"
)
@ -67,7 +66,7 @@ func (b *EthAPIBackend) SetHead(number uint64) {
func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) {
// Pending block is only known by the miner
if number == rpc.PendingBlockNumber {
block := b.eth.miner.PendingBlock()
block, _, _ := b.eth.miner.Pending()
if block == nil {
return nil, errors.New("pending block is not available")
}
@ -118,7 +117,7 @@ func (b *EthAPIBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*ty
func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) {
// Pending block is only known by the miner
if number == rpc.PendingBlockNumber {
block := b.eth.miner.PendingBlock()
block, _, _ := b.eth.miner.Pending()
if block == nil {
return nil, errors.New("pending block is not available")
}
@ -182,14 +181,14 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r
return nil, errors.New("invalid arguments; neither block nor hash specified")
}
func (b *EthAPIBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) {
return b.eth.miner.PendingBlockAndReceipts()
func (b *EthAPIBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) {
return b.eth.miner.Pending()
}
func (b *EthAPIBackend) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) {
// Pending state is only known by the miner
if number == rpc.PendingBlockNumber {
block, state := b.eth.miner.Pending()
block, _, state := b.eth.miner.Pending()
if block == nil || state == nil {
return nil, nil, errors.New("pending state is not available")
}
@ -267,10 +266,6 @@ func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEven
return b.eth.BlockChain().SubscribeRemovedLogsEvent(ch)
}
func (b *EthAPIBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription {
return b.eth.miner.SubscribePendingLogs(ch)
}
func (b *EthAPIBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription {
return b.eth.BlockChain().SubscribeChainEvent(ch)
}
@ -292,7 +287,7 @@ func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction)
}
func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) {
pending := b.eth.txPool.Pending(nil, nil, nil)
pending := b.eth.txPool.Pending(txpool.PendingFilter{})
var txs types.Transactions
for _, batch := range pending {
for _, lazy := range batch {
@ -421,14 +416,6 @@ func (b *EthAPIBackend) CurrentHeader() *types.Header {
return b.eth.blockchain.CurrentHeader()
}
func (b *EthAPIBackend) Miner() *miner.Miner {
return b.eth.Miner()
}
func (b *EthAPIBackend) StartMining() error {
return b.eth.StartMining()
}
func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, reexec uint64, base *state.StateDB, readOnly bool, preferDisk bool) (*state.StateDB, tracers.StateReleaseFunc, error) {
return b.eth.stateAtBlock(ctx, block, reexec, base, readOnly, preferDisk)
}

4
eth/api_debug.go
View file

@ -56,7 +56,7 @@ func (api *DebugAPI) DumpBlock(blockNr rpc.BlockNumber) (state.Dump, error) {
// If we're dumping the pending state, we need to request
// both the pending block as well as the pending state from
// the miner and operate on those
_, stateDb := api.eth.miner.Pending()
_, _, stateDb := api.eth.miner.Pending()
if stateDb == nil {
return state.Dump{}, errors.New("pending state is not available")
}
@ -142,7 +142,7 @@ func (api *DebugAPI) AccountRange(blockNrOrHash rpc.BlockNumberOrHash, start hex
// If we're dumping the pending state, we need to request
// both the pending block as well as the pending state from
// the miner and operate on those
_, stateDb = api.eth.miner.Pending()
_, _, stateDb = api.eth.miner.Pending()
if stateDb == nil {
return state.Dump{}, errors.New("pending state is not available")
}

30
eth/api_miner.go
View file

@ -18,9 +18,7 @@ package eth
import (
"math/big"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
)
@ -29,26 +27,11 @@ type MinerAPI struct {
e *Ethereum
}
// NewMinerAPI create a new MinerAPI instance.
// NewMinerAPI creates a new MinerAPI instance.
func NewMinerAPI(e *Ethereum) *MinerAPI {
return &MinerAPI{e}
}
// Start starts the miner with the given number of threads. If threads is nil,
// the number of workers started is equal to the number of logical CPUs that are
// usable by this process. If mining is already running, this method adjust the
// number of threads allowed to use and updates the minimum price required by the
// transaction pool.
func (api *MinerAPI) Start() error {
return api.e.StartMining()
}
// Stop terminates the miner, both at the consensus engine level as well as at
// the block creation level.
func (api *MinerAPI) Stop() {
api.e.StopMining()
}
// SetExtra sets the extra data string that is included when this miner mines a block.
func (api *MinerAPI) SetExtra(extra string) (bool, error) {
if err := api.e.Miner().SetExtra([]byte(extra)); err != nil {
@ -73,14 +56,3 @@ func (api *MinerAPI) SetGasLimit(gasLimit hexutil.Uint64) bool {
api.e.Miner().SetGasCeil(uint64(gasLimit))
return true
}
// SetEtherbase sets the etherbase of the miner.
func (api *MinerAPI) SetEtherbase(etherbase common.Address) bool {
api.e.SetEtherbase(etherbase)
return true
}
// SetRecommitInterval updates the interval for miner sealing work recommitting.
func (api *MinerAPI) SetRecommitInterval(interval int) {
api.e.Miner().SetRecommitInterval(time.Duration(interval) * time.Millisecond)
}

187
eth/backend.go
View file

@ -28,8 +28,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/consensus"
"github.com/ethereum/go-ethereum/consensus/beacon"
"github.com/ethereum/go-ethereum/consensus/clique"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/bloombits"
"github.com/ethereum/go-ethereum/core/rawdb"
@ -74,7 +72,6 @@ type Ethereum struct {
handler *handler
ethDialCandidates enode.Iterator
snapDialCandidates enode.Iterator
merger *consensus.Merger
// DB interfaces
chainDb ethdb.Database // Block chain database
@ -89,9 +86,8 @@ type Ethereum struct {
APIBackend *EthAPIBackend
miner *miner.Miner
gasPrice *big.Int
etherbase common.Address
miner *miner.Miner
gasPrice *big.Int
networkID uint64
netRPCService *ethapi.NetAPI
@ -158,7 +154,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
}
eth := &Ethereum{
config: config,
merger: consensus.NewMerger(chainDb),
chainDb: chainDb,
eventMux: stack.EventMux(),
accountManager: stack.AccountManager(),
@ -166,7 +161,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
closeBloomHandler: make(chan struct{}),
networkID: networkID,
gasPrice: config.Miner.GasPrice,
etherbase: config.Miner.Etherbase,
bloomRequests: make(chan chan *bloombits.Retrieval),
bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms),
p2pServer: stack.Server(),
@ -213,7 +207,11 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
if config.OverrideVerkle != nil {
overrides.OverrideVerkle = config.OverrideVerkle
}
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TransactionHistory)
// TODO (MariusVanDerWijden) get rid of shouldPreserve in a follow-up PR
shouldPreserve := func(header *types.Header) bool {
return false
}
eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, shouldPreserve, &config.TransactionHistory)
if err != nil {
return nil, err
}
@ -236,10 +234,10 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
// Permit the downloader to use the trie cache allowance during fast sync
cacheLimit := cacheConfig.TrieCleanLimit + cacheConfig.TrieDirtyLimit + cacheConfig.SnapshotLimit
if eth.handler, err = newHandler(&handlerConfig{
NodeID: eth.p2pServer.Self().ID(),
Database: chainDb,
Chain: eth.blockchain,
TxPool: eth.txPool,
Merger: eth.merger,
Network: networkID,
Sync: config.SyncMode,
BloomCache: uint64(cacheLimit),
@ -249,7 +247,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
return nil, err
}
eth.miner = miner.New(eth, &config.Miner, eth.blockchain.Config(), eth.EventMux(), eth.engine, eth.isLocalBlock)
eth.miner = miner.New(eth, config.Miner, eth.engine)
eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData))
eth.APIBackend = &EthAPIBackend{stack.Config().ExtRPCEnabled(), stack.Config().AllowUnprotectedTxs, eth, nil}
@ -315,9 +313,6 @@ func (s *Ethereum) APIs() []rpc.API {
// Append all the local APIs and return
return append(apis, []rpc.API{
{
Namespace: "eth",
Service: NewEthereumAPI(s),
}, {
Namespace: "miner",
Service: NewMinerAPI(s),
}, {
@ -340,138 +335,6 @@ func (s *Ethereum) ResetWithGenesisBlock(gb *types.Block) {
s.blockchain.ResetWithGenesisBlock(gb)
}
func (s *Ethereum) Etherbase() (eb common.Address, err error) {
s.lock.RLock()
etherbase := s.etherbase
s.lock.RUnlock()
if etherbase != (common.Address{}) {
return etherbase, nil
}
return common.Address{}, errors.New("etherbase must be explicitly specified")
}
// isLocalBlock checks whether the specified block is mined
// by local miner accounts.
//
// We regard two types of accounts as local miner account: etherbase
// and accounts specified via `txpool.locals` flag.
func (s *Ethereum) isLocalBlock(header *types.Header) bool {
author, err := s.engine.Author(header)
if err != nil {
log.Warn("Failed to retrieve block author", "number", header.Number.Uint64(), "hash", header.Hash(), "err", err)
return false
}
// Check whether the given address is etherbase.
s.lock.RLock()
etherbase := s.etherbase
s.lock.RUnlock()
if author == etherbase {
return true
}
// Check whether the given address is specified by `txpool.local`
// CLI flag.
for _, account := range s.config.TxPool.Locals {
if account == author {
return true
}
}
return false
}
// shouldPreserve checks whether we should preserve the given block
// during the chain reorg depending on whether the author of block
// is a local account.
func (s *Ethereum) shouldPreserve(header *types.Header) bool {
// The reason we need to disable the self-reorg preserving for clique
// is it can be probable to introduce a deadlock.
//
// e.g. If there are 7 available signers
//
// r1 A
// r2 B
// r3 C
// r4 D
// r5 A [X] F G
// r6 [X]
//
// In the round5, the in-turn signer E is offline, so the worst case
// is A, F and G sign the block of round5 and reject the block of opponents
// and in the round6, the last available signer B is offline, the whole
// network is stuck.
if _, ok := s.engine.(*clique.Clique); ok {
return false
}
return s.isLocalBlock(header)
}
// SetEtherbase sets the mining reward address.
func (s *Ethereum) SetEtherbase(etherbase common.Address) {
s.lock.Lock()
s.etherbase = etherbase
s.lock.Unlock()
s.miner.SetEtherbase(etherbase)
}
// StartMining starts the miner with the given number of CPU threads. If mining
// is already running, this method adjust the number of threads allowed to use
// and updates the minimum price required by the transaction pool.
func (s *Ethereum) StartMining() error {
// If the miner was not running, initialize it
if !s.IsMining() {
// Propagate the initial price point to the transaction pool
s.lock.RLock()
price := s.gasPrice
s.lock.RUnlock()
s.txPool.SetGasTip(price)
// Configure the local mining address
eb, err := s.Etherbase()
if err != nil {
log.Error("Cannot start mining without etherbase", "err", err)
return fmt.Errorf("etherbase missing: %v", err)
}
var cli *clique.Clique
if c, ok := s.engine.(*clique.Clique); ok {
cli = c
} else if cl, ok := s.engine.(*beacon.Beacon); ok {
if c, ok := cl.InnerEngine().(*clique.Clique); ok {
cli = c
}
}
if cli != nil {
wallet, err := s.accountManager.Find(accounts.Account{Address: eb})
if wallet == nil || err != nil {
log.Error("Etherbase account unavailable locally", "err", err)
return fmt.Errorf("signer missing: %v", err)
}
cli.Authorize(eb, wallet.SignData)
}
// If mining is started, we can disable the transaction rejection mechanism
// introduced to speed sync times.
s.handler.enableSyncedFeatures()
go s.miner.Start()
}
return nil
}
// StopMining terminates the miner, both at the consensus engine level as well as
// at the block creation level.
func (s *Ethereum) StopMining() {
// Update the thread count within the consensus engine
type threaded interface {
SetThreads(threads int)
}
if th, ok := s.engine.(threaded); ok {
th.SetThreads(-1)
}
// Stop the block creating itself
s.miner.Stop()
}
func (s *Ethereum) IsMining() bool { return s.miner.Mining() }
func (s *Ethereum) Miner() *miner.Miner { return s.miner }
func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager }
@ -486,11 +349,6 @@ func (s *Ethereum) Synced() bool { return s.handler.synced
func (s *Ethereum) SetSynced() { s.handler.enableSyncedFeatures() }
func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning }
func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer }
func (s *Ethereum) Merger() *consensus.Merger { return s.merger }
func (s *Ethereum) SyncMode() downloader.SyncMode {
mode, _ := s.handler.chainSync.modeAndLocalHead()
return mode
}
// Protocols returns all the currently configured
// network protocols to start.
@ -538,7 +396,6 @@ func (s *Ethereum) Stop() error {
s.bloomIndexer.Close()
close(s.closeBloomHandler)
s.txPool.Close()
s.miner.Close()
s.blockchain.Stop()
s.engine.Close()
@ -550,3 +407,29 @@ func (s *Ethereum) Stop() error {
return nil
}
// SyncMode retrieves the current sync mode, either explicitly set, or derived
// from the chain status.
func (s *Ethereum) SyncMode() downloader.SyncMode {
// If we're in snap sync mode, return that directly
if s.handler.snapSync.Load() {
return downloader.SnapSync
}
// We are probably in full sync, but we might have rewound to before the
// snap sync pivot, check if we should re-enable snap sync.
head := s.blockchain.CurrentBlock()
if pivot := rawdb.ReadLastPivotNumber(s.chainDb); pivot != nil {
if head.Number.Uint64() < *pivot {
return downloader.SnapSync
}
}
// We are in a full sync, but the associated head state is missing. To complete
// the head state, forcefully rerun the snap sync. Note it doesn't mean the
// persistent state is corrupted, just mismatch with the head block.
if !s.blockchain.HasState(head.Root) {
log.Info("Reenabled snap sync as chain is stateless")
return downloader.SnapSync
}
// Nope, we're really full syncing
return downloader.FullSync
}

78
eth/catalyst/api.go
View file

@ -190,21 +190,21 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, pa
// attributes. It supports both PayloadAttributesV1 and PayloadAttributesV2.
func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) {
if params != nil {
if params.BeaconRoot != nil {
return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("unexpected beacon root"))
}
switch api.eth.BlockChain().Config().LatestFork(params.Timestamp) {
case forks.Paris:
if params.Withdrawals != nil {
return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("withdrawals before shanghai"))
return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("withdrawals before shanghai"))
}
case forks.Shanghai:
if params.Withdrawals == nil {
return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals"))
return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing withdrawals"))
}
default:
return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV2 must only be called with paris and shanghai payloads"))
}
if params.BeaconRoot != nil {
return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("unexpected beacon root"))
}
}
return api.forkchoiceUpdated(update, params, engine.PayloadV2, false)
}
@ -213,15 +213,11 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, pa
// in the payload attributes. It supports only PayloadAttributesV3.
func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, params *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) {
if params != nil {
// TODO(matt): according to https://github.com/ethereum/execution-apis/pull/498,
// payload attributes that are invalid should return error
// engine.InvalidPayloadAttributes. Once hive updates this, we should update
// on our end.
if params.Withdrawals == nil {
return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing withdrawals"))
return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing withdrawals"))
}
if params.BeaconRoot == nil {
return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("missing beacon root"))
return engine.STATUS_INVALID, engine.InvalidPayloadAttributes.With(errors.New("missing beacon root"))
}
if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun {
return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV3 must only be called for cancun payloads"))
@ -271,12 +267,6 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl
finalized := api.remoteBlocks.get(update.FinalizedBlockHash)
// Header advertised via a past newPayload request. Start syncing to it.
// Before we do however, make sure any legacy sync in switched off so we
// don't accidentally have 2 cycles running.
if merger := api.eth.Merger(); !merger.TDDReached() {
merger.ReachTTD()
api.eth.Downloader().Cancel()
}
context := []interface{}{"number", header.Number, "hash", header.Hash()}
if update.FinalizedBlockHash != (common.Hash{}) {
if finalized == nil {
@ -338,9 +328,6 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl
// If the beacon client also advertised a finalized block, mark the local
// chain final and completely in PoS mode.
if update.FinalizedBlockHash != (common.Hash{}) {
if merger := api.eth.Merger(); !merger.PoSFinalized() {
merger.FinalizePoS()
}
// If the finalized block is not in our canonical tree, something is wrong
finalBlock := api.eth.BlockChain().GetBlockByHash(update.FinalizedBlockHash)
if finalBlock == nil {
@ -488,7 +475,7 @@ func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.Payl
// NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain.
func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.PayloadStatusV1, error) {
if api.eth.BlockChain().Config().IsCancun(api.eth.BlockChain().Config().LondonBlock, params.Timestamp) {
return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use new payload v2 post-shanghai"))
return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("can't use newPayloadV2 post-cancun"))
}
if api.eth.BlockChain().Config().LatestFork(params.Timestamp) == forks.Shanghai {
if params.Withdrawals == nil {
@ -503,7 +490,7 @@ func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.Payl
return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil excessBlobGas pre-cancun"))
}
if params.BlobGasUsed != nil {
return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil params.BlobGasUsed pre-cancun"))
return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil blobGasUsed pre-cancun"))
}
return api.newPayload(params, nil, nil)
}
@ -517,14 +504,14 @@ func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHas
return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil excessBlobGas post-cancun"))
}
if params.BlobGasUsed == nil {
return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil params.BlobGasUsed post-cancun"))
return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil blobGasUsed post-cancun"))
}
if versionedHashes == nil {
return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil versionedHashes post-cancun"))
}
if beaconRoot == nil {
return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil parentBeaconBlockRoot post-cancun"))
return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("nil beaconRoot post-cancun"))
}
if api.eth.BlockChain().Config().LatestFork(params.Timestamp) != forks.Cancun {
@ -624,13 +611,6 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashe
return api.invalid(err, parent.Header()), nil
}
// We've accepted a valid payload from the beacon client. Mark the local
// chain transitions to notify other subsystems (e.g. downloader) of the
// behavioral change.
if merger := api.eth.Merger(); !merger.TDDReached() {
merger.ReachTTD()
api.eth.Downloader().Cancel()
}
hash := block.Hash()
return engine.PayloadStatusV1{Status: engine.VALID, LatestValidHash: &hash}, nil
}
@ -788,26 +768,23 @@ func (api *ConsensusAPI) heartbeat() {
// If there have been no updates for the past while, warn the user
// that the beacon client is probably offline
if api.eth.BlockChain().Config().TerminalTotalDifficultyPassed || api.eth.Merger().TDDReached() {
if time.Since(lastForkchoiceUpdate) <= beaconUpdateConsensusTimeout || time.Since(lastNewPayloadUpdate) <= beaconUpdateConsensusTimeout {
offlineLogged = time.Time{}
continue
}
if time.Since(offlineLogged) > beaconUpdateWarnFrequency {
if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() {
if lastTransitionUpdate.IsZero() {
log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!")
} else {
log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!")
}
} else {
log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!")
}
offlineLogged = time.Now()
}
if time.Since(lastForkchoiceUpdate) <= beaconUpdateConsensusTimeout || time.Since(lastNewPayloadUpdate) <= beaconUpdateConsensusTimeout {
offlineLogged = time.Time{}
continue
}
if time.Since(offlineLogged) > beaconUpdateWarnFrequency {
if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() {
if lastTransitionUpdate.IsZero() {
log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!")
} else {
log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!")
}
} else {
log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!")
}
offlineLogged = time.Now()
}
continue
}
}
@ -879,8 +856,7 @@ func getBody(block *types.Block) *engine.ExecutionPayloadBodyV1 {
)
for j, tx := range body.Transactions {
data, _ := tx.MarshalBinary()
txs[j] = hexutil.Bytes(data)
txs[j], _ = tx.MarshalBinary()
}
// Post-shanghai withdrawals MUST be set to empty slice instead of nil

16
eth/catalyst/api_test.go
View file

@ -262,11 +262,8 @@ func TestInvalidPayloadTimestamp(t *testing.T) {
{0, true},
{parent.Time, true},
{parent.Time - 1, true},
// TODO (MariusVanDerWijden) following tests are currently broken,
// fixed in upcoming merge-kiln-v2 pr
//{parent.Time() + 1, false},
//{uint64(time.Now().Unix()) + uint64(time.Minute), false},
{parent.Time + 1, false},
{uint64(time.Now().Unix()) + uint64(time.Minute), false},
}
for i, test := range tests {
@ -450,7 +447,9 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block)
t.Fatal("can't create node:", err)
}
ethcfg := &ethconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256}
mcfg := miner.DefaultConfig
mcfg.PendingFeeRecipient = testAddr
ethcfg := &ethconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256, Miner: mcfg}
ethservice, err := eth.New(n, ethcfg)
if err != nil {
t.Fatal("can't create eth service:", err)
@ -463,7 +462,6 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block)
t.Fatal("can't import test blocks:", err)
}
ethservice.SetEtherbase(testAddr)
ethservice.SetSynced()
return n, ethservice
}
@ -865,7 +863,6 @@ func TestTrickRemoteBlockCache(t *testing.T) {
func TestInvalidBloom(t *testing.T) {
genesis, preMergeBlocks := generateMergeChain(10, false)
n, ethservice := startEthService(t, genesis, preMergeBlocks)
ethservice.Merger().ReachTTD()
defer n.Close()
commonAncestor := ethservice.BlockChain().CurrentBlock()
@ -1047,7 +1044,6 @@ func TestWithdrawals(t *testing.T) {
genesis.Config.ShanghaiTime = &time
n, ethservice := startEthService(t, genesis, blocks)
ethservice.Merger().ReachTTD()
defer n.Close()
api := NewConsensusAPI(ethservice)
@ -1165,7 +1161,6 @@ func TestNilWithdrawals(t *testing.T) {
genesis.Config.ShanghaiTime = &time
n, ethservice := startEthService(t, genesis, blocks)
ethservice.Merger().ReachTTD()
defer n.Close()
api := NewConsensusAPI(ethservice)
@ -1592,7 +1587,6 @@ func TestParentBeaconBlockRoot(t *testing.T) {
genesis.Config.CancunTime = &time
n, ethservice := startEthService(t, genesis, blocks)
ethservice.Merger().ReachTTD()
defer n.Close()
api := NewConsensusAPI(ethservice)

88
eth/catalyst/blsync.go Normal file
View file

@ -0,0 +1,88 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package catalyst
import (
"github.com/ethereum/go-ethereum/beacon/engine"
"github.com/ethereum/go-ethereum/beacon/types"
"github.com/ethereum/go-ethereum/eth"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/log"
)
// Blsync tracks the head of the beacon chain through the beacon light client
// and drives the local node via ConsensusAPI.
type Blsync struct {
engine *ConsensusAPI
client Client
headCh chan types.ChainHeadEvent
headSub event.Subscription
quitCh chan struct{}
}
type Client interface {
SubscribeChainHeadEvent(ch chan<- types.ChainHeadEvent) event.Subscription
Start()
Stop()
}
// NewBlsync creates a new beacon light syncer.
func NewBlsync(client Client, eth *eth.Ethereum) *Blsync {
return &Blsync{
engine: newConsensusAPIWithoutHeartbeat(eth),
client: client,
headCh: make(chan types.ChainHeadEvent, 16),
quitCh: make(chan struct{}),
}
}
// Start starts underlying beacon light client and the sync logic for driving
// the local node.
func (b *Blsync) Start() error {
log.Info("Beacon light sync started")
b.headSub = b.client.SubscribeChainHeadEvent(b.headCh)
go b.client.Start()
for {
select {
case <-b.quitCh:
return nil
case head := <-b.headCh:
if _, err := b.engine.NewPayloadV2(*head.HeadBlock); err != nil {
log.Error("failed to send new payload", "err", err)
continue
}
update := engine.ForkchoiceStateV1{
HeadBlockHash: head.HeadBlock.BlockHash,
SafeBlockHash: head.Finalized, //TODO pass finalized or empty hash here?
FinalizedBlockHash: head.Finalized,
}
if _, err := b.engine.ForkchoiceUpdatedV1(update, nil); err != nil {
log.Error("failed to send forkchoice updated", "err", err)
continue
}
}
}
}
// Stop signals to the light client and syncer to exit.
func (b *Blsync) Stop() error {
b.client.Stop()
close(b.quitCh)
return nil
}

Some files were not shown because too many files have changed in this diff Show more