From f0cdc40cebd3fcb26d21ced0f1093efd7a2c949f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 31 Mar 2025 18:29:33 +0200 Subject: [PATCH 01/35] version: begin v1.15.8 release cycle reloaded --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index 2098af17b5..72fd903a1f 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 15 // Minor version component of the current release - Patch = 7 // Patch version component of the current release - Meta = "stable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 15 // Minor version component of the current release + Patch = 8 // Patch version component of the current release + Meta = "unstable" // Version metadata to append to the version string ) From bc36f2de83a5ba8e805af6ce4ea9da3ab7cb1df9 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Tue, 1 Apr 2025 13:42:01 +0200 Subject: [PATCH 02/35] eth, eth/filters: implement API error code for pruned blocks (#31361) Implements #31275 --------- Co-authored-by: Jared Wasinger Co-authored-by: Felix Lange --- core/blockchain_reader.go | 5 ++ eth/api_backend.go | 46 ++++++++++++++++--- eth/ethconfig/historymode.go | 6 +++ eth/filters/api.go | 5 ++ eth/filters/filter.go | 20 ++++++-- eth/filters/filter_system.go | 10 ++++ eth/filters/filter_system_test.go | 4 ++ internal/ethapi/api.go | 58 +++++++++++++----------- internal/ethapi/api_test.go | 7 +++ internal/ethapi/backend.go | 1 + internal/ethapi/transaction_args_test.go | 2 + rpc/types.go | 2 +- 12 files changed, 128 insertions(+), 38 deletions(-) diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 025b912ceb..415a0f5442 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -86,6 +86,11 @@ func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header { return bc.hc.GetHeaderByNumber(number) } +// GetBlockNumber retrieves the block number associated with a block hash. +func (bc *BlockChain) GetBlockNumber(hash common.Hash) *uint64 { + return bc.hc.GetBlockNumber(hash) +} + // GetHeadersFrom returns a contiguous segment of headers, in rlp-form, going // backwards from the given number. func (bc *BlockChain) GetHeadersFrom(number, count uint64) []rlp.RawValue { diff --git a/eth/api_backend.go b/eth/api_backend.go index c8c5ca707f..944c357e78 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" @@ -91,7 +92,13 @@ func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumb } return block, nil } - return b.eth.blockchain.GetHeaderByNumber(uint64(number)), nil + var bn uint64 + if number == rpc.EarliestBlockNumber { + bn = b.eth.blockchain.HistoryPruningCutoff() + } else { + bn = uint64(number) + } + return b.eth.blockchain.GetHeaderByNumber(bn), nil } func (b *EthAPIBackend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) { @@ -143,11 +150,27 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe } return b.eth.blockchain.GetBlock(header.Hash(), header.Number.Uint64()), nil } - return b.eth.blockchain.GetBlockByNumber(uint64(number)), nil + bn := uint64(number) // the resolved number + if number == rpc.EarliestBlockNumber { + bn = b.eth.blockchain.HistoryPruningCutoff() + } + block := b.eth.blockchain.GetBlockByNumber(bn) + if block == nil && bn < b.eth.blockchain.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } + return block, nil } func (b *EthAPIBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { - return b.eth.blockchain.GetBlockByHash(hash), nil + number := b.eth.blockchain.GetBlockNumber(hash) + if number == nil { + return nil, nil + } + block := b.eth.blockchain.GetBlock(hash, *number) + if block == nil && *number < b.eth.blockchain.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } + return block, nil } // GetBody returns body of a block. It does not resolve special block numbers. @@ -155,10 +178,14 @@ func (b *EthAPIBackend) GetBody(ctx context.Context, hash common.Hash, number rp if number < 0 || hash == (common.Hash{}) { return nil, errors.New("invalid arguments; expect hash and no special block numbers") } - if body := b.eth.blockchain.GetBody(hash); body != nil { - return body, nil + body := b.eth.blockchain.GetBody(hash) + if body == nil { + if uint64(number) < b.eth.blockchain.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } + return nil, errors.New("block body not found") } - return nil, errors.New("block body not found") + return body, nil } func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) { @@ -175,6 +202,9 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r } block := b.eth.blockchain.GetBlock(hash, header.Number.Uint64()) if block == nil { + if header.Number.Uint64() < b.eth.blockchain.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } return nil, errors.New("header found, but block body is missing") } return block, nil @@ -234,6 +264,10 @@ func (b *EthAPIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockN return nil, nil, errors.New("invalid arguments; neither block nor hash specified") } +func (b *EthAPIBackend) HistoryPruningCutoff() uint64 { + return b.eth.blockchain.HistoryPruningCutoff() +} + func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { return b.eth.blockchain.GetReceiptsByHash(hash), nil } diff --git a/eth/ethconfig/historymode.go b/eth/ethconfig/historymode.go index c3661004e8..a595d72feb 100644 --- a/eth/ethconfig/historymode.go +++ b/eth/ethconfig/historymode.go @@ -90,3 +90,9 @@ var HistoryPrunePoints = map[common.Hash]*HistoryPrunePoint{ BlockHash: common.HexToHash("0x229f6b18ca1552f1d5146deceb5387333f40dc6275aebee3f2c5c4ece07d02db"), }, } + +// PrunedHistoryError is returned when the requested history is pruned. +type PrunedHistoryError struct{} + +func (e *PrunedHistoryError) Error() string { return "pruned history unavailable" } +func (e *PrunedHistoryError) ErrorCode() int { return 4444 } diff --git a/eth/filters/api.go b/eth/filters/api.go index e3057a2af2..6593cbef27 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rpc" ) @@ -354,9 +355,13 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type if crit.ToBlock != nil { end = crit.ToBlock.Int64() } + // Block numbers below 0 are special cases. if begin > 0 && end > 0 && begin > end { return nil, errInvalidBlockRange } + if begin > 0 && begin < int64(api.events.backend.HistoryPruningCutoff()) { + return nil, ðconfig.PrunedHistoryError{} + } // Construct the range filter filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) } diff --git a/eth/filters/filter.go b/eth/filters/filter.go index b743c25994..e44b37d047 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/filtermaps" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -86,6 +87,9 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { if header == nil { return nil, errors.New("unknown block") } + if header.Number.Uint64() < f.sys.backend.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } return f.blockLogs(ctx, header) } @@ -114,11 +118,19 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { return 0, errors.New("safe header not found") } return hdr.Number.Uint64(), nil + case rpc.EarliestBlockNumber.Int64(): + earliest := f.sys.backend.HistoryPruningCutoff() + hdr, _ := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(earliest)) + if hdr == nil { + return 0, errors.New("earliest header not found") + } + return hdr.Number.Uint64(), nil + default: + if number < 0 { + return 0, errors.New("negative block number") + } + return uint64(number), nil } - if number < 0 { - return 0, errors.New("negative block number") - } - return uint64(number), nil } // range query need to resolve the special begin/end block number diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 7531a1ecfc..aca3c03ad6 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/filtermaps" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" @@ -64,6 +65,7 @@ type Backend interface { CurrentHeader() *types.Header ChainConfig() *params.ChainConfig + HistoryPruningCutoff() uint64 SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription @@ -304,6 +306,14 @@ func (es *EventSystem) SubscribeLogs(crit ethereum.FilterQuery, logs chan []*typ return nil, errPendingLogsUnsupported } + if from == rpc.EarliestBlockNumber { + from = rpc.BlockNumber(es.backend.HistoryPruningCutoff()) + } + // Queries beyond the pruning cutoff are not supported. + if uint64(from) < es.backend.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } + // only interested in new mined logs if from == rpc.LatestBlockNumber && to == rpc.LatestBlockNumber { return es.subscribeLogs(crit, logs), nil diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index c35d823f5a..3bb019d105 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -181,6 +181,10 @@ func (b *testBackend) setPending(block *types.Block, receipts types.Receipts) { b.pendingReceipts = receipts } +func (b *testBackend) HistoryPruningCutoff() uint64 { + return 0 +} + func newTestFilterSystem(db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { backend := &testBackend{db: db} sys := NewFilterSystem(backend, cfg) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 36a5df8087..3b699748b8 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -549,21 +549,23 @@ func (api *BlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, block } // GetUncleCountByBlockNumber returns number of uncles in the block for the given block number -func (api *BlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { - if block, _ := api.b.BlockByNumber(ctx, blockNr); block != nil { +func (api *BlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) (*hexutil.Uint, error) { + block, err := api.b.BlockByNumber(ctx, blockNr) + if block != nil { n := hexutil.Uint(len(block.Uncles())) - return &n + return &n, nil } - return nil + return nil, err } // GetUncleCountByBlockHash returns number of uncles in the block for the given block hash -func (api *BlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { - if block, _ := api.b.BlockByHash(ctx, blockHash); block != nil { +func (api *BlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) (*hexutil.Uint, error) { + block, err := api.b.BlockByHash(ctx, blockHash) + if block != nil { n := hexutil.Uint(len(block.Uncles())) - return &n + return &n, nil } - return nil + return nil, err } // GetCode returns the code stored at the given address in the state for the given block number. @@ -596,9 +598,7 @@ func (api *BlockChainAPI) GetStorageAt(ctx context.Context, address common.Addre func (api *BlockChainAPI) GetBlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]map[string]interface{}, error) { block, err := api.b.BlockByNumberOrHash(ctx, blockNrOrHash) if block == nil || err != nil { - // When the block doesn't exist, the RPC method should return JSON null - // as per specification. - return nil, nil + return nil, err } receipts, err := api.b.GetReceipts(ctx, block.Hash()) if err != nil { @@ -1258,37 +1258,41 @@ func NewTransactionAPI(b Backend, nonceLock *AddrLocker) *TransactionAPI { } // GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. -func (api *TransactionAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { - if block, _ := api.b.BlockByNumber(ctx, blockNr); block != nil { +func (api *TransactionAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*hexutil.Uint, error) { + block, err := api.b.BlockByNumber(ctx, blockNr) + if block != nil { n := hexutil.Uint(len(block.Transactions())) - return &n + return &n, nil } - return nil + return nil, err } // GetBlockTransactionCountByHash returns the number of transactions in the block with the given hash. -func (api *TransactionAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { - if block, _ := api.b.BlockByHash(ctx, blockHash); block != nil { +func (api *TransactionAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) (*hexutil.Uint, error) { + block, err := api.b.BlockByHash(ctx, blockHash) + if block != nil { n := hexutil.Uint(len(block.Transactions())) - return &n + return &n, nil } - return nil + return nil, err } // GetTransactionByBlockNumberAndIndex returns the transaction for the given block number and index. -func (api *TransactionAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) *RPCTransaction { - if block, _ := api.b.BlockByNumber(ctx, blockNr); block != nil { - return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig()) +func (api *TransactionAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (*RPCTransaction, error) { + block, err := api.b.BlockByNumber(ctx, blockNr) + if block != nil { + return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig()), nil } - return nil + return nil, err } // GetTransactionByBlockHashAndIndex returns the transaction for the given block hash and index. -func (api *TransactionAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) *RPCTransaction { - if block, _ := api.b.BlockByHash(ctx, blockHash); block != nil { - return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig()) +func (api *TransactionAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) (*RPCTransaction, error) { + block, err := api.b.BlockByHash(ctx, blockHash) + if block != nil { + return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig()), nil } - return nil + return nil, err } // GetRawTransactionByBlockNumberAndIndex returns the bytes of the transaction for the given block number and index. diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 1ed1a8c8d8..37210aa78b 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -520,8 +520,12 @@ func (b testBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) if number == rpc.PendingBlockNumber { return b.pending, nil } + if number == rpc.EarliestBlockNumber { + number = 0 + } return b.chain.GetBlockByNumber(uint64(number)), nil } + func (b testBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { return b.chain.GetBlockByHash(hash), nil } @@ -618,6 +622,9 @@ func (b testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscripti func (b testBackend) NewMatcherBackend() filtermaps.MatcherBackend { panic("implement me") } + +func (b testBackend) HistoryPruningCutoff() uint64 { return b.chain.HistoryPruningCutoff() } + func TestEstimateGas(t *testing.T) { t.Parallel() // Initialize test accounts diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 9e2ea2c876..c4bf2e0591 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -85,6 +85,7 @@ type Backend interface { ChainConfig() *params.ChainConfig Engine() consensus.Engine + HistoryPruningCutoff() uint64 // This is copied from filters.Backend // eth/filters needs to be initialized from this backend type, so methods needed by diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index a5fd9bb0d4..b4d11a3033 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -402,3 +402,5 @@ func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) func (b *backendMock) Engine() consensus.Engine { return nil } func (b *backendMock) NewMatcherBackend() filtermaps.MatcherBackend { return nil } + +func (b *backendMock) HistoryPruningCutoff() uint64 { return 0 } diff --git a/rpc/types.go b/rpc/types.go index 2e53174b87..85f15344e8 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -63,11 +63,11 @@ type jsonWriter interface { type BlockNumber int64 const ( + EarliestBlockNumber = BlockNumber(-5) SafeBlockNumber = BlockNumber(-4) FinalizedBlockNumber = BlockNumber(-3) LatestBlockNumber = BlockNumber(-2) PendingBlockNumber = BlockNumber(-1) - EarliestBlockNumber = BlockNumber(0) ) // UnmarshalJSON parses the given JSON fragment into a BlockNumber. It supports: From 1bd70ba57aa943340e7a685ee3d89db072ac6ef6 Mon Sep 17 00:00:00 2001 From: John <33443230+gazzua@users.noreply.github.com> Date: Tue, 1 Apr 2025 21:07:47 +0900 Subject: [PATCH 03/35] p2p/nat: improve AddMapping code (#31486) It introduces a new variable to store the external port returned by the addAnyPortMapping function and ensures that the correct external port is returned even in case of an error. --------- Co-authored-by: Felix Lange --- p2p/nat/natupnp.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index f1bb955892..1014dc69d2 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -82,7 +82,7 @@ func (n *upnp) ExternalIP() (addr net.IP, err error) { func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) (uint16, error) { ip, err := n.internalAddress() if err != nil { - return 0, nil // TODO: Shouldn't we return the error? + return 0, err } protocol = strings.ToUpper(protocol) lifetimeS := uint32(lifetime / time.Second) @@ -94,14 +94,15 @@ func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, li if err == nil { return uint16(extport), nil } - - return uint16(extport), n.withRateLimit(func() error { + // Try addAnyPortMapping if mapping specified port didn't work. + err = n.withRateLimit(func() error { p, err := n.addAnyPortMapping(protocol, extport, intport, ip, desc, lifetimeS) if err == nil { extport = int(p) } return err }) + return uint16(extport), err } func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.IP, desc string, lifetimeS uint32) (uint16, error) { From 4add312c8a8332b76e5263066a475e962637c9ac Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 1 Apr 2025 20:10:22 +0800 Subject: [PATCH 04/35] cmd: apply snapshot cache flag in the MakeChain (#31534) --- cmd/utils/flags.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index ae58c2d053..fb2892d2c1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -2189,6 +2189,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh } if !ctx.Bool(SnapshotFlag.Name) { cache.SnapshotLimit = 0 // Disabled + } else if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheSnapshotFlag.Name) { + cache.SnapshotLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheSnapshotFlag.Name) / 100 } // If we're in readonly, do not bother generating snapshot data. if readonly { From 7e3170fb5ce4e03348d29d1a153c2591680058a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 1 Apr 2025 14:29:20 +0200 Subject: [PATCH 05/35] core/filtermaps: add metrics (#31511) This PR adds metrics related to map rendering and pattern matching to the `core/filtermaps` package. --- core/filtermaps/filtermaps.go | 21 +++++++++++++++++++++ core/filtermaps/map_renderer.go | 22 +++++++++++++++++++++- core/filtermaps/matcher.go | 12 ++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index 5722f17daa..18b1c7dc79 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -30,6 +30,23 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +var ( + mapCountGauge = metrics.NewRegisteredGauge("filtermaps/maps/count", nil) // actual number of rendered maps + mapLogValueMeter = metrics.NewRegisteredMeter("filtermaps/maps/logvalues", nil) // number of log values processed + mapBlockMeter = metrics.NewRegisteredMeter("filtermaps/maps/blocks", nil) // number of block delimiters processed + mapRenderTimer = metrics.NewRegisteredTimer("filtermaps/maps/rendertime", nil) // time elapsed while rendering a single map + mapWriteTimer = metrics.NewRegisteredTimer("filtermaps/maps/writetime", nil) // time elapsed while writing a batch of finished maps to db + matchRequestTimer = metrics.NewRegisteredTimer("filtermaps/match/requesttime", nil) // processing time a matching request in a single epoch + matchEpochTimer = metrics.NewRegisteredTimer("filtermaps/match/epochtime", nil) // total processing time a matching request + matchBaseRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowaccess", nil) // number of accessed rows on layer 0 + matchBaseRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowsize", nil) // size of accessed rows on layer 0 + matchExtRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowaccess", nil) // number of accessed rows on higher layers + matchExtRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowsize", nil) // size of accessed rows on higher layers + matchLogLookup = metrics.NewRegisteredMeter("filtermaps/match/loglookup", nil) // number of log lookups based on potential matches + matchAllMeter = metrics.NewRegisteredMeter("filtermaps/match/matchall", nil) // number of requests returned with ErrMatchAll ) const ( @@ -429,8 +446,12 @@ func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, ne TailPartialEpoch: newRange.tailPartialEpoch, } rawdb.WriteFilterMapsRange(batch, rs) + if !isTempRange { + mapCountGauge.Update(int64(newRange.maps.Count() + newRange.tailPartialEpoch)) + } } else { rawdb.DeleteFilterMapsRange(batch) + mapCountGauge.Update(0) } } diff --git a/core/filtermaps/map_renderer.go b/core/filtermaps/map_renderer.go index 1eaaa9bb1a..28f943abb3 100644 --- a/core/filtermaps/map_renderer.go +++ b/core/filtermaps/map_renderer.go @@ -21,6 +21,7 @@ import ( "fmt" "math" "sort" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" @@ -301,6 +302,11 @@ func (r *mapRenderer) run(stopCb func() bool, writeCb func()) (bool, error) { // renderCurrentMap renders a single map. func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) { + var ( + totalTime time.Duration + logValuesProcessed, blocksProcessed int64 + ) + start := time.Now() if !r.iterator.updateChainView(r.f.targetView) { return false, errChainUpdate } @@ -316,9 +322,11 @@ func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) { for r.iterator.lvIndex < uint64(r.currentMap.mapIndex+1)<= valuesPerCallback { + totalTime += time.Since(start) if stopCb() { return false, nil } + start = time.Now() if !r.iterator.updateChainView(r.f.targetView) { return false, errChainUpdate } @@ -343,8 +351,10 @@ func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) { return false, fmt.Errorf("failed to advance log iterator at %d while rendering map %d: %v", r.iterator.lvIndex, r.currentMap.mapIndex, err) } if !r.iterator.skipToBoundary { + logValuesProcessed++ r.currentMap.lastBlock = r.iterator.blockNumber if r.iterator.blockStart { + blocksProcessed++ r.currentMap.blockLvPtrs = append(r.currentMap.blockLvPtrs, r.iterator.lvIndex) } if !r.f.testDisableSnapshots && r.renderBefore >= r.f.indexedRange.maps.AfterLast() && @@ -358,12 +368,18 @@ func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) { r.currentMap.headDelimiter = r.iterator.lvIndex } r.currentMap.lastBlockId = r.f.targetView.getBlockId(r.currentMap.lastBlock) + totalTime += time.Since(start) + mapRenderTimer.Update(totalTime) + mapLogValueMeter.Mark(logValuesProcessed) + mapBlockMeter.Mark(blocksProcessed) return true, nil } // writeFinishedMaps writes rendered maps to the database and updates // filterMapsRange and indexedView accordingly. func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error { + var totalTime time.Duration + start := time.Now() if len(r.finishedMaps) == 0 { return nil } @@ -379,7 +395,7 @@ func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error { if err != nil { return fmt.Errorf("failed to get updated rendered range: %v", err) } - renderedView := r.f.targetView // stopCb callback might still change targetView while writing finished maps + renderedView := r.f.targetView // pauseCb callback might still change targetView while writing finished maps batch := r.f.db.NewBatch() var writeCnt int @@ -393,7 +409,9 @@ func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error { // do not exit while in partially written state but do allow processing // events and pausing while block processing is in progress r.f.indexLock.Unlock() + totalTime += time.Since(start) pauseCb() + start = time.Now() r.f.indexLock.Lock() batch = r.f.db.NewBatch() } @@ -477,6 +495,8 @@ func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error { if err := batch.Write(); err != nil { log.Crit("Error writing log index update batch", "error", err) } + totalTime += time.Since(start) + mapWriteTimer.Update(totalTime) return nil } diff --git a/core/filtermaps/matcher.go b/core/filtermaps/matcher.go index 6c05672cbc..5738bf166a 100644 --- a/core/filtermaps/matcher.go +++ b/core/filtermaps/matcher.go @@ -125,6 +125,7 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock start := time.Now() res, err := m.process() + matchRequestTimer.Update(time.Since(start)) if doRuntimeStats { log.Info("Log search finished", "elapsed", time.Since(start)) @@ -202,6 +203,7 @@ func (m *matcherEnv) process() ([]*types.Log, error) { logs = append(logs, tasks[waitEpoch].logs...) if err := tasks[waitEpoch].err; err != nil { if err == ErrMatchAll { + matchAllMeter.Mark(1) return logs, err } return logs, fmt.Errorf("failed to process log index epoch %d: %v", waitEpoch, err) @@ -220,6 +222,7 @@ func (m *matcherEnv) process() ([]*types.Log, error) { // processEpoch returns the potentially matching logs from the given epoch. func (m *matcherEnv) processEpoch(epochIndex uint32) ([]*types.Log, error) { + start := time.Now() var logs []*types.Log // create a list of map indices to process fm, lm := epochIndex< Date: Tue, 1 Apr 2025 22:13:37 +0800 Subject: [PATCH 06/35] accounts/abi/abigen: fix a flaky bind test case `NewSingleStructArgument` (#31501) found the failed testcase here https://ci.appveyor.com/project/ethereum/go-ethereum/builds/51767091/job/rbjke432c05pufja add a timeout to wait the tx to be mined. --------- Signed-off-by: jsvisa Co-authored-by: Jared Wasinger --- accounts/abi/abigen/bind_test.go | 55 ++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/accounts/abi/abigen/bind_test.go b/accounts/abi/abigen/bind_test.go index 3871560912..195064fb7a 100644 --- a/accounts/abi/abigen/bind_test.go +++ b/accounts/abi/abigen/bind_test.go @@ -541,7 +541,7 @@ var bindTests = []struct { struct A { bytes32 B; } - + function F() public view returns (A[] memory a, uint256[] memory c, bool[] memory d) { A[] memory a = new A[](2); a[0].B = bytes32(uint256(1234) << 96); @@ -549,7 +549,7 @@ var bindTests = []struct { bool[] memory d; return (a, c, d); } - + function G() public view returns (A[] memory a) { A[] memory a = new A[](2); a[0].B = bytes32(uint256(1234) << 96); @@ -571,10 +571,10 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() - + // Deploy a structs method invoker contract and execute its default method _, _, structs, err := DeployStructs(auth, sim) if err != nil { @@ -1701,13 +1701,13 @@ var bindTests = []struct { `NewFallbacks`, ` pragma solidity >=0.6.0 <0.7.0; - + contract NewFallbacks { event Fallback(bytes data); fallback() external { emit Fallback(msg.data); } - + event Received(address addr, uint value); receive() external payable { emit Received(msg.sender, msg.value); @@ -1719,7 +1719,7 @@ var bindTests = []struct { ` "bytes" "math/big" - + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/core/types" @@ -1728,22 +1728,22 @@ var bindTests = []struct { ` key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) - + sim := backends.NewSimulatedBackend(types.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 1000000) defer sim.Close() - + opts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) _, _, c, err := DeployNewFallbacks(opts, sim) if err != nil { t.Fatalf("Failed to deploy contract: %v", err) } sim.Commit() - + // Test receive function opts.Value = big.NewInt(100) c.Receive(opts) sim.Commit() - + var gotEvent bool iter, _ := c.FilterReceived(nil) defer iter.Close() @@ -1760,14 +1760,14 @@ var bindTests = []struct { if !gotEvent { t.Fatal("Expect to receive event emitted by receive") } - + // Test fallback function gotEvent = false opts.Value = nil calldata := []byte{0x01, 0x02, 0x03} c.Fallback(opts, calldata) sim.Commit() - + iter2, _ := c.FilterFallback(nil) defer iter2.Close() for iter2.Next() { @@ -1806,7 +1806,9 @@ var bindTests = []struct { []string{"608060405234801561001057600080fd5b50610113806100206000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806324ec1d3f14602d575b600080fd5b60336035565b005b7fb4b2ff75e30cb4317eaae16dd8a187dd89978df17565104caa6c2797caae27d460405180604001604052806001815260200160028152506040516078919060ba565b60405180910390a1565b6040820160008201516096600085018260ad565b50602082015160a7602085018260ad565b50505050565b60b48160d3565b82525050565b600060408201905060cd60008301846082565b92915050565b600081905091905056fea26469706673582212208823628796125bf9941ce4eda18da1be3cf2931b231708ab848e1bd7151c0c9a64736f6c63430008070033"}, []string{`[{"anonymous":false,"inputs":[{"components":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"uint256","name":"b","type":"uint256"}],"indexed":false,"internalType":"struct Test.MyStruct","name":"s","type":"tuple"}],"name":"StructEvent","type":"event"},{"inputs":[],"name":"TestEvent","outputs":[],"stateMutability":"nonpayable","type":"function"}]`}, ` + "context" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" @@ -1828,12 +1830,23 @@ var bindTests = []struct { } sim.Commit() - _, err = d.TestEvent(user) + tx, err := d.TestEvent(user) if err != nil { t.Fatalf("Failed to call contract %v", err) } sim.Commit() + // Wait for the transaction to be mined + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + receipt, err := bind.WaitMined(ctx, sim, tx) + if err != nil { + t.Fatalf("Failed to wait for tx to be mined: %v", err) + } + if receipt.Status != types.ReceiptStatusSuccessful { + t.Fatal("Transaction failed") + } + it, err := d.FilterStructEvent(nil) if err != nil { t.Fatalf("Failed to filter contract event %v", err) @@ -1862,7 +1875,7 @@ var bindTests = []struct { `NewErrors`, ` pragma solidity >0.8.4; - + contract NewErrors { error MyError(uint256); error MyError1(uint256); @@ -1878,7 +1891,7 @@ var bindTests = []struct { ` "context" "math/big" - + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/core/types" @@ -1892,7 +1905,7 @@ var bindTests = []struct { sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) defer sim.Close() - + _, tx, contract, err := DeployNewErrors(user, sim) if err != nil { t.Fatal(err) @@ -1917,12 +1930,12 @@ var bindTests = []struct { name: `ConstructorWithStructParam`, contract: ` pragma solidity >=0.8.0 <0.9.0; - + contract ConstructorWithStructParam { struct StructType { uint256 field; } - + constructor(StructType memory st) {} } `, @@ -1951,7 +1964,7 @@ var bindTests = []struct { t.Fatalf("DeployConstructorWithStructParam() got err %v; want nil err", err) } sim.Commit() - + if _, err = bind.WaitDeployed(context.Background(), sim, tx); err != nil { t.Logf("Deployment tx: %+v", tx) t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) @@ -2000,7 +2013,7 @@ var bindTests = []struct { t.Fatalf("DeployNameConflict() got err %v; want nil err", err) } sim.Commit() - + if _, err = bind.WaitDeployed(context.Background(), sim, tx); err != nil { t.Logf("Deployment tx: %+v", tx) t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) From a9e6c8daae7aa2691e227d7a79323f306f529ffd Mon Sep 17 00:00:00 2001 From: Ng Wei Han <47109095+weiihann@users.noreply.github.com> Date: Wed, 2 Apr 2025 15:06:54 +0800 Subject: [PATCH 07/35] triedb/pathdb: improve perf by separating nodes map (#31306) This PR refactors the `nodeSet` structure in the path database to use separate maps for account and storage trie nodes, resulting in performance improvements. The change maintains the same API while optimizing the internal data structure. --- triedb/pathdb/nodes.go | 195 ++++++++++++++++++++++++++--------------- 1 file changed, 126 insertions(+), 69 deletions(-) diff --git a/triedb/pathdb/nodes.go b/triedb/pathdb/nodes.go index c56e38066b..f90bd0f01c 100644 --- a/triedb/pathdb/nodes.go +++ b/triedb/pathdb/nodes.go @@ -36,8 +36,9 @@ import ( // transition, typically corresponding to a block execution. It can also represent // the combined trie node set from several aggregated state transitions. type nodeSet struct { - size uint64 // aggregated size of the trie node - nodes map[common.Hash]map[string]*trienode.Node // node set, mapped by owner and path + size uint64 // aggregated size of the trie node + accountNodes map[string]*trienode.Node // account trie nodes, mapped by path + storageNodes map[common.Hash]map[string]*trienode.Node // storage trie nodes, mapped by owner and path } // newNodeSet constructs the set with the provided dirty trie nodes. @@ -46,7 +47,17 @@ func newNodeSet(nodes map[common.Hash]map[string]*trienode.Node) *nodeSet { if nodes == nil { nodes = make(map[common.Hash]map[string]*trienode.Node) } - s := &nodeSet{nodes: nodes} + s := &nodeSet{ + accountNodes: make(map[string]*trienode.Node), + storageNodes: make(map[common.Hash]map[string]*trienode.Node), + } + for owner, subset := range nodes { + if owner == (common.Hash{}) { + s.accountNodes = subset + } else { + s.storageNodes[owner] = subset + } + } s.computeSize() return s } @@ -54,13 +65,12 @@ func newNodeSet(nodes map[common.Hash]map[string]*trienode.Node) *nodeSet { // computeSize calculates the database size of the held trie nodes. func (s *nodeSet) computeSize() { var size uint64 - for owner, subset := range s.nodes { - var prefix int - if owner != (common.Hash{}) { - prefix = common.HashLength // owner (32 bytes) for storage trie nodes - } + for path, n := range s.accountNodes { + size += uint64(len(n.Blob) + len(path)) + } + for _, subset := range s.storageNodes { for path, n := range subset { - size += uint64(prefix + len(n.Blob) + len(path)) + size += uint64(common.HashLength + len(n.Blob) + len(path)) } } s.size = size @@ -79,15 +89,18 @@ func (s *nodeSet) updateSize(delta int64) { // node retrieves the trie node with node path and its trie identifier. func (s *nodeSet) node(owner common.Hash, path []byte) (*trienode.Node, bool) { - subset, ok := s.nodes[owner] + // Account trie node + if owner == (common.Hash{}) { + n, ok := s.accountNodes[string(path)] + return n, ok + } + // Storage trie node + subset, ok := s.storageNodes[owner] if !ok { return nil, false } n, ok := subset[string(path)] - if !ok { - return nil, false - } - return n, true + return n, ok } // merge integrates the provided dirty nodes into the set. The provided nodeset @@ -97,15 +110,24 @@ func (s *nodeSet) merge(set *nodeSet) { delta int64 // size difference resulting from node merging overwrite counter // counter of nodes being overwritten ) - for owner, subset := range set.nodes { - var prefix int - if owner != (common.Hash{}) { - prefix = common.HashLength + + // Merge account nodes + for path, n := range set.accountNodes { + if orig, exist := s.accountNodes[path]; !exist { + delta += int64(len(n.Blob) + len(path)) + } else { + delta += int64(len(n.Blob) - len(orig.Blob)) + overwrite.add(len(orig.Blob) + len(path)) } - current, exist := s.nodes[owner] + s.accountNodes[path] = n + } + + // Merge storage nodes + for owner, subset := range set.storageNodes { + current, exist := s.storageNodes[owner] if !exist { for path, n := range subset { - delta += int64(prefix + len(n.Blob) + len(path)) + delta += int64(common.HashLength + len(n.Blob) + len(path)) } // Perform a shallow copy of the map for the subset instead of claiming it // directly from the provided nodeset to avoid potential concurrent map @@ -113,19 +135,19 @@ func (s *nodeSet) merge(set *nodeSet) { // accessible even after merging. Therefore, ownership of the nodes map // should still belong to the original layer, and any modifications to it // should be prevented. - s.nodes[owner] = maps.Clone(subset) + s.storageNodes[owner] = maps.Clone(subset) continue } for path, n := range subset { if orig, exist := current[path]; !exist { - delta += int64(prefix + len(n.Blob) + len(path)) + delta += int64(common.HashLength + len(n.Blob) + len(path)) } else { delta += int64(len(n.Blob) - len(orig.Blob)) - overwrite.add(prefix + len(orig.Blob) + len(path)) + overwrite.add(common.HashLength + len(orig.Blob) + len(path)) } current[path] = n } - s.nodes[owner] = current + s.storageNodes[owner] = current } overwrite.report(gcTrieNodeMeter, gcTrieNodeBytesMeter) s.updateSize(delta) @@ -136,34 +158,38 @@ func (s *nodeSet) merge(set *nodeSet) { func (s *nodeSet) revertTo(db ethdb.KeyValueReader, nodes map[common.Hash]map[string]*trienode.Node) { var delta int64 for owner, subset := range nodes { - current, ok := s.nodes[owner] - if !ok { - panic(fmt.Sprintf("non-existent subset (%x)", owner)) - } - for path, n := range subset { - orig, ok := current[path] - if !ok { - // There is a special case in merkle tree that one child is removed - // from a fullNode which only has two children, and then a new child - // with different position is immediately inserted into the fullNode. - // In this case, the clean child of the fullNode will also be marked - // as dirty because of node collapse and expansion. In case of database - // rollback, don't panic if this "clean" node occurs which is not - // present in buffer. - var blob []byte - if owner == (common.Hash{}) { - blob = rawdb.ReadAccountTrieNode(db, []byte(path)) - } else { - blob = rawdb.ReadStorageTrieNode(db, owner, []byte(path)) + if owner == (common.Hash{}) { + // Account trie nodes + for path, n := range subset { + orig, ok := s.accountNodes[path] + if !ok { + blob := rawdb.ReadAccountTrieNode(db, []byte(path)) + if bytes.Equal(blob, n.Blob) { + continue + } + panic(fmt.Sprintf("non-existent account node (%v) blob: %v", path, crypto.Keccak256Hash(n.Blob).Hex())) } - // Ignore the clean node in the case described above. - if bytes.Equal(blob, n.Blob) { - continue - } - panic(fmt.Sprintf("non-existent node (%x %v) blob: %v", owner, path, crypto.Keccak256Hash(n.Blob).Hex())) + s.accountNodes[path] = n + delta += int64(len(n.Blob)) - int64(len(orig.Blob)) + } + } else { + // Storage trie nodes + current, ok := s.storageNodes[owner] + if !ok { + panic(fmt.Sprintf("non-existent subset (%x)", owner)) + } + for path, n := range subset { + orig, ok := current[path] + if !ok { + blob := rawdb.ReadStorageTrieNode(db, owner, []byte(path)) + if bytes.Equal(blob, n.Blob) { + continue + } + panic(fmt.Sprintf("non-existent storage node (%x %v) blob: %v", owner, path, crypto.Keccak256Hash(n.Blob).Hex())) + } + current[path] = n + delta += int64(len(n.Blob)) - int64(len(orig.Blob)) } - current[path] = n - delta += int64(len(n.Blob)) - int64(len(orig.Blob)) } } s.updateSize(delta) @@ -184,8 +210,21 @@ type journalNodes struct { // encode serializes the content of trie nodes into the provided writer. func (s *nodeSet) encode(w io.Writer) error { - nodes := make([]journalNodes, 0, len(s.nodes)) - for owner, subset := range s.nodes { + nodes := make([]journalNodes, 0, len(s.storageNodes)+1) + + // Encode account nodes + if len(s.accountNodes) > 0 { + entry := journalNodes{Owner: common.Hash{}} + for path, node := range s.accountNodes { + entry.Nodes = append(entry.Nodes, journalNode{ + Path: []byte(path), + Blob: node.Blob, + }) + } + nodes = append(nodes, entry) + } + // Encode storage nodes + for owner, subset := range s.storageNodes { entry := journalNodes{Owner: owner} for path, node := range subset { entry.Nodes = append(entry.Nodes, journalNode{ @@ -204,43 +243,61 @@ func (s *nodeSet) decode(r *rlp.Stream) error { if err := r.Decode(&encoded); err != nil { return fmt.Errorf("load nodes: %v", err) } - nodes := make(map[common.Hash]map[string]*trienode.Node) + s.accountNodes = make(map[string]*trienode.Node) + s.storageNodes = make(map[common.Hash]map[string]*trienode.Node) + for _, entry := range encoded { - subset := make(map[string]*trienode.Node) - for _, n := range entry.Nodes { - if len(n.Blob) > 0 { - subset[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob) - } else { - subset[string(n.Path)] = trienode.NewDeleted() + if entry.Owner == (common.Hash{}) { + // Account nodes + for _, n := range entry.Nodes { + if len(n.Blob) > 0 { + s.accountNodes[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob) + } else { + s.accountNodes[string(n.Path)] = trienode.NewDeleted() + } } + } else { + // Storage nodes + subset := make(map[string]*trienode.Node) + for _, n := range entry.Nodes { + if len(n.Blob) > 0 { + subset[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob) + } else { + subset[string(n.Path)] = trienode.NewDeleted() + } + } + s.storageNodes[entry.Owner] = subset } - nodes[entry.Owner] = subset } - s.nodes = nodes s.computeSize() return nil } // write flushes nodes into the provided database batch as a whole. func (s *nodeSet) write(batch ethdb.Batch, clean *fastcache.Cache) int { - return writeNodes(batch, s.nodes, clean) + nodes := make(map[common.Hash]map[string]*trienode.Node) + if len(s.accountNodes) > 0 { + nodes[common.Hash{}] = s.accountNodes + } + for owner, subset := range s.storageNodes { + nodes[owner] = subset + } + return writeNodes(batch, nodes, clean) } // reset clears all cached trie node data. func (s *nodeSet) reset() { - s.nodes = make(map[common.Hash]map[string]*trienode.Node) + s.accountNodes = make(map[string]*trienode.Node) + s.storageNodes = make(map[common.Hash]map[string]*trienode.Node) s.size = 0 } // dbsize returns the approximate size of db write. func (s *nodeSet) dbsize() int { var m int - for owner, nodes := range s.nodes { - if owner == (common.Hash{}) { - m += len(nodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix - } else { - m += len(nodes) * (len(rawdb.TrieNodeStoragePrefix)) // database key prefix - } + m += len(s.accountNodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix + for _, nodes := range s.storageNodes { + m += len(nodes) * (len(rawdb.TrieNodeStoragePrefix)) // database key prefix } return m + int(s.size) } From ee30681a8d4d176a3561db20e9c8867dafe97441 Mon Sep 17 00:00:00 2001 From: minh-bq Date: Wed, 2 Apr 2025 14:47:56 +0700 Subject: [PATCH 08/35] core/txpool: add GetMetadata to transaction pool (#31433) This is an alternative to #31309 With eth/68, transaction announcement must have transaction type and size. So in announceTransactions, we need to query the transaction from transaction pool with its hash. This creates overhead in case of blob transaction which needs to load data from billy and RLP decode. This commit creates a lightweight lookup from transaction hash to transaction size and a function GetMetadata to query transaction type and transaction size given the transaction hash. --------- Co-authored-by: Gary Rong --- core/txpool/blobpool/blobpool.go | 81 ++++++++++++++++---------- core/txpool/blobpool/blobpool_test.go | 12 +++- core/txpool/blobpool/evictheap_test.go | 8 +-- core/txpool/blobpool/lookup.go | 35 ++++++++--- core/txpool/legacypool/legacypool.go | 13 +++++ core/txpool/subpool.go | 10 ++++ core/txpool/txpool.go | 11 ++++ eth/handler.go | 4 ++ eth/handler_test.go | 16 +++++ eth/protocols/eth/broadcast.go | 6 +- eth/protocols/eth/handler.go | 5 ++ 11 files changed, 156 insertions(+), 45 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 59a5645040..5a20c3ce5a 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -87,8 +87,9 @@ type blobTxMeta struct { hash common.Hash // Transaction hash to maintain the lookup table vhashes []common.Hash // Blob versioned hashes to maintain the lookup table - id uint64 // Storage ID in the pool's persistent store - size uint32 // Byte size in the pool's persistent store + id uint64 // Storage ID in the pool's persistent store + storageSize uint32 // Byte size in the pool's persistent store + size uint64 // RLP-encoded size of transaction including the attached blob nonce uint64 // Needed to prioritize inclusion order within an account costCap *uint256.Int // Needed to validate cumulative balance sufficiency @@ -108,19 +109,20 @@ type blobTxMeta struct { // newBlobTxMeta retrieves the indexed metadata fields from a blob transaction // and assembles a helper struct to track in memory. -func newBlobTxMeta(id uint64, size uint32, tx *types.Transaction) *blobTxMeta { +func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transaction) *blobTxMeta { meta := &blobTxMeta{ - hash: tx.Hash(), - vhashes: tx.BlobHashes(), - id: id, - size: size, - nonce: tx.Nonce(), - costCap: uint256.MustFromBig(tx.Cost()), - execTipCap: uint256.MustFromBig(tx.GasTipCap()), - execFeeCap: uint256.MustFromBig(tx.GasFeeCap()), - blobFeeCap: uint256.MustFromBig(tx.BlobGasFeeCap()), - execGas: tx.Gas(), - blobGas: tx.BlobGas(), + hash: tx.Hash(), + vhashes: tx.BlobHashes(), + id: id, + storageSize: storageSize, + size: size, + nonce: tx.Nonce(), + costCap: uint256.MustFromBig(tx.Cost()), + execTipCap: uint256.MustFromBig(tx.GasTipCap()), + execFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + blobFeeCap: uint256.MustFromBig(tx.BlobGasFeeCap()), + execGas: tx.Gas(), + blobGas: tx.BlobGas(), } meta.basefeeJumps = dynamicFeeJumps(meta.execFeeCap) meta.blobfeeJumps = dynamicFeeJumps(meta.blobFeeCap) @@ -480,7 +482,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { return errors.New("missing blob sidecar") } - meta := newBlobTxMeta(id, size, tx) + meta := newBlobTxMeta(id, tx.Size(), size, tx) if p.lookup.exists(meta.hash) { // This path is only possible after a crash, where deleted items are not // removed via the normal shutdown-startup procedure and thus may get @@ -507,7 +509,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { p.spent[sender] = new(uint256.Int).Add(p.spent[sender], meta.costCap) p.lookup.track(meta) - p.stored += uint64(meta.size) + p.stored += uint64(meta.storageSize) return nil } @@ -539,7 +541,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 ids = append(ids, txs[i].id) nonces = append(nonces, txs[i].nonce) - p.stored -= uint64(txs[i].size) + p.stored -= uint64(txs[i].storageSize) p.lookup.untrack(txs[i]) // Included transactions blobs need to be moved to the limbo @@ -580,7 +582,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 nonces = append(nonces, txs[0].nonce) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[0].costCap) - p.stored -= uint64(txs[0].size) + p.stored -= uint64(txs[0].storageSize) p.lookup.untrack(txs[0]) // Included transactions blobs need to be moved to the limbo @@ -636,7 +638,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 dropRepeatedMeter.Mark(1) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[i].costCap) - p.stored -= uint64(txs[i].size) + p.stored -= uint64(txs[i].storageSize) p.lookup.untrack(txs[i]) if err := p.store.Delete(id); err != nil { @@ -658,7 +660,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 nonces = append(nonces, txs[j].nonce) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[j].costCap) - p.stored -= uint64(txs[j].size) + p.stored -= uint64(txs[j].storageSize) p.lookup.untrack(txs[j]) } txs = txs[:i] @@ -696,7 +698,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 nonces = append(nonces, last.nonce) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], last.costCap) - p.stored -= uint64(last.size) + p.stored -= uint64(last.storageSize) p.lookup.untrack(last) } if len(txs) == 0 { @@ -736,7 +738,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 nonces = append(nonces, last.nonce) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], last.costCap) - p.stored -= uint64(last.size) + p.stored -= uint64(last.storageSize) p.lookup.untrack(last) } p.index[addr] = txs @@ -1002,7 +1004,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error { } // Update the indices and metrics - meta := newBlobTxMeta(id, p.store.Size(id), tx) + meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), tx) if _, ok := p.index[addr]; !ok { if err := p.reserve(addr, true); err != nil { log.Warn("Failed to reserve account for blob pool", "tx", tx.Hash(), "from", addr, "err", err) @@ -1016,7 +1018,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error { p.spent[addr] = new(uint256.Int).Add(p.spent[addr], meta.costCap) } p.lookup.track(meta) - p.stored += uint64(meta.size) + p.stored += uint64(meta.storageSize) return nil } @@ -1041,7 +1043,7 @@ func (p *BlobPool) SetGasTip(tip *big.Int) { nonces = []uint64{tx.nonce} ) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[i].costCap) - p.stored -= uint64(tx.size) + p.stored -= uint64(tx.storageSize) p.lookup.untrack(tx) txs[i] = nil @@ -1051,7 +1053,7 @@ func (p *BlobPool) SetGasTip(tip *big.Int) { nonces = append(nonces, tx.nonce) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], tx.costCap) - p.stored -= uint64(tx.size) + p.stored -= uint64(tx.storageSize) p.lookup.untrack(tx) txs[i+1+j] = nil } @@ -1236,6 +1238,25 @@ func (p *BlobPool) GetRLP(hash common.Hash) []byte { return p.getRLP(hash) } +// GetMetadata returns the transaction type and transaction size with the +// given transaction hash. +// +// The size refers the length of the 'rlp encoding' of a blob transaction +// including the attached blobs. +func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { + p.lock.RLock() + defer p.lock.RUnlock() + + size, ok := p.lookup.sizeOfTx(hash) + if !ok { + return nil + } + return &txpool.TxMetadata{ + Type: types.BlobTxType, + Size: size, + } +} + // GetBlobs returns a number of blobs are proofs for the given versioned hashes. // This is a utility method for the engine API, enabling consensus clients to // retrieve blobs from the pools directly instead of the network. @@ -1375,7 +1396,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { if err != nil { return err } - meta := newBlobTxMeta(id, p.store.Size(id), tx) + meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), tx) var ( next = p.state.GetNonce(from) @@ -1403,7 +1424,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { p.lookup.untrack(prev) p.lookup.track(meta) - p.stored += uint64(meta.size) - uint64(prev.size) + p.stored += uint64(meta.storageSize) - uint64(prev.storageSize) } else { // Transaction extends previously scheduled ones p.index[from] = append(p.index[from], meta) @@ -1413,7 +1434,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { } p.spent[from] = new(uint256.Int).Add(p.spent[from], meta.costCap) p.lookup.track(meta) - p.stored += uint64(meta.size) + p.stored += uint64(meta.storageSize) } // Recompute the rolling eviction fields. In case of a replacement, this will // recompute all subsequent fields. In case of an append, this will only do @@ -1500,7 +1521,7 @@ func (p *BlobPool) drop() { p.index[from] = txs p.spent[from] = new(uint256.Int).Sub(p.spent[from], drop.costCap) } - p.stored -= uint64(drop.size) + p.stored -= uint64(drop.storageSize) p.lookup.untrack(drop) // Remove the transaction from the pool's eviction heap: diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index d9137cb679..b7c6cfa51e 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -376,7 +376,7 @@ func verifyPoolInternals(t *testing.T, pool *BlobPool) { var stored uint64 for _, txs := range pool.index { for _, tx := range txs { - stored += uint64(tx.size) + stored += uint64(tx.storageSize) } } if pool.stored != stored { @@ -1553,6 +1553,16 @@ func TestAdd(t *testing.T) { if err := pool.add(signed); !errors.Is(err, add.err) { t.Errorf("test %d, tx %d: adding transaction error mismatch: have %v, want %v", i, j, err, add.err) } + if add.err == nil { + size, exist := pool.lookup.sizeOfTx(signed.Hash()) + if !exist { + t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j) + } + if size != signed.Size() { + t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v", + i, j, size, signed.Size()) + } + } verifyPoolInternals(t, pool) } verifyPoolInternals(t, pool) diff --git a/core/txpool/blobpool/evictheap_test.go b/core/txpool/blobpool/evictheap_test.go index e392932401..de4076e298 100644 --- a/core/txpool/blobpool/evictheap_test.go +++ b/core/txpool/blobpool/evictheap_test.go @@ -146,7 +146,7 @@ func TestPriceHeapSorting(t *testing.T) { ) index[addr] = []*blobTxMeta{{ id: uint64(j), - size: 128 * 1024, + storageSize: 128 * 1024, nonce: 0, execTipCap: execTip, execFeeCap: execFee, @@ -205,7 +205,7 @@ func benchmarkPriceHeapReinit(b *testing.B, datacap uint64) { ) index[addr] = []*blobTxMeta{{ id: uint64(i), - size: 128 * 1024, + storageSize: 128 * 1024, nonce: 0, execTipCap: execTip, execFeeCap: execFee, @@ -281,7 +281,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) { ) index[addr] = []*blobTxMeta{{ id: uint64(i), - size: 128 * 1024, + storageSize: 128 * 1024, nonce: 0, execTipCap: execTip, execFeeCap: execFee, @@ -312,7 +312,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) { ) metas[i] = &blobTxMeta{ id: uint64(int(blobs) + i), - size: 128 * 1024, + storageSize: 128 * 1024, nonce: 0, execTipCap: execTip, execFeeCap: execFee, diff --git a/core/txpool/blobpool/lookup.go b/core/txpool/blobpool/lookup.go index b5cf4d3799..7607cd487a 100644 --- a/core/txpool/blobpool/lookup.go +++ b/core/txpool/blobpool/lookup.go @@ -20,18 +20,24 @@ import ( "github.com/ethereum/go-ethereum/common" ) +type txMetadata struct { + id uint64 // the billy id of transction + size uint64 // the RLP encoded size of transaction (blobs are included) +} + // lookup maps blob versioned hashes to transaction hashes that include them, -// and transaction hashes to billy entries that include them. +// transaction hashes to billy entries that include them, transaction hashes +// to the transaction size type lookup struct { blobIndex map[common.Hash]map[common.Hash]struct{} - txIndex map[common.Hash]uint64 + txIndex map[common.Hash]*txMetadata } // newLookup creates a new index for tracking blob to tx; and tx to billy mappings. func newLookup() *lookup { return &lookup{ blobIndex: make(map[common.Hash]map[common.Hash]struct{}), - txIndex: make(map[common.Hash]uint64), + txIndex: make(map[common.Hash]*txMetadata), } } @@ -43,8 +49,11 @@ func (l *lookup) exists(txhash common.Hash) bool { // storeidOfTx returns the datastore storage item id of a transaction. func (l *lookup) storeidOfTx(txhash common.Hash) (uint64, bool) { - id, ok := l.txIndex[txhash] - return id, ok + meta, ok := l.txIndex[txhash] + if !ok { + return 0, false + } + return meta.id, true } // storeidOfBlob returns the datastore storage item id of a blob. @@ -61,6 +70,15 @@ func (l *lookup) storeidOfBlob(vhash common.Hash) (uint64, bool) { return 0, false // Weird, don't choke } +// sizeOfTx returns the RLP-encoded size of transaction +func (l *lookup) sizeOfTx(txhash common.Hash) (uint64, bool) { + meta, ok := l.txIndex[txhash] + if !ok { + return 0, false + } + return meta.size, true +} + // track inserts a new set of mappings from blob versioned hashes to transaction // hashes; and from transaction hashes to datastore storage item ids. func (l *lookup) track(tx *blobTxMeta) { @@ -71,8 +89,11 @@ func (l *lookup) track(tx *blobTxMeta) { } l.blobIndex[vhash][tx.hash] = struct{}{} // may be double mapped if a tx contains the same blob twice } - // Map the transaction hash to the datastore id - l.txIndex[tx.hash] = tx.id + // Map the transaction hash to the datastore id and RLP-encoded transaction size + l.txIndex[tx.hash] = &txMetadata{ + id: tx.id, + size: tx.size, + } } // untrack removes a set of mappings from blob versioned hashes to transaction diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index dafd185836..9066f3e16b 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1035,6 +1035,19 @@ func (pool *LegacyPool) GetRLP(hash common.Hash) []byte { return encoded } +// GetMetadata returns the transaction type and transaction size with the +// given transaction hash. +func (pool *LegacyPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { + tx := pool.all.Get(hash) + if tx == nil { + return nil + } + return &txpool.TxMetadata{ + Type: tx.Type(), + Size: tx.Size(), + } +} + // GetBlobs is not supported by the legacy transaction pool, it is just here to // implement the txpool.SubPool interface. func (pool *LegacyPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Proof) { diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index 1392cfb274..f5cb852d8f 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -86,6 +86,12 @@ type PendingFilter struct { OnlyBlobTxs bool // Return only blob transactions (block blob-space filling) } +// TxMetadata denotes the metadata of a transaction. +type TxMetadata struct { + Type uint8 // The type of the transaction + Size uint64 // The length of the 'rlp encoding' of a transaction +} + // SubPool represents a specialized transaction pool that lives on its own (e.g. // blob pool). Since independent of how many specialized pools we have, they do // need to be updated in lockstep and assemble into one coherent view for block @@ -127,6 +133,10 @@ type SubPool interface { // GetRLP returns a RLP-encoded transaction if it is contained in the pool. GetRLP(hash common.Hash) []byte + // GetMetadata returns the transaction type and transaction size with the + // given transaction hash. + GetMetadata(hash common.Hash) *TxMetadata + // GetBlobs returns a number of blobs are proofs for the given versioned hashes. // This is a utility method for the engine API, enabling consensus clients to // retrieve blobs from the pools directly instead of the network. diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 649c5d1a78..083aac92c6 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -354,6 +354,17 @@ func (p *TxPool) GetRLP(hash common.Hash) []byte { return nil } +// GetMetadata returns the transaction type and transaction size with the given +// hash. +func (p *TxPool) GetMetadata(hash common.Hash) *TxMetadata { + for _, subpool := range p.subpools { + if meta := subpool.GetMetadata(hash); meta != nil { + return meta + } + } + return nil +} + // GetBlobs returns a number of blobs are proofs for the given versioned hashes. // This is a utility method for the engine API, enabling consensus clients to // retrieve blobs from the pools directly instead of the network. diff --git a/eth/handler.go b/eth/handler.go index 7179c9980b..b2ad6effdb 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -71,6 +71,10 @@ type txPool interface { // with given tx hash. GetRLP(hash common.Hash) []byte + // GetMetadata returns the transaction type and transaction size with the + // given transaction hash. + GetMetadata(hash common.Hash) *txpool.TxMetadata + // Add should add the given transactions to the pool. Add(txs []*types.Transaction, sync bool) []error diff --git a/eth/handler_test.go b/eth/handler_test.go index 0c6b9854e6..fb3103f241 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -93,6 +93,22 @@ func (p *testTxPool) GetRLP(hash common.Hash) []byte { return nil } +// GetMetadata returns the transaction type and transaction size with the given +// hash. +func (p *testTxPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { + p.lock.Lock() + defer p.lock.Unlock() + + tx := p.pool[hash] + if tx != nil { + return &txpool.TxMetadata{ + Type: tx.Type(), + Size: tx.Size(), + } + } + return nil +} + // Add appends a batch of transactions to the pool, and notifies any // listeners if the addition channel is non nil func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error { diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index f0ed1d6bc9..21cea0d4ef 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -116,10 +116,10 @@ func (p *Peer) announceTransactions() { size common.StorageSize ) for count = 0; count < len(queue) && size < maxTxPacketSize; count++ { - if tx := p.txpool.Get(queue[count]); tx != nil { + if meta := p.txpool.GetMetadata(queue[count]); meta != nil { pending = append(pending, queue[count]) - pendingTypes = append(pendingTypes, tx.Type()) - pendingSizes = append(pendingSizes, uint32(tx.Size())) + pendingTypes = append(pendingTypes, meta.Type) + pendingSizes = append(pendingSizes, uint32(meta.Size)) size += common.HashLength } } diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index eca6777bd6..f2a3cb0292 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" @@ -90,6 +91,10 @@ type TxPool interface { // GetRLP retrieves the RLP-encoded transaction from the local txpool with // the given hash. GetRLP(hash common.Hash) []byte + + // GetMetadata returns the transaction type and transaction size with the + // given transaction hash. + GetMetadata(hash common.Hash) *txpool.TxMetadata } // MakeProtocols constructs the P2P protocol definitions for `eth`. From 3e4fbce034b384c99afeead6cf0f72be0a2b8f13 Mon Sep 17 00:00:00 2001 From: thinkAfCod Date: Wed, 2 Apr 2025 19:47:44 +0800 Subject: [PATCH 09/35] p2p/discover: repeat exact encoding when resending WHOAREYOU packet (#31543) When resending the WHOAREYOU packet, a new nonce and random IV should not be generated. The sent packet needs to match the previously-sent one exactly in order to make the handshake retry work. --------- Co-authored-by: Felix Lange --- p2p/discover/v5_udp_test.go | 40 +++++++++++++++++++++++---------- p2p/discover/v5wire/encoding.go | 16 +++++++++++-- p2p/discover/v5wire/msg.go | 3 +++ 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 3026dff538..606b35c4f2 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -181,29 +181,35 @@ func TestUDPv5_handshakeRepeatChallenge(t *testing.T) { nonce1 := v5wire.Nonce{1} nonce2 := v5wire.Nonce{2} nonce3 := v5wire.Nonce{3} - check := func(p *v5wire.Whoareyou, wantNonce v5wire.Nonce) { + var firstAuthTag *v5wire.Nonce + check := func(p *v5wire.Whoareyou, authTag, wantNonce v5wire.Nonce) { t.Helper() if p.Nonce != wantNonce { - t.Error("wrong nonce in WHOAREYOU:", p.Nonce, wantNonce) + t.Error("wrong nonce in WHOAREYOU:", p.Nonce, "want:", wantNonce) + } + if firstAuthTag == nil { + firstAuthTag = &authTag + } else if authTag != *firstAuthTag { + t.Error("wrong auth tag in WHOAREYOU header:", authTag, "want:", *firstAuthTag) } } // Unknown packet from unknown node. test.packetIn(&v5wire.Unknown{Nonce: nonce1}) - test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, _ v5wire.Nonce) { - check(p, nonce1) + test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, authTag v5wire.Nonce) { + check(p, authTag, nonce1) }) // Second unknown packet. Here we expect the response to reference the // first unknown packet. test.packetIn(&v5wire.Unknown{Nonce: nonce2}) - test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, _ v5wire.Nonce) { - check(p, nonce1) + test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, authTag v5wire.Nonce) { + check(p, authTag, nonce1) }) // Third unknown packet. This should still return the first nonce. test.packetIn(&v5wire.Unknown{Nonce: nonce3}) - test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, _ v5wire.Nonce) { - check(p, nonce1) + test.waitPacketOut(func(p *v5wire.Whoareyou, addr netip.AddrPort, authTag v5wire.Nonce) { + check(p, authTag, nonce1) }) } @@ -766,20 +772,30 @@ type testCodecFrame struct { } func (c *testCodec) Encode(toID enode.ID, addr string, p v5wire.Packet, _ *v5wire.Whoareyou) ([]byte, v5wire.Nonce, error) { + // To match the behavior of v5wire.Codec, we return the cached encoding of + // WHOAREYOU challenges. + if wp, ok := p.(*v5wire.Whoareyou); ok && len(wp.Encoded) > 0 { + return wp.Encoded, wp.Nonce, nil + } + c.ctr++ var authTag v5wire.Nonce binary.BigEndian.PutUint64(authTag[:], c.ctr) + penc, _ := rlp.EncodeToBytes(p) + frame, err := rlp.EncodeToBytes(testCodecFrame{c.id, authTag, p.Kind(), penc}) + if err != nil { + return frame, authTag, err + } + // Store recently sent challenges. if w, ok := p.(*v5wire.Whoareyou); ok { - // Store recently sent Whoareyou challenges. + w.Nonce = authTag + w.Encoded = frame if c.sentChallenges == nil { c.sentChallenges = make(map[enode.ID]*v5wire.Whoareyou) } c.sentChallenges[toID] = w } - - penc, _ := rlp.EncodeToBytes(p) - frame, err := rlp.EncodeToBytes(testCodecFrame{c.id, authTag, p.Kind(), penc}) return frame, authTag, err } diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index e50b7cd16d..b16d14eda5 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -189,6 +189,11 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar ) switch { case packet.Kind() == WhoareyouPacket: + // just send the WHOAREYOU packet raw again, rather than the re-encoded challenge data + w := packet.(*Whoareyou) + if len(w.Encoded) > 0 { + return w.Encoded, w.Nonce, nil + } head, err = c.encodeWhoareyou(id, packet.(*Whoareyou)) case challenge != nil: // We have an unanswered challenge, send handshake. @@ -218,15 +223,22 @@ func (c *Codec) Encode(id enode.ID, addr string, packet Packet, challenge *Whoar // Store sent WHOAREYOU challenges. if challenge, ok := packet.(*Whoareyou); ok { challenge.ChallengeData = bytesCopy(&c.buf) + enc, err := c.EncodeRaw(id, head, msgData) + if err != nil { + return nil, Nonce{}, err + } + challenge.Encoded = bytes.Clone(enc) c.sc.storeSentHandshake(id, addr, challenge) - } else if msgData == nil { + return enc, head.Nonce, err + } + + if msgData == nil { headerData := c.buf.Bytes() msgData, err = c.encryptMessage(session, packet, &head, headerData) if err != nil { return nil, Nonce{}, err } } - enc, err := c.EncodeRaw(id, head, msgData) return enc, head.Nonce, err } diff --git a/p2p/discover/v5wire/msg.go b/p2p/discover/v5wire/msg.go index 401db2f6c5..089fd4ebdc 100644 --- a/p2p/discover/v5wire/msg.go +++ b/p2p/discover/v5wire/msg.go @@ -73,6 +73,9 @@ type ( Node *enode.Node sent mclock.AbsTime // for handshake GC. + + // Encoded is packet raw data for sending out, but should not be include in the RLP encoding. + Encoded []byte `rlp:"-"` } // PING is sent during liveness checks. From d2176f463b873567ff8982f362b843a898c429d1 Mon Sep 17 00:00:00 2001 From: thinkAfCod Date: Wed, 2 Apr 2025 20:56:21 +0800 Subject: [PATCH 10/35] p2p/discover: pass node instead of node ID to TALKREQ handler (#31075) This is for the implementation of Portal Network in the Shisui client. Their handler needs access to the node object in order to send further calls to the requesting node. This is a breaking API change but it should be fine, since there are basically no known users of TALKREQ outside of Portal network. --------- Signed-off-by: thinkAfCod Co-authored-by: Felix Lange --- p2p/discover/v5_talk.go | 14 ++++++++++---- p2p/discover/v5_udp.go | 3 +++ p2p/discover/v5_udp_test.go | 6 +++++- p2p/discover/v5wire/encoding.go | 8 ++++++-- p2p/discover/v5wire/encoding_test.go | 14 +++++++------- p2p/discover/v5wire/session.go | 16 ++++++++++++++-- 6 files changed, 45 insertions(+), 16 deletions(-) diff --git a/p2p/discover/v5_talk.go b/p2p/discover/v5_talk.go index 2246b47141..dca09870d8 100644 --- a/p2p/discover/v5_talk.go +++ b/p2p/discover/v5_talk.go @@ -39,7 +39,7 @@ const talkHandlerLaunchTimeout = 400 * time.Millisecond // Note that talk handlers are expected to come up with a response very quickly, within at // most 200ms or so. If the handler takes longer than that, the remote end may time out // and wont receive the response. -type TalkRequestHandler func(enode.ID, *net.UDPAddr, []byte) []byte +type TalkRequestHandler func(*enode.Node, *net.UDPAddr, []byte) []byte type talkSystem struct { transport *UDPv5 @@ -72,13 +72,19 @@ func (t *talkSystem) register(protocol string, handler TalkRequestHandler) { // handleRequest handles a talk request. func (t *talkSystem) handleRequest(id enode.ID, addr netip.AddrPort, req *v5wire.TalkRequest) { + n := t.transport.codec.SessionNode(id, addr.String()) + if n == nil { + // The node must be contained in the session here, since we wouldn't have + // received the request otherwise. + panic("missing node in session") + } t.mutex.Lock() handler, ok := t.handlers[req.Protocol] t.mutex.Unlock() if !ok { resp := &v5wire.TalkResponse{ReqID: req.ReqID} - t.transport.sendResponse(id, addr, resp) + t.transport.sendResponse(n.ID(), addr, resp) return } @@ -90,9 +96,9 @@ func (t *talkSystem) handleRequest(id enode.ID, addr netip.AddrPort, req *v5wire go func() { defer func() { t.slots <- struct{}{} }() udpAddr := &net.UDPAddr{IP: addr.Addr().AsSlice(), Port: int(addr.Port())} - respMessage := handler(id, udpAddr, req.Message) + respMessage := handler(n, udpAddr, req.Message) resp := &v5wire.TalkResponse{ReqID: req.ReqID, Message: respMessage} - t.transport.sendFromAnotherThread(id, addr, resp) + t.transport.sendFromAnotherThread(n.ID(), addr, resp) }() case <-timeout.C: // Couldn't get it in time, drop the request. diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 6f7c797152..9679f5c61a 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -64,6 +64,9 @@ type codecV5 interface { // CurrentChallenge returns the most recent WHOAREYOU challenge that was encoded to given node. // This will return a non-nil value if there is an active handshake attempt with the node, and nil otherwise. CurrentChallenge(id enode.ID, addr string) *v5wire.Whoareyou + + // SessionNode returns a node that has completed the handshake. + SessionNode(id enode.ID, addr string) *enode.Node } // UDPv5 is the implementation of protocol version 5. diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 606b35c4f2..3a384aab12 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -492,7 +492,7 @@ func TestUDPv5_talkHandling(t *testing.T) { defer test.close() var recvMessage []byte - test.udp.RegisterTalkHandler("test", func(id enode.ID, addr *net.UDPAddr, message []byte) []byte { + test.udp.RegisterTalkHandler("test", func(n *enode.Node, addr *net.UDPAddr, message []byte) []byte { recvMessage = message return []byte("test response") }) @@ -811,6 +811,10 @@ func (c *testCodec) Decode(input []byte, addr string) (enode.ID, *enode.Node, v5 return frame.NodeID, nil, p, nil } +func (c *testCodec) SessionNode(id enode.ID, addr string) *enode.Node { + return c.test.nodesByID[id].Node() +} + func (c *testCodec) decodeFrame(input []byte) (frame testCodecFrame, p v5wire.Packet, err error) { if err = rlp.DecodeBytes(input, &frame); err != nil { return frame, nil, fmt.Errorf("invalid frame: %v", err) diff --git a/p2p/discover/v5wire/encoding.go b/p2p/discover/v5wire/encoding.go index b16d14eda5..ec5ef8a261 100644 --- a/p2p/discover/v5wire/encoding.go +++ b/p2p/discover/v5wire/encoding.go @@ -359,7 +359,7 @@ func (c *Codec) encodeHandshakeHeader(toID enode.ID, addr string, challenge *Who } // TODO: this should happen when the first authenticated message is received - c.sc.storeNewSession(toID, addr, session) + c.sc.storeNewSession(toID, addr, session, challenge.Node) // Encode the auth header. var ( @@ -534,7 +534,7 @@ func (c *Codec) decodeHandshakeMessage(fromAddr string, head *Header, headerData } // Handshake OK, drop the challenge and store the new session keys. - c.sc.storeNewSession(auth.h.SrcID, fromAddr, session) + c.sc.storeNewSession(auth.h.SrcID, fromAddr, session, node) c.sc.deleteHandshake(auth.h.SrcID, fromAddr) return node, msg, nil } @@ -656,6 +656,10 @@ func (c *Codec) decryptMessage(input, nonce, headerData, readKey []byte) (Packet return DecodeMessage(msgdata[0], msgdata[1:]) } +func (c *Codec) SessionNode(id enode.ID, addr string) *enode.Node { + return c.sc.readNode(id, addr) +} + // checkValid performs some basic validity checks on the header. // The packetLen here is the length remaining after the static header. func (h *StaticHeader) checkValid(packetLen int, protocolID [6]byte) error { diff --git a/p2p/discover/v5wire/encoding_test.go b/p2p/discover/v5wire/encoding_test.go index df97e40e89..2304d0f132 100644 --- a/p2p/discover/v5wire/encoding_test.go +++ b/p2p/discover/v5wire/encoding_test.go @@ -166,7 +166,7 @@ func TestHandshake_rekey(t *testing.T) { readKey: []byte("BBBBBBBBBBBBBBBB"), writeKey: []byte("AAAAAAAAAAAAAAAA"), } - net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session) + net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session, net.nodeB.n()) // A -> B FINDNODE (encrypted with zero keys) findnode, authTag := net.nodeA.encode(t, net.nodeB, &Findnode{}) @@ -209,8 +209,8 @@ func TestHandshake_rekey2(t *testing.T) { readKey: []byte("CCCCCCCCCCCCCCCC"), writeKey: []byte("DDDDDDDDDDDDDDDD"), } - net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), initKeysA) - net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), initKeysB) + net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), initKeysA, net.nodeB.n()) + net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), initKeysB, net.nodeA.n()) // A -> B FINDNODE encrypted with initKeysA findnode, authTag := net.nodeA.encode(t, net.nodeB, &Findnode{Distances: []uint{3}}) @@ -362,8 +362,8 @@ func TestTestVectorsV5(t *testing.T) { ENRSeq: 2, }, prep: func(net *handshakeTest) { - net.nodeA.c.sc.storeNewSession(idB, addr, session) - net.nodeB.c.sc.storeNewSession(idA, addr, session.keysFlipped()) + net.nodeA.c.sc.storeNewSession(idB, addr, session, net.nodeB.n()) + net.nodeB.c.sc.storeNewSession(idA, addr, session.keysFlipped(), net.nodeA.n()) }, }, { @@ -499,8 +499,8 @@ func BenchmarkV5_DecodePing(b *testing.B) { readKey: []byte{233, 203, 93, 195, 86, 47, 177, 186, 227, 43, 2, 141, 244, 230, 120, 17}, writeKey: []byte{79, 145, 252, 171, 167, 216, 252, 161, 208, 190, 176, 106, 214, 39, 178, 134}, } - net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session) - net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), session.keysFlipped()) + net.nodeA.c.sc.storeNewSession(net.nodeB.id(), net.nodeB.addr(), session, net.nodeB.n()) + net.nodeB.c.sc.storeNewSession(net.nodeA.id(), net.nodeA.addr(), session.keysFlipped(), net.nodeA.n()) addrB := net.nodeA.addr() ping := &Ping{ReqID: []byte("reqid"), ENRSeq: 5} enc, _, err := net.nodeA.c.Encode(net.nodeB.id(), addrB, ping, nil) diff --git a/p2p/discover/v5wire/session.go b/p2p/discover/v5wire/session.go index 862c21fcee..5a2166b143 100644 --- a/p2p/discover/v5wire/session.go +++ b/p2p/discover/v5wire/session.go @@ -54,11 +54,12 @@ type session struct { writeKey []byte readKey []byte nonceCounter uint32 + node *enode.Node } // keysFlipped returns a copy of s with the read and write keys flipped. func (s *session) keysFlipped() *session { - return &session{s.readKey, s.writeKey, s.nonceCounter} + return &session{s.readKey, s.writeKey, s.nonceCounter, s.node} } func NewSessionCache(maxItems int, clock mclock.Clock) *SessionCache { @@ -103,8 +104,19 @@ func (sc *SessionCache) readKey(id enode.ID, addr string) []byte { return nil } +func (sc *SessionCache) readNode(id enode.ID, addr string) *enode.Node { + if s := sc.session(id, addr); s != nil { + return s.node + } + return nil +} + // storeNewSession stores new encryption keys in the cache. -func (sc *SessionCache) storeNewSession(id enode.ID, addr string, s *session) { +func (sc *SessionCache) storeNewSession(id enode.ID, addr string, s *session, n *enode.Node) { + if n == nil { + panic("nil node in storeNewSession") + } + s.node = n sc.sessions.Add(sessionID{id, addr}, s) } From 82f01f9f24192f85de0019c1778ea239af701a41 Mon Sep 17 00:00:00 2001 From: owen <158327443+owenzimmew06@users.noreply.github.com> Date: Wed, 2 Apr 2025 20:19:25 +0300 Subject: [PATCH 11/35] README: fixup typos (#31540) Fixes a few typos in readme. --- cmd/clef/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/clef/README.md b/cmd/clef/README.md index a73ffd5b45..d23e70a3d4 100644 --- a/cmd/clef/README.md +++ b/cmd/clef/README.md @@ -1,6 +1,6 @@ # Clef -Clef can be used to sign transactions and data and is meant as a(n eventual) replacement for Geth's account management. This allows DApps to not depend on Geth's account management. When a DApp wants to sign data (or a transaction), it can send the content to Clef, which will then provide the user with context and asks for permission to sign the content. If the users grants the signing request, Clef will send the signature back to the DApp. +Clef can be used to sign transactions and data and is meant as a(n eventual) replacement for Geth's account management. This allows DApps to not depend on Geth's account management. When a DApp wants to sign data (or a transaction), it can send the content to Clef, which will then provide the user with context and ask for permission to sign the content. If the user grants the signing request, Clef will send the signature back to the DApp. This setup allows a DApp to connect to a remote Ethereum node and send transactions that are locally signed. This can help in situations when a DApp is connected to an untrusted remote Ethereum node, because a local one is not available, not synchronized with the chain, or is a node that has no built-in (or limited) account management. @@ -123,7 +123,7 @@ The External API is **untrusted**: it does not accept credentials, nor does it e Clef has one native console-based UI, for operation without any standalone tools. However, there is also an API to communicate with an external UI. To enable that UI, the signer needs to be executed with the `--stdio-ui` option, which allocates `stdin` / `stdout` for the UI API. -An example (insecure) proof-of-concept of has been implemented in `pythonsigner.py`. +An example (insecure) proof-of-concept has been implemented in `pythonsigner.py`. The model is as follows: @@ -335,7 +335,7 @@ Bash example: #### Arguments - content type [string]: type of signed data - - `text/validator`: hex data with custom validator defined in a contract + - `text/validator`: hex data with a custom validator defined in a contract - `application/clique`: [clique](https://github.com/ethereum/EIPs/issues/225) headers - `text/plain`: simple hex data validated by `account_ecRecover` - account [address]: account to sign with From e6098437a6b15c74ef5eb80ca0b99d38d2dd23d6 Mon Sep 17 00:00:00 2001 From: "fuder.eth" <139509124+vtjl10@users.noreply.github.com> Date: Wed, 2 Apr 2025 22:52:40 +0300 Subject: [PATCH 12/35] all: fix typos in docs and comments (#31548) Co-authored-by: lightclient --- cmd/clef/rules.md | 2 +- cmd/evm/README.md | 2 +- cmd/evm/eest.go | 2 +- eth/downloader/skeleton.go | 2 +- internal/guide/guide_test.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/clef/rules.md b/cmd/clef/rules.md index 3cd1bb52c6..ef89ec00c8 100644 --- a/cmd/clef/rules.md +++ b/cmd/clef/rules.md @@ -2,7 +2,7 @@ The `signer` binary contains a ruleset engine, implemented with [OttoVM](https://github.com/robertkrimen/otto) -It enables usecases like the following: +It enables use cases like the following: * I want to auto-approve transactions with contract `CasinoDapp`, with up to `0.05 ether` in value to maximum `1 ether` per 24h period * I want to auto-approve transaction to contract `EthAlarmClock` with `data`=`0xdeadbeef`, if `value=0`, `gas < 44k` and `gasPrice < 40Gwei` diff --git a/cmd/evm/README.md b/cmd/evm/README.md index f95b6b4d7b..17d663a50c 100644 --- a/cmd/evm/README.md +++ b/cmd/evm/README.md @@ -16,7 +16,7 @@ which can 1. Take a prestate, including - Accounts, - Block context information, - - Previous blockshashes (*optional) + - Previous block hashes (*optional) 2. Apply a set of transactions, 3. Apply a mining-reward (*optional), 4. And generate a post-state, including diff --git a/cmd/evm/eest.go b/cmd/evm/eest.go index 43071a3e5d..4cda2fc517 100644 --- a/cmd/evm/eest.go +++ b/cmd/evm/eest.go @@ -22,7 +22,7 @@ import "regexp" // within its filename by the execution spec test (EEST). type testMetadata struct { fork string - module string // which python module gnerated the test, e.g. eip7702 + module string // which python module generated the test, e.g. eip7702 file string // exact file the test came from, e.g. test_gas.py function string // func that created the test, e.g. test_valid_mcopy_operations parameters string // the name of the parameters which were used to fill the test, e.g. zero_inputs diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index 04421a2bf5..ad74dab844 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -275,7 +275,7 @@ func (s *skeleton) startup() { for { // If the sync cycle terminated or was terminated, propagate up when // higher layers request termination. There's no fancy explicit error - // signalling as the sync loop should never terminate (TM). + // signaling as the sync loop should never terminate (TM). newhead, err := s.sync(head) switch { case err == errSyncLinked: diff --git a/internal/guide/guide_test.go b/internal/guide/guide_test.go index 573898d7d0..71614c125d 100644 --- a/internal/guide/guide_test.go +++ b/internal/guide/guide_test.go @@ -15,7 +15,7 @@ // along with the go-ethereum library. If not, see . // This file contains the code snippets from the developer's guide embedded into -// Go tests. This ensures that any code published in out guides will not break +// Go tests. This ensures that any code published in our guides will not break // accidentally via some code update. If some API changes nonetheless that needs // modifying this file, please port any modification over into the developer's // guide wiki pages too! From 22c0605b68f544cbc6095c50e734eba9758ec34e Mon Sep 17 00:00:00 2001 From: antonis19 Date: Thu, 3 Apr 2025 06:35:52 +0200 Subject: [PATCH 13/35] eth/protocols/eth: improve over/underflow handling in `GetBlockHeaders` (#31522) --- eth/protocols/eth/handler_test.go | 28 ++++++++++++++++++++++++++++ eth/protocols/eth/handlers.go | 17 ++++++++++++----- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 36f5e90c9a..bbbd888701 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -297,6 +297,34 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { backend.chain.GetBlockByNumber(0).Hash(), }, }, + // Check a corner case where skipping causes overflow with reverse=false + { + &GetBlockHeadersRequest{Origin: HashOrNumber{Number: 1}, Amount: 2, Reverse: false, Skip: math.MaxUint64 - 1}, + []common.Hash{ + backend.chain.GetBlockByNumber(1).Hash(), + }, + }, + // Check a corner case where skipping causes overflow with reverse=true + { + &GetBlockHeadersRequest{Origin: HashOrNumber{Number: 1}, Amount: 2, Reverse: true, Skip: math.MaxUint64 - 1}, + []common.Hash{ + backend.chain.GetBlockByNumber(1).Hash(), + }, + }, + // Check another corner case where skipping causes overflow with reverse=false + { + &GetBlockHeadersRequest{Origin: HashOrNumber{Number: 1}, Amount: 2, Reverse: false, Skip: math.MaxUint64}, + []common.Hash{ + backend.chain.GetBlockByNumber(1).Hash(), + }, + }, + // Check another corner case where skipping causes overflow with reverse=true + { + &GetBlockHeadersRequest{Origin: HashOrNumber{Number: 1}, Amount: 2, Reverse: true, Skip: math.MaxUint64}, + []common.Hash{ + backend.chain.GetBlockByNumber(1).Hash(), + }, + }, // Check a corner case where skipping overflow loops back into the chain start { &GetBlockHeadersRequest{Origin: HashOrNumber{Hash: backend.chain.GetBlockByNumber(3).Hash()}, Amount: 2, Reverse: false, Skip: math.MaxUint64 - 1}, diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index ab56be7ffc..fda650da1c 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -128,15 +128,22 @@ func serviceNonContiguousBlockHeaderQuery(chain *core.BlockChain, query *GetBloc } case query.Reverse: // Number based traversal towards the genesis block - if query.Origin.Number >= query.Skip+1 { - query.Origin.Number -= query.Skip + 1 - } else { + current := query.Origin.Number + ancestor := current - (query.Skip + 1) + if ancestor >= current { // check for underflow unknown = true + } else { + query.Origin.Number = ancestor } case !query.Reverse: - // Number based traversal towards the leaf block - query.Origin.Number += query.Skip + 1 + current := query.Origin.Number + next := current + query.Skip + 1 + if next <= current { // check for overflow + unknown = true + } else { + query.Origin.Number = next + } } } return headers From 90d44e715d4f44fdad582f458c5973b0e6463082 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 3 Apr 2025 21:16:35 +0800 Subject: [PATCH 14/35] core, eth/downloader: implement pruning mode sync (#31414) This pull request introduces new sync logic for pruning mode. The downloader will now skip insertion of block bodies and receipts before the configured history cutoff point. Originally, in snap sync, the header chain and other components (bodies and receipts) were inserted separately. However, in Proof-of-Stake, this separation is unnecessary since the sync target is already verified by the CL. To simplify the process, this pull request modifies `InsertReceiptChain` to insert headers along with block bodies and receipts together. Besides, `InsertReceiptChain` doesn't have the notion of reorg, as the common ancestor is always be found before the sync and extra side chain is truncated at the beginning if they fall in the ancient store. The stale canonical chain flags will always be rewritten by the new chain. Explicit reorg logic is no longer required in `InsertReceiptChain`. --- cmd/geth/chaincmd.go | 2 +- cmd/utils/cmd.go | 10 +- cmd/utils/history_test.go | 2 +- core/blockchain.go | 249 +++++++++++---------- core/blockchain_reader.go | 6 + core/blockchain_test.go | 341 +++++++++++++++++++---------- core/error.go | 2 - core/headerchain.go | 10 +- core/rawdb/accessors_chain.go | 24 ++ core/rawdb/accessors_chain_test.go | 42 ++++ core/txindexer.go | 3 +- eth/backend.go | 34 +-- eth/downloader/beaconsync.go | 24 +- eth/downloader/downloader.go | 144 ++++++++---- eth/downloader/metrics.go | 1 - eth/downloader/queue.go | 242 +------------------- eth/downloader/resultstore.go | 4 +- 17 files changed, 598 insertions(+), 542 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 708dd8fb7a..0c36b82af5 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -457,7 +457,7 @@ func importHistory(ctx *cli.Context) error { network = networks[0] } - if err := utils.ImportHistory(chain, db, dir, network); err != nil { + if err := utils.ImportHistory(chain, dir, network); err != nil { return err } fmt.Printf("Import done in %v\n", time.Since(start)) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index d91d6ca5a8..d34e15ebc0 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -246,8 +246,9 @@ func readList(filename string) ([]string, error) { } // ImportHistory imports Era1 files containing historical block information, -// starting from genesis. -func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error { +// starting from genesis. The assumption is held that the provided chain +// segment in Era1 file should all be canonical and verified. +func ImportHistory(chain *core.BlockChain, dir string, network string) error { if chain.CurrentSnapBlock().Number.BitLen() != 0 { return errors.New("history import only supported when starting from genesis") } @@ -308,11 +309,6 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ if err != nil { return fmt.Errorf("error reading receipts %d: %w", it.Number(), err) } - if status, err := chain.HeaderChain().InsertHeaderChain([]*types.Header{block.Header()}, start); err != nil { - return fmt.Errorf("error inserting header %d: %w", it.Number(), err) - } else if status != core.CanonStatTy { - return fmt.Errorf("error inserting header %d, not canon: %v", it.Number(), status) - } if _, err := chain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{receipts}, 2^64-1); err != nil { return fmt.Errorf("error inserting body %d: %w", it.Number(), err) } diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index 8654c454f9..d3c6bda1c5 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -171,7 +171,7 @@ func TestHistoryImportAndExport(t *testing.T) { if err != nil { t.Fatalf("unable to initialize chain: %v", err) } - if err := ImportHistory(imported, db2, dir, "mainnet"); err != nil { + if err := ImportHistory(imported, dir, "mainnet"); err != nil { t.Fatalf("failed to import chain: %v", err) } if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { diff --git a/core/blockchain.go b/core/blockchain.go index d80236c902..d56996dadb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -24,6 +24,7 @@ import ( "math/big" "runtime" "slices" + "sort" "strings" "sync" "sync/atomic" @@ -157,7 +158,8 @@ type CacheConfig struct { // This defines the cutoff block for history expiry. // Blocks before this number may be unavailable in the chain database. - HistoryPruningCutoff uint64 + HistoryPruningCutoffNumber uint64 + HistoryPruningCutoffHash common.Hash } // triedbConfig derives the configures for trie database. @@ -262,7 +264,6 @@ type BlockChain struct { txLookupLock sync.RWMutex txLookupCache *lru.Cache[common.Hash, txLookup] - wg sync.WaitGroup quit chan struct{} // shutdown signal, closed in Stop. stopping atomic.Bool // false if chain is running, true when stopped procInterrupt atomic.Bool // interrupt signaler for block processing @@ -333,10 +334,10 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis bc.processor = NewStateProcessor(chainConfig, bc.hc) genesisHeader := bc.GetHeaderByNumber(0) - bc.genesisBlock = types.NewBlockWithHeader(genesisHeader) - if bc.genesisBlock == nil { + if genesisHeader == nil { return nil, ErrNoGenesis } + bc.genesisBlock = types.NewBlockWithHeader(genesisHeader) bc.currentBlock.Store(nil) bc.currentSnapBlock.Store(nil) @@ -1110,7 +1111,6 @@ func (bc *BlockChain) stopWithoutSaving() { // the mutex should become available quickly. It cannot be taken again after Close has // returned. bc.chainmu.Close() - bc.wg.Wait() } // Stop stops the blockchain service. If any imports are currently in progress @@ -1197,79 +1197,64 @@ const ( SideStatTy ) -// InsertReceiptChain attempts to complete an already existing header chain with -// transaction and receipt data. +// InsertReceiptChain inserts a batch of blocks along with their receipts into +// the database. Unlike InsertChain, this function does not verify the state root +// in the blocks. It is used exclusively for snap sync. All the inserted blocks +// will be regarded as canonical, chain reorg is not supported. +// +// The optional ancientLimit can also be specified and chain segment before that +// will be directly stored in the ancient, getting rid of the chain migration. func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts, ancientLimit uint64) (int, error) { - // We don't require the chainMu here since we want to maximize the - // concurrency of header insertion and receipt insertion. - bc.wg.Add(1) - defer bc.wg.Done() + // Verify the supplied headers before insertion without lock + var headers []*types.Header + for _, block := range blockChain { + headers = append(headers, block.Header()) - var ( - ancientBlocks, liveBlocks types.Blocks - ancientReceipts, liveReceipts []types.Receipts - ) - // Do a sanity check that the provided chain is actually ordered and linked - for i, block := range blockChain { - if i != 0 { - prev := blockChain[i-1] - if block.NumberU64() != prev.NumberU64()+1 || block.ParentHash() != prev.Hash() { - log.Error("Non contiguous receipt insert", - "number", block.Number(), "hash", block.Hash(), "parent", block.ParentHash(), - "prevnumber", prev.Number(), "prevhash", prev.Hash()) - return 0, fmt.Errorf("non contiguous insert: item %d is #%d [%x..], item %d is #%d [%x..] (parent [%x..])", - i-1, prev.NumberU64(), prev.Hash().Bytes()[:4], - i, block.NumberU64(), block.Hash().Bytes()[:4], block.ParentHash().Bytes()[:4]) - } - } - if block.NumberU64() <= ancientLimit { - ancientBlocks, ancientReceipts = append(ancientBlocks, block), append(ancientReceipts, receiptChain[i]) - } else { - liveBlocks, liveReceipts = append(liveBlocks, block), append(liveReceipts, receiptChain[i]) - } - - // Here we also validate that blob transactions in the block do not contain a sidecar. - // While the sidecar does not affect the block hash / tx hash, sending blobs within a block is not allowed. + // Here we also validate that blob transactions in the block do not + // contain a sidecar. While the sidecar does not affect the block hash + // or tx hash, sending blobs within a block is not allowed. for txIndex, tx := range block.Transactions() { if tx.Type() == types.BlobTxType && tx.BlobTxSidecar() != nil { return 0, fmt.Errorf("block #%d contains unexpected blob sidecar in tx at index %d", block.NumberU64(), txIndex) } } } + if n, err := bc.hc.ValidateHeaderChain(headers); err != nil { + return n, err + } + // Hold the mutation lock + if !bc.chainmu.TryLock() { + return 0, errChainStopped + } + defer bc.chainmu.Unlock() var ( stats = struct{ processed, ignored int32 }{} start = time.Now() size = int64(0) ) - - // updateHead updates the head snap sync block if the inserted blocks are better - // and returns an indicator whether the inserted blocks are canonical. - updateHead := func(head *types.Block) bool { - if !bc.chainmu.TryLock() { - return false + // updateHead updates the head header and head snap block flags. + updateHead := func(header *types.Header) error { + batch := bc.db.NewBatch() + hash := header.Hash() + rawdb.WriteHeadHeaderHash(batch, hash) + rawdb.WriteHeadFastBlockHash(batch, hash) + if err := batch.Write(); err != nil { + return err } - defer bc.chainmu.Unlock() - - // Rewind may have occurred, skip in that case. - if bc.CurrentHeader().Number.Cmp(head.Number()) >= 0 { - rawdb.WriteHeadFastBlockHash(bc.db, head.Hash()) - bc.currentSnapBlock.Store(head.Header()) - headFastBlockGauge.Update(int64(head.NumberU64())) - return true - } - return false + bc.hc.currentHeader.Store(header) + bc.currentSnapBlock.Store(header) + headHeaderGauge.Update(header.Number.Int64()) + headFastBlockGauge.Update(header.Number.Int64()) + return nil } // writeAncient writes blockchain and corresponding receipt chain into ancient store. // // this function only accepts canonical chain data. All side chain will be reverted // eventually. writeAncient := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { - first := blockChain[0] - last := blockChain[len(blockChain)-1] - - // Ensure genesis is in ancients. - if first.NumberU64() == 1 { + // Ensure genesis is in the ancient store + if blockChain[0].NumberU64() == 1 { if frozen, _ := bc.db.Ancients(); frozen == 0 { writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil}) if err != nil { @@ -1280,12 +1265,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ log.Info("Wrote genesis to ancients") } } - // Before writing the blocks to the ancients, we need to ensure that - // they correspond to the what the headerchain 'expects'. - // We only check the last block/header, since it's a contiguous chain. - if !bc.HasHeader(last.Hash(), last.NumberU64()) { - return 0, fmt.Errorf("containing header #%d [%x..] unknown", last.Number(), last.Hash().Bytes()[:4]) - } // Write all chain data to ancients. writeSize, err := rawdb.WriteAncientBlocks(bc.db, blockChain, receiptChain) if err != nil { @@ -1298,44 +1277,28 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if err := bc.db.Sync(); err != nil { return 0, err } - // Update the current snap block because all block data is now present in DB. - previousSnapBlock := bc.CurrentSnapBlock().Number.Uint64() - if !updateHead(blockChain[len(blockChain)-1]) { - // We end up here if the header chain has reorg'ed, and the blocks/receipts - // don't match the canonical chain. - if _, err := bc.db.TruncateHead(previousSnapBlock + 1); err != nil { - log.Error("Can't truncate ancient store after failed insert", "err", err) - } - return 0, errSideChainReceipts - } - - // Delete block data from the main database. - var ( - batch = bc.db.NewBatch() - canonHashes = make(map[common.Hash]struct{}, len(blockChain)) - ) + // Write hash to number mappings + batch := bc.db.NewBatch() for _, block := range blockChain { - canonHashes[block.Hash()] = struct{}{} - if block.NumberU64() == 0 { - continue - } - rawdb.DeleteCanonicalHash(batch, block.NumberU64()) - rawdb.DeleteBlockWithoutNumber(batch, block.Hash(), block.NumberU64()) - } - // Delete side chain hash-to-number mappings. - for _, nh := range rawdb.ReadAllHashesInRange(bc.db, first.NumberU64(), last.NumberU64()) { - if _, canon := canonHashes[nh.Hash]; !canon { - rawdb.DeleteHeader(batch, nh.Hash, nh.Number) - } + rawdb.WriteHeaderNumber(batch, block.Hash(), block.NumberU64()) } if err := batch.Write(); err != nil { return 0, err } + // Update the current snap block because all block data is now present in DB. + if err := updateHead(blockChain[len(blockChain)-1].Header()); err != nil { + return 0, err + } stats.processed += int32(len(blockChain)) return 0, nil } - // writeLive writes blockchain and corresponding receipt chain into active store. + // writeLive writes the blockchain and corresponding receipt chain to the active store. + // + // Notably, in different snap sync cycles, the supplied chain may partially reorganize + // existing local chain segments (reorg around the chain tip). The reorganized part + // will be included in the provided chain segment, and stale canonical markers will be + // silently rewritten. Therefore, no explicit reorg logic is needed. writeLive := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { var ( skipPresenceCheck = false @@ -1346,10 +1309,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if bc.insertStopped() { return 0, errInsertionInterrupted } - // Short circuit if the owner header is unknown - if !bc.HasHeader(block.Hash(), block.NumberU64()) { - return i, fmt.Errorf("containing header #%d [%x..] unknown", block.Number(), block.Hash().Bytes()[:4]) - } if !skipPresenceCheck { // Ignore if the entire data is already known if bc.HasBlock(block.Hash(), block.NumberU64()) { @@ -1363,7 +1322,8 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } } // Write all the data out into the database - rawdb.WriteBody(batch, block.Hash(), block.NumberU64(), block.Body()) + rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64()) + rawdb.WriteBlock(batch, block) rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i]) // Write everything belongs to the blocks into the database. So that @@ -1387,21 +1347,27 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ return 0, err } } - updateHead(blockChain[len(blockChain)-1]) + if err := updateHead(blockChain[len(blockChain)-1].Header()); err != nil { + return 0, err + } return 0, nil } - // Write downloaded chain data and corresponding receipt chain data - if len(ancientBlocks) > 0 { - if n, err := writeAncient(ancientBlocks, ancientReceipts); err != nil { + // Split the supplied blocks into two groups, according to the + // given ancient limit. + index := sort.Search(len(blockChain), func(i int) bool { + return blockChain[i].NumberU64() >= ancientLimit + }) + if index > 0 { + if n, err := writeAncient(blockChain[:index], receiptChain[:index]); err != nil { if err == errInsertionInterrupted { return 0, nil } return n, err } } - if len(liveBlocks) > 0 { - if n, err := writeLive(liveBlocks, liveReceipts); err != nil { + if index != len(blockChain) { + if n, err := writeLive(blockChain[index:], receiptChain[index:]); err != nil { if err == errInsertionInterrupted { return 0, nil } @@ -1420,7 +1386,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ context = append(context, []interface{}{"ignored", stats.ignored}...) } log.Debug("Imported new block receipts", context...) - return 0, nil } @@ -2505,15 +2470,83 @@ func (bc *BlockChain) InsertHeaderChain(chain []*types.Header) (int, error) { if i, err := bc.hc.ValidateHeaderChain(chain); err != nil { return i, err } - if !bc.chainmu.TryLock() { return 0, errChainStopped } defer bc.chainmu.Unlock() + _, err := bc.hc.InsertHeaderChain(chain, start) return 0, err } +// InsertHeadersBeforeCutoff inserts the given headers into the ancient store +// as they are claimed older than the configured chain cutoff point. All the +// inserted headers are regarded as canonical and chain reorg is not supported. +func (bc *BlockChain) InsertHeadersBeforeCutoff(headers []*types.Header) (int, error) { + if len(headers) == 0 { + return 0, nil + } + // TODO(rjl493456442): Headers before the configured cutoff have already + // been verified by the hash of cutoff header. Theoretically, header validation + // could be skipped here. + if n, err := bc.hc.ValidateHeaderChain(headers); err != nil { + return n, err + } + if !bc.chainmu.TryLock() { + return 0, errChainStopped + } + defer bc.chainmu.Unlock() + + // Initialize the ancient store with genesis block if it's empty. + var ( + frozen, _ = bc.db.Ancients() + first = headers[0].Number.Uint64() + ) + if first == 1 && frozen == 0 { + _, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil}) + if err != nil { + log.Error("Error writing genesis to ancients", "err", err) + return 0, err + } + log.Info("Wrote genesis to ancient store") + } else if frozen != first { + return 0, fmt.Errorf("headers are gapped with the ancient store, first: %d, ancient: %d", first, frozen) + } + + // Write headers to the ancient store, with block bodies and receipts set to nil + // to ensure consistency across tables in the freezer. + _, err := rawdb.WriteAncientHeaderChain(bc.db, headers) + if err != nil { + return 0, err + } + if err := bc.db.Sync(); err != nil { + return 0, err + } + // Write hash to number mappings + batch := bc.db.NewBatch() + for _, header := range headers { + rawdb.WriteHeaderNumber(batch, header.Hash(), header.Number.Uint64()) + } + // Write head header and head snap block flags + last := headers[len(headers)-1] + rawdb.WriteHeadHeaderHash(batch, last.Hash()) + rawdb.WriteHeadFastBlockHash(batch, last.Hash()) + if err := batch.Write(); err != nil { + return 0, err + } + // Truncate the useless chain segment (zero bodies and receipts) in the + // ancient store. + if _, err := bc.db.TruncateTail(last.Number.Uint64() + 1); err != nil { + return 0, err + } + // Last step update all in-memory markers + bc.hc.currentHeader.Store(last) + bc.currentSnapBlock.Store(last) + headHeaderGauge.Update(last.Number.Int64()) + headFastBlockGauge.Update(last.Number.Int64()) + return 0, nil +} + // SetBlockValidatorAndProcessorForTesting sets the current validator and processor. // This method can be used to force an invalid blockchain to be verified for tests. // This method is unsafe and should only be used before block import starts. @@ -2533,9 +2566,3 @@ func (bc *BlockChain) SetTrieFlushInterval(interval time.Duration) { func (bc *BlockChain) GetTrieFlushInterval() time.Duration { return time.Duration(bc.flushInterval.Load()) } - -// HistoryPruningCutoff returns the configured history pruning point. -// Blocks before this might not be available in the database. -func (bc *BlockChain) HistoryPruningCutoff() uint64 { - return bc.cacheConfig.HistoryPruningCutoff -} diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 415a0f5442..4114723469 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -407,6 +407,12 @@ func (bc *BlockChain) TxIndexProgress() (TxIndexProgress, error) { return bc.txIndexer.txIndexProgress() } +// HistoryPruningCutoff returns the configured history pruning point. +// Blocks before this might not be available in the database. +func (bc *BlockChain) HistoryPruningCutoff() (uint64, common.Hash) { + return bc.cacheConfig.HistoryPruningCutoffNumber, bc.cacheConfig.HistoryPruningCutoffHash +} + // TrieDB retrieves the low level trie database used for data storage. func (bc *BlockChain) TrieDB() *triedb.Database { return bc.triedb diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 04dfa87b8b..8f5a64e206 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -733,13 +733,6 @@ func testFastVsFullChains(t *testing.T, scheme string) { fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) defer fast.Stop() - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - if n, err := fast.InsertHeaderChain(headers); err != nil { - t.Fatalf("failed to insert header %d: %v", n, err) - } if n, err := fast.InsertReceiptChain(blocks, receipts, 0); err != nil { t.Fatalf("failed to insert receipt %d: %v", n, err) } @@ -753,9 +746,6 @@ func testFastVsFullChains(t *testing.T, scheme string) { ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) defer ancient.Stop() - if n, err := ancient.InsertHeaderChain(headers); err != nil { - t.Fatalf("failed to insert header %d: %v", n, err) - } if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(len(blocks)/2)); err != nil { t.Fatalf("failed to insert receipt %d: %v", n, err) } @@ -880,13 +870,6 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { fast, _ := NewBlockChain(fastDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) defer fast.Stop() - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - if n, err := fast.InsertHeaderChain(headers); err != nil { - t.Fatalf("failed to insert header %d: %v", n, err) - } if n, err := fast.InsertReceiptChain(blocks, receipts, 0); err != nil { t.Fatalf("failed to insert receipt %d: %v", n, err) } @@ -900,9 +883,6 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) defer ancient.Stop() - if n, err := ancient.InsertHeaderChain(headers); err != nil { - t.Fatalf("failed to insert header %d: %v", n, err) - } if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err != nil { t.Fatalf("failed to insert receipt %d: %v", n, err) } @@ -916,6 +896,11 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) { // Import the chain as a light node and ensure all pointers are updated lightDb := makeDb() defer lightDb.Close() + + headers := make([]*types.Header, len(blocks)) + for i, block := range blocks { + headers[i] = block.Header() + } light, _ := NewBlockChain(lightDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) if n, err := light.InsertHeaderChain(headers); err != nil { t.Fatalf("failed to insert header %d: %v", n, err) @@ -1710,13 +1695,6 @@ func testBlockchainRecovery(t *testing.T, scheme string) { defer ancientDb.Close() ancient, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - if n, err := ancient.InsertHeaderChain(headers); err != nil { - t.Fatalf("failed to insert header %d: %v", n, err) - } if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err != nil { t.Fatalf("failed to insert receipt %d: %v", n, err) } @@ -1741,82 +1719,6 @@ func testBlockchainRecovery(t *testing.T, scheme string) { } } -// This test checks that InsertReceiptChain will roll back correctly when attempting to insert a side chain. -func TestInsertReceiptChainRollback(t *testing.T) { - testInsertReceiptChainRollback(t, rawdb.HashScheme) - testInsertReceiptChainRollback(t, rawdb.PathScheme) -} - -func testInsertReceiptChainRollback(t *testing.T, scheme string) { - // Generate forked chain. The returned BlockChain object is used to process the side chain blocks. - tmpChain, sideblocks, canonblocks, gspec, err := getLongAndShortChains(scheme) - if err != nil { - t.Fatal(err) - } - defer tmpChain.Stop() - // Get the side chain receipts. - if _, err := tmpChain.InsertChain(sideblocks); err != nil { - t.Fatal("processing side chain failed:", err) - } - t.Log("sidechain head:", tmpChain.CurrentBlock().Number, tmpChain.CurrentBlock().Hash()) - sidechainReceipts := make([]types.Receipts, len(sideblocks)) - for i, block := range sideblocks { - sidechainReceipts[i] = tmpChain.GetReceiptsByHash(block.Hash()) - } - // Get the canon chain receipts. - if _, err := tmpChain.InsertChain(canonblocks); err != nil { - t.Fatal("processing canon chain failed:", err) - } - t.Log("canon head:", tmpChain.CurrentBlock().Number, tmpChain.CurrentBlock().Hash()) - canonReceipts := make([]types.Receipts, len(canonblocks)) - for i, block := range canonblocks { - canonReceipts[i] = tmpChain.GetReceiptsByHash(block.Hash()) - } - - // Set up a BlockChain that uses the ancient store. - ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false) - if err != nil { - t.Fatalf("failed to create temp freezer db: %v", err) - } - defer ancientDb.Close() - - ancientChain, _ := NewBlockChain(ancientDb, DefaultCacheConfigWithScheme(scheme), gspec, nil, ethash.NewFaker(), vm.Config{}, nil) - defer ancientChain.Stop() - - // Import the canonical header chain. - canonHeaders := make([]*types.Header, len(canonblocks)) - for i, block := range canonblocks { - canonHeaders[i] = block.Header() - } - if _, err = ancientChain.InsertHeaderChain(canonHeaders); err != nil { - t.Fatal("can't import canon headers:", err) - } - - // Try to insert blocks/receipts of the side chain. - _, err = ancientChain.InsertReceiptChain(sideblocks, sidechainReceipts, uint64(len(sideblocks))) - if err == nil { - t.Fatal("expected error from InsertReceiptChain.") - } - if ancientChain.CurrentSnapBlock().Number.Uint64() != 0 { - t.Fatalf("failed to rollback ancient data, want %d, have %d", 0, ancientChain.CurrentSnapBlock().Number) - } - if frozen, err := ancientChain.db.Ancients(); err != nil || frozen != 1 { - t.Fatalf("failed to truncate ancient data, frozen index is %d", frozen) - } - - // Insert blocks/receipts of the canonical chain. - _, err = ancientChain.InsertReceiptChain(canonblocks, canonReceipts, uint64(len(canonblocks))) - if err != nil { - t.Fatalf("can't import canon chain receipts: %v", err) - } - if ancientChain.CurrentSnapBlock().Number.Uint64() != canonblocks[len(canonblocks)-1].NumberU64() { - t.Fatalf("failed to insert ancient recept chain after rollback") - } - if frozen, _ := ancientChain.db.Ancients(); frozen != uint64(len(canonblocks))+1 { - t.Fatalf("wrong ancients count %d", frozen) - } -} - // Tests that importing a very large side fork, which is larger than the canon chain, // but where the difficulty per block is kept low: this means that it will not // overtake the 'canon' chain until after it's passed canon by about 200 blocks. @@ -2088,14 +1990,6 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) { } } else if typ == "receipts" { inserter = func(blocks []*types.Block, receipts []types.Receipts) error { - headers := make([]*types.Header, 0, len(blocks)) - for _, block := range blocks { - headers = append(headers, block.Header()) - } - _, err := chain.InsertHeaderChain(headers) - if err != nil { - return err - } _, err = chain.InsertReceiptChain(blocks, receipts, 0) return err } @@ -2262,14 +2156,6 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i } } else if typ == "receipts" { inserter = func(blocks []*types.Block, receipts []types.Receipts) error { - headers := make([]*types.Header, 0, len(blocks)) - for _, block := range blocks { - headers = append(headers, block.Header()) - } - i, err := chain.InsertHeaderChain(headers) - if err != nil { - return fmt.Errorf("index %d: %w", i, err) - } _, err = chain.InsertReceiptChain(blocks, receipts, 0) return err } @@ -4265,3 +4151,220 @@ func TestEIP7702(t *testing.T) { t.Fatalf("addr2 storage wrong: expected %d, got %d", fortyTwo, actual) } } + +// Tests the scenario that the synchronization target in snap sync has been changed +// with a chain reorg at the tip. In this case the reorg'd segment should be unmarked +// with canonical flags. +func TestChainReorgSnapSync(t *testing.T) { + testChainReorgSnapSync(t, 0) + testChainReorgSnapSync(t, 32) + testChainReorgSnapSync(t, gomath.MaxUint64) +} + +func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) { + // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) + + // Configure and generate a sample block chain + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + gspec = &Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + signer = types.LatestSigner(gspec.Config) + engine = beacon.New(ethash.NewFaker()) + ) + genDb, blocks, receipts := GenerateChainWithGenesis(gspec, engine, 32, func(i int, block *BlockGen) { + block.SetCoinbase(common.Address{0x00}) + + // If the block number is multiple of 3, send a few bonus transactions to the miner + if i%3 == 2 { + for j := 0; j < i%4+1; j++ { + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) + if err != nil { + panic(err) + } + block.AddTx(tx) + } + } + }) + chainA, receiptsA := GenerateChain(gspec.Config, blocks[len(blocks)-1], engine, genDb, 16, func(i int, gen *BlockGen) { + gen.SetCoinbase(common.Address{0: byte(0xa), 19: byte(i)}) + }) + chainB, receiptsB := GenerateChain(gspec.Config, blocks[len(blocks)-1], engine, genDb, 20, func(i int, gen *BlockGen) { + gen.SetCoinbase(common.Address{0: byte(0xb), 19: byte(i)}) + }) + + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false) + defer db.Close() + + chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil) + defer chain.Stop() + + if n, err := chain.InsertReceiptChain(blocks, receipts, ancientLimit); err != nil { + t.Fatalf("failed to insert receipt %d: %v", n, err) + } + if n, err := chain.InsertReceiptChain(chainA, receiptsA, ancientLimit); err != nil { + t.Fatalf("failed to insert receipt %d: %v", n, err) + } + // If the common ancestor is below the ancient limit, rewind the chain head. + // It's aligned with the behavior in the snap sync + ancestor := blocks[len(blocks)-1].NumberU64() + if ancestor < ancientLimit { + rawdb.WriteLastPivotNumber(db, ancestor) + chain.SetHead(ancestor) + } + if n, err := chain.InsertReceiptChain(chainB, receiptsB, ancientLimit); err != nil { + t.Fatalf("failed to insert receipt %d: %v", n, err) + } + head := chain.CurrentSnapBlock() + if head.Hash() != chainB[len(chainB)-1].Hash() { + t.Errorf("head snap block #%d: header mismatch: want: %v, got: %v", head.Number, chainB[len(chainB)-1].Hash(), head.Hash()) + } + + // Iterate over all chain data components, and cross reference + for i := 0; i < len(blocks); i++ { + num, hash := blocks[i].NumberU64(), blocks[i].Hash() + header := chain.GetHeaderByNumber(num) + if header.Hash() != hash { + t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash()) + } + } + for i := 0; i < len(chainA); i++ { + num, hash := chainA[i].NumberU64(), chainA[i].Hash() + header := chain.GetHeaderByNumber(num) + if header == nil { + continue + } + if header.Hash() == hash { + t.Errorf("block #%d: unexpected canonical header: %v", num, hash) + } + } + for i := 0; i < len(chainB); i++ { + num, hash := chainB[i].NumberU64(), chainB[i].Hash() + header := chain.GetHeaderByNumber(num) + if header.Hash() != hash { + t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash()) + } + } +} + +// Tests the scenario that all the inserted chain segment are with the configured +// chain cutoff point. In this case the chain segment before the cutoff should +// be persisted without the receipts and bodies; chain after should be persisted +// normally. +func TestInsertChainWithCutoff(t *testing.T) { + testInsertChainWithCutoff(t, 32, 32) // cutoff = 32, ancientLimit = 32 + testInsertChainWithCutoff(t, 32, 64) // cutoff = 32, ancientLimit = 64 (entire chain in ancient) + testInsertChainWithCutoff(t, 32, 65) // cutoff = 32, ancientLimit = 65 (64 blocks in ancient, 1 block in live) +} + +func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64) { + // log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true))) + + // Configure and generate a sample block chain + var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + gspec = &Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{address: {Balance: funds}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + signer = types.LatestSigner(gspec.Config) + engine = beacon.New(ethash.NewFaker()) + ) + _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, int(2*cutoff), func(i int, block *BlockGen) { + block.SetCoinbase(common.Address{0x00}) + + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, block.header.BaseFee, nil), signer, key) + if err != nil { + panic(err) + } + block.AddTx(tx) + }) + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false) + defer db.Close() + + cutoffBlock := blocks[cutoff-1] + config := DefaultCacheConfigWithScheme(rawdb.PathScheme) + config.HistoryPruningCutoffNumber = cutoffBlock.NumberU64() + config.HistoryPruningCutoffHash = cutoffBlock.Hash() + + chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil) + defer chain.Stop() + + var ( + headersBefore []*types.Header + blocksAfter []*types.Block + receiptsAfter []types.Receipts + ) + for i, b := range blocks { + if b.NumberU64() < cutoffBlock.NumberU64() { + headersBefore = append(headersBefore, b.Header()) + } else { + blocksAfter = append(blocksAfter, b) + receiptsAfter = append(receiptsAfter, receipts[i]) + } + } + if n, err := chain.InsertHeadersBeforeCutoff(headersBefore); err != nil { + t.Fatalf("failed to insert headers before cutoff %d: %v", n, err) + } + if n, err := chain.InsertReceiptChain(blocksAfter, receiptsAfter, ancientLimit); err != nil { + t.Fatalf("failed to insert receipt %d: %v", n, err) + } + headSnap := chain.CurrentSnapBlock() + if headSnap.Hash() != blocks[len(blocks)-1].Hash() { + t.Errorf("head snap block #%d: header mismatch: want: %v, got: %v", headSnap.Number, blocks[len(blocks)-1].Hash(), headSnap.Hash()) + } + headHeader := chain.CurrentHeader() + if headHeader.Hash() != blocks[len(blocks)-1].Hash() { + t.Errorf("head header #%d: header mismatch: want: %v, got: %v", headHeader.Number, blocks[len(blocks)-1].Hash(), headHeader.Hash()) + } + headBlock := chain.CurrentBlock() + if headBlock.Hash() != gspec.ToBlock().Hash() { + t.Errorf("head block #%d: header mismatch: want: %v, got: %v", headBlock.Number, gspec.ToBlock().Hash(), headBlock.Hash()) + } + + // Iterate over all chain data components, and cross reference + for i := 0; i < len(blocks); i++ { + num, hash := blocks[i].NumberU64(), blocks[i].Hash() + + // Canonical headers should be visible regardless of cutoff + header := chain.GetHeaderByNumber(num) + if header.Hash() != hash { + t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash()) + } + tail, err := db.Tail() + if err != nil { + t.Fatalf("Failed to get chain tail, %v", err) + } + if tail != cutoffBlock.NumberU64() { + t.Fatalf("Unexpected chain tail, want: %d, got: %d", cutoffBlock.NumberU64(), tail) + } + // Block bodies and receipts before the cutoff should be non-existent + if num < cutoffBlock.NumberU64() { + body := chain.GetBody(hash) + if body != nil { + t.Fatalf("Unexpected block body: %d, cutoff: %d", num, cutoffBlock.NumberU64()) + } + receipts := chain.GetReceiptsByHash(hash) + if receipts != nil { + t.Fatalf("Unexpected block receipts: %d, cutoff: %d", num, cutoffBlock.NumberU64()) + } + } else { + body := chain.GetBody(hash) + if body == nil || len(body.Transactions) != 1 { + t.Fatalf("Missed block body: %d, cutoff: %d", num, cutoffBlock.NumberU64()) + } + receipts := chain.GetReceiptsByHash(hash) + if receipts == nil || len(receipts) != 1 { + t.Fatalf("Missed block receipts: %d, cutoff: %d", num, cutoffBlock.NumberU64()) + } + } + } +} diff --git a/core/error.go b/core/error.go index ce3bbb7888..de95e64636 100644 --- a/core/error.go +++ b/core/error.go @@ -28,8 +28,6 @@ var ( // ErrNoGenesis is returned when there is no Genesis Block. ErrNoGenesis = errors.New("genesis not found in chain") - - errSideChainReceipts = errors.New("side blocks can't be accepted as ancient chain data") ) // List of evm-call-message pre-checking errors. All state transition messages will diff --git a/core/headerchain.go b/core/headerchain.go index cb707a152f..f7acc49bef 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -237,7 +237,8 @@ func (hc *HeaderChain) WriteHeaders(headers []*types.Header) (int, error) { } // writeHeadersAndSetHead writes a batch of block headers and applies the last -// header as the chain head if the fork choicer says it's ok to update the chain. +// header as the chain head. +// // Note: This method is not concurrent-safe with inserting blocks simultaneously // into the chain, as side effects caused by reorganisations cannot be emulated // without the real blocks. Hence, writing headers directly should only be done @@ -272,12 +273,14 @@ func (hc *HeaderChain) writeHeadersAndSetHead(headers []*types.Header) (*headerW return result, nil } +// ValidateHeaderChain verifies that the supplied header chain is contiguous +// and conforms to consensus rules. func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header) (int, error) { // Do a sanity check that the provided chain is actually ordered and linked for i := 1; i < len(chain); i++ { if chain[i].Number.Uint64() != chain[i-1].Number.Uint64()+1 { - hash := chain[i].Hash() - parentHash := chain[i-1].Hash() + hash, parentHash := chain[i].Hash(), chain[i-1].Hash() + // Chain broke ancestry, log a message (programming error) and skip insertion log.Error("Non contiguous header insert", "number", chain[i].Number, "hash", hash, "parent", chain[i].ParentHash, "prevnumber", chain[i-1].Number, "prevhash", parentHash) @@ -302,7 +305,6 @@ func (hc *HeaderChain) ValidateHeaderChain(chain []*types.Header) (int, error) { return i, err } } - return 0, nil } diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 020d35619e..2f62d86e4b 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -737,6 +737,30 @@ func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *type return nil } +// WriteAncientHeaderChain writes the supplied headers along with nil block +// bodies and receipts into the ancient store. It's supposed to be used for +// storing chain segment before the chain cutoff. +func WriteAncientHeaderChain(db ethdb.AncientWriter, headers []*types.Header) (int64, error) { + return db.ModifyAncients(func(op ethdb.AncientWriteOp) error { + for _, header := range headers { + num := header.Number.Uint64() + if err := op.AppendRaw(ChainFreezerHashTable, num, header.Hash().Bytes()); err != nil { + return fmt.Errorf("can't add block %d hash: %v", num, err) + } + if err := op.Append(ChainFreezerHeaderTable, num, header); err != nil { + return fmt.Errorf("can't append block header %d: %v", num, err) + } + if err := op.AppendRaw(ChainFreezerBodiesTable, num, nil); err != nil { + return fmt.Errorf("can't append block body %d: %v", num, err) + } + if err := op.AppendRaw(ChainFreezerReceiptTable, num, nil); err != nil { + return fmt.Errorf("can't append block %d receipts: %v", num, err) + } + } + return nil + }) +} + // DeleteBlock removes all block data associated with a hash. func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { DeleteReceipts(db, hash, number) diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index efd16d5fa7..247e277582 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -464,6 +464,48 @@ func TestAncientStorage(t *testing.T) { } } +func TestWriteAncientHeaderChain(t *testing.T) { + db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), t.TempDir(), "", false) + if err != nil { + t.Fatalf("failed to create database with ancient backend") + } + defer db.Close() + + // Create a test block + var headers []*types.Header + headers = append(headers, &types.Header{ + Number: big.NewInt(0), + Extra: []byte("test block"), + UncleHash: types.EmptyUncleHash, + TxHash: types.EmptyTxsHash, + ReceiptHash: types.EmptyReceiptsHash, + }) + headers = append(headers, &types.Header{ + Number: big.NewInt(1), + Extra: []byte("test block"), + UncleHash: types.EmptyUncleHash, + TxHash: types.EmptyTxsHash, + ReceiptHash: types.EmptyReceiptsHash, + }) + // Write and verify the header in the database + WriteAncientHeaderChain(db, headers) + + for _, header := range headers { + if blob := ReadHeaderRLP(db, header.Hash(), header.Number.Uint64()); len(blob) == 0 { + t.Fatalf("no header returned") + } + if h := ReadCanonicalHash(db, header.Number.Uint64()); h != header.Hash() { + t.Fatalf("no canonical hash returned") + } + if blob := ReadBodyRLP(db, header.Hash(), header.Number.Uint64()); len(blob) != 0 { + t.Fatalf("unexpected body returned") + } + if blob := ReadReceiptsRLP(db, header.Hash(), header.Number.Uint64()); len(blob) != 0 { + t.Fatalf("unexpected body returned") + } + } +} + func TestCanonicalHashIteration(t *testing.T) { var cases = []struct { from, to uint64 diff --git a/core/txindexer.go b/core/txindexer.go index d0fce302f3..31f069995b 100644 --- a/core/txindexer.go +++ b/core/txindexer.go @@ -58,9 +58,10 @@ type txIndexer struct { // newTxIndexer initializes the transaction indexer. func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer { + cutoff, _ := chain.HistoryPruningCutoff() indexer := &txIndexer{ limit: limit, - cutoff: chain.HistoryPruningCutoff(), + cutoff: cutoff, db: chain.db, progress: make(chan chan TxIndexProgress), term: make(chan chan struct{}), diff --git a/eth/backend.go b/eth/backend.go index ab612b1de7..79759710b6 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -154,13 +154,18 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } // Validate history pruning configuration. - var historyPruningCutoff uint64 + var ( + cutoffNumber uint64 + cutoffHash common.Hash + ) if config.HistoryMode == ethconfig.PostMergeHistory { prunecfg, ok := ethconfig.HistoryPrunePoints[genesisHash] if !ok { return nil, fmt.Errorf("no history pruning point is defined for genesis %x", genesisHash) } - historyPruningCutoff = prunecfg.BlockNumber + cutoffNumber = prunecfg.BlockNumber + cutoffHash = prunecfg.BlockHash + log.Info("Chain cutoff configured", "number", cutoffNumber, "hash", cutoffHash) } // Set networkID to chainID by default. @@ -204,16 +209,17 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { EnablePreimageRecording: config.EnablePreimageRecording, } cacheConfig = &core.CacheConfig{ - TrieCleanLimit: config.TrieCleanCache, - TrieCleanNoPrefetch: config.NoPrefetch, - TrieDirtyLimit: config.TrieDirtyCache, - TrieDirtyDisabled: config.NoPruning, - TrieTimeLimit: config.TrieTimeout, - SnapshotLimit: config.SnapshotCache, - Preimages: config.Preimages, - StateHistory: config.StateHistory, - StateScheme: scheme, - HistoryPruningCutoff: historyPruningCutoff, + TrieCleanLimit: config.TrieCleanCache, + TrieCleanNoPrefetch: config.NoPrefetch, + TrieDirtyLimit: config.TrieDirtyCache, + TrieDirtyDisabled: config.NoPruning, + TrieTimeLimit: config.TrieTimeout, + SnapshotLimit: config.SnapshotCache, + Preimages: config.Preimages, + StateHistory: config.StateHistory, + StateScheme: scheme, + HistoryPruningCutoffNumber: cutoffNumber, + HistoryPruningCutoffHash: cutoffHash, } ) if config.VMTrace != "" { @@ -246,7 +252,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { HashScheme: scheme == rawdb.HashScheme, } chainView := eth.newChainView(eth.blockchain.CurrentBlock()) - historyCutoff := eth.blockchain.HistoryPruningCutoff() + historyCutoff, _ := eth.blockchain.HistoryPruningCutoff() var finalBlock uint64 if fb := eth.blockchain.CurrentFinalBlock(); fb != nil { finalBlock = fb.Number.Uint64() @@ -443,7 +449,7 @@ func (s *Ethereum) updateFilterMapsHeads() { if head == nil || newHead.Hash() != head.Hash() { head = newHead chainView := s.newChainView(head) - historyCutoff := s.blockchain.HistoryPruningCutoff() + historyCutoff, _ := s.blockchain.HistoryPruningCutoff() var finalBlock uint64 if fb := s.blockchain.CurrentFinalBlock(); fb != nil { finalBlock = fb.Number.Uint64() diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index c142ea7435..33ad0f8971 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -273,8 +273,7 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { // fetchHeaders feeds skeleton headers to the downloader queue for scheduling // until sync errors or is finished. func (d *Downloader) fetchHeaders(from uint64) error { - var head *types.Header - _, tail, _, err := d.skeleton.Bounds() + head, tail, _, err := d.skeleton.Bounds() if err != nil { return err } @@ -294,6 +293,27 @@ func (d *Downloader) fetchHeaders(from uint64) error { fsHeaderContCheckTimer := time.NewTimer(fsHeaderContCheck) defer fsHeaderContCheckTimer.Stop() + // Verify the header at configured chain cutoff, ensuring it's matched with + // the configured hash. Skip the check if the configured cutoff is even higher + // than the sync target, which is definitely not a common case. + if d.chainCutoffNumber != 0 && d.chainCutoffNumber >= from && d.chainCutoffNumber <= head.Number.Uint64() { + h := d.skeleton.Header(d.chainCutoffNumber) + if h == nil { + if d.chainCutoffNumber < tail.Number.Uint64() { + dist := tail.Number.Uint64() - d.chainCutoffNumber + if len(localHeaders) >= int(dist) { + h = localHeaders[dist-1] + } + } + } + if h == nil { + return fmt.Errorf("header at chain cutoff is not available, cutoff: %d", d.chainCutoffNumber) + } + if h.Hash() != d.chainCutoffHash { + return fmt.Errorf("header at chain cutoff mismatched, want: %v, got: %v", d.chainCutoffHash, h.Hash()) + } + } + for { // Some beacon headers might have appeared since the last cycle, make // sure we're always syncing to all available ones diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 3f3f9b7f0c..4d13ae304c 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -20,6 +20,7 @@ package downloader import ( "errors" "fmt" + "sort" "sync" "sync/atomic" "time" @@ -120,6 +121,12 @@ type Downloader struct { committed atomic.Bool ancientLimit uint64 // The maximum block number which can be regarded as ancient data. + // The cutoff block number and hash before which chain segments (bodies + // and receipts) are skipped during synchronization. 0 means the entire + // chain segment is aimed for synchronization. + chainCutoffNumber uint64 + chainCutoffHash common.Hash + // Channels headerProcCh chan *headerTask // Channel to feed the header processor new tasks @@ -163,9 +170,6 @@ type BlockChain interface { // CurrentHeader retrieves the head header from the local chain. CurrentHeader() *types.Header - // InsertHeaderChain inserts a batch of headers into the local chain. - InsertHeaderChain([]*types.Header) (int, error) - // SetHead rewinds the local chain to a new head. SetHead(uint64) error @@ -187,10 +191,17 @@ type BlockChain interface { // SnapSyncCommitHead directly commits the head block to a certain entity. SnapSyncCommitHead(common.Hash) error + // InsertHeadersBeforeCutoff inserts a batch of headers before the configured + // chain cutoff into the ancient store. + InsertHeadersBeforeCutoff([]*types.Header) (int, error) + // InsertChain inserts a batch of blocks into the local chain. InsertChain(types.Blocks) (int, error) - // InsertReceiptChain inserts a batch of receipts into the local chain. + // InsertReceiptChain inserts a batch of blocks along with their receipts + // into the local chain. Blocks older than the specified `ancientLimit` + // are stored directly in the ancient store, while newer blocks are stored + // in the live key-value store. InsertReceiptChain(types.Blocks, []types.Receipts, uint64) (int, error) // Snapshots returns the blockchain snapshot tree to paused it during sync. @@ -199,22 +210,29 @@ type BlockChain interface { // TrieDB retrieves the low level trie database used for interacting // with trie nodes. TrieDB() *triedb.Database + + // HistoryPruningCutoff returns the configured history pruning point. + // Block bodies along with the receipts will be skipped for synchronization. + HistoryPruningCutoff() (uint64, common.Hash) } // New creates a new downloader to fetch hashes and blocks from remote peers. func New(stateDb ethdb.Database, mux *event.TypeMux, chain BlockChain, dropPeer peerDropFn, success func()) *Downloader { + cutoffNumber, cutoffHash := chain.HistoryPruningCutoff() dl := &Downloader{ - stateDB: stateDb, - mux: mux, - queue: newQueue(blockCacheMaxItems, blockCacheInitialItems), - peers: newPeerSet(), - blockchain: chain, - dropPeer: dropPeer, - headerProcCh: make(chan *headerTask, 1), - quitCh: make(chan struct{}), - SnapSyncer: snap.NewSyncer(stateDb, chain.TrieDB().Scheme()), - stateSyncStart: make(chan *stateSync), - syncStartBlock: chain.CurrentSnapBlock().Number.Uint64(), + stateDB: stateDb, + mux: mux, + queue: newQueue(blockCacheMaxItems, blockCacheInitialItems), + peers: newPeerSet(), + blockchain: chain, + chainCutoffNumber: cutoffNumber, + chainCutoffHash: cutoffHash, + dropPeer: dropPeer, + headerProcCh: make(chan *headerTask, 1), + quitCh: make(chan struct{}), + SnapSyncer: snap.NewSyncer(stateDb, chain.TrieDB().Scheme()), + stateSyncStart: make(chan *stateSync), + syncStartBlock: chain.CurrentSnapBlock().Number.Uint64(), } // Create the post-merge skeleton syncer and start the process dl.skeleton = newSkeleton(stateDb, dl.peers, dropPeer, newBeaconBackfiller(dl, success)) @@ -503,6 +521,12 @@ func (d *Downloader) syncToHead() (err error) { } else { d.ancientLimit = 0 } + // Extend the ancient chain segment range if the ancient limit is even + // below the pre-configured chain cutoff. + if d.chainCutoffNumber != 0 && d.chainCutoffNumber > d.ancientLimit { + d.ancientLimit = d.chainCutoffNumber + log.Info("Extend the ancient range with configured cutoff", "cutoff", d.chainCutoffNumber) + } frozen, _ := d.stateDB.Ancients() // Ignore the error here since light client can also hit here. // If a part of blockchain data has already been written into active store, @@ -521,14 +545,23 @@ func (d *Downloader) syncToHead() (err error) { log.Info("Truncated excess ancient chain segment", "oldhead", frozen-1, "newhead", origin) } } + // Skip ancient chain segments if Geth is running with a configured chain cutoff. + // These segments are not guaranteed to be available in the network. + chainOffset := origin + 1 + if mode == ethconfig.SnapSync && d.chainCutoffNumber != 0 { + if chainOffset < d.chainCutoffNumber { + chainOffset = d.chainCutoffNumber + log.Info("Skip chain segment before cutoff", "origin", origin, "cutoff", d.chainCutoffNumber) + } + } // Initiate the sync using a concurrent header and content retrieval algorithm - d.queue.Prepare(origin+1, mode) + d.queue.Prepare(chainOffset, mode) // In beacon mode, headers are served by the skeleton syncer fetchers := []func() error{ - func() error { return d.fetchHeaders(origin + 1) }, // Headers are always retrieved - func() error { return d.fetchBodies(origin + 1) }, // Bodies are retrieved during normal and snap sync - func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during snap sync + func() error { return d.fetchHeaders(origin + 1) }, // Headers are always retrieved + func() error { return d.fetchBodies(chainOffset) }, // Bodies are retrieved during normal and snap sync + func() error { return d.fetchReceipts(chainOffset) }, // Receipts are retrieved during snap sync func() error { return d.processHeaders(origin + 1) }, } if mode == ethconfig.SnapSync { @@ -666,7 +699,7 @@ func (d *Downloader) processHeaders(origin uint64) error { return nil } // Otherwise split the chunk of headers into batches and process them - headers, hashes := task.headers, task.hashes + headers, hashes, scheduled := task.headers, task.hashes, false for len(headers) > 0 { // Terminate if something failed in between processing chunks @@ -683,17 +716,21 @@ func (d *Downloader) processHeaders(origin uint64) error { chunkHeaders := headers[:limit] chunkHashes := hashes[:limit] - // In case of header only syncing, validate the chunk immediately - if mode == ethconfig.SnapSync { - // Although the received headers might be all valid, a legacy - // PoW/PoA sync must not accept post-merge headers. Make sure - // that any transition is rejected at this point. - if len(chunkHeaders) > 0 { - if n, err := d.blockchain.InsertHeaderChain(chunkHeaders); err != nil { - log.Warn("Invalid header encountered", "number", chunkHeaders[n].Number, "hash", chunkHashes[n], "parent", chunkHeaders[n].ParentHash, "err", err) - return fmt.Errorf("%w: %v", errInvalidChain, err) - } + // Split the headers around the chain cutoff + var cutoff int + if mode == ethconfig.SnapSync && d.chainCutoffNumber != 0 { + cutoff = sort.Search(len(chunkHeaders), func(i int) bool { + return chunkHeaders[i].Number.Uint64() >= d.chainCutoffNumber + }) + } + // Insert the header chain into the ancient store (with block bodies and + // receipts set to nil) if they fall before the cutoff. + if mode == ethconfig.SnapSync && cutoff != 0 { + if n, err := d.blockchain.InsertHeadersBeforeCutoff(chunkHeaders[:cutoff]); err != nil { + log.Warn("Failed to insert ancient header chain", "number", chunkHeaders[n].Number, "hash", chunkHashes[n], "parent", chunkHeaders[n].ParentHash, "err", err) + return fmt.Errorf("%w: %v", errInvalidChain, err) } + log.Debug("Inserted headers before cutoff", "number", chunkHeaders[cutoff-1].Number, "hash", chunkHashes[cutoff-1]) } // If we've reached the allowed number of pending headers, stall a bit for d.queue.PendingBodies() >= maxQueuedHeaders || d.queue.PendingReceipts() >= maxQueuedHeaders { @@ -704,12 +741,21 @@ func (d *Downloader) processHeaders(origin uint64) error { case <-timer.C: } } - // Otherwise insert the headers for content retrieval - inserts := d.queue.Schedule(chunkHeaders, chunkHashes, origin) - if len(inserts) != len(chunkHeaders) { - return fmt.Errorf("%w: stale headers", errBadPeer) + // Otherwise, schedule the headers for content retrieval (block bodies and + // potentially receipts in snap sync). + // + // Skip the bodies/receipts retrieval scheduling before the cutoff in snap + // sync if chain pruning is configured. + if mode == ethconfig.SnapSync && cutoff != 0 { + chunkHeaders = chunkHeaders[cutoff:] + chunkHashes = chunkHashes[cutoff:] + } + if len(chunkHeaders) > 0 { + scheduled = true + if d.queue.Schedule(chunkHeaders, chunkHashes, origin+uint64(cutoff)) != len(chunkHeaders) { + return fmt.Errorf("%w: stale headers", errBadPeer) + } } - headers = headers[limit:] hashes = hashes[limit:] origin += uint64(limit) @@ -721,11 +767,13 @@ func (d *Downloader) processHeaders(origin uint64) error { } d.syncStatsLock.Unlock() - // Signal the content downloaders of the availability of new tasks - for _, ch := range []chan bool{d.queue.blockWakeCh, d.queue.receiptWakeCh} { - select { - case ch <- true: - default: + // Signal the downloader of the availability of new tasks + if scheduled { + for _, ch := range []chan bool{d.queue.blockWakeCh, d.queue.receiptWakeCh} { + select { + case ch <- true: + default: + } } } } @@ -1085,10 +1133,20 @@ func (d *Downloader) reportSnapSyncProgress(force bool) { header = d.blockchain.CurrentHeader() block = d.blockchain.CurrentSnapBlock() ) - syncedBlocks := block.Number.Uint64() - d.syncStartBlock - if syncedBlocks == 0 { + // Prevent reporting if nothing has been synchronized yet + if block.Number.Uint64() <= d.syncStartBlock { return } + // Prevent reporting noise if the actual chain synchronization (headers + // and bodies) hasn't started yet. Inserting the ancient header chain is + // fast enough and would introduce significant bias if included in the count. + if d.chainCutoffNumber != 0 && block.Number.Uint64() <= d.chainCutoffNumber { + return + } + fetchedBlocks := block.Number.Uint64() - d.syncStartBlock + if d.chainCutoffNumber != 0 && d.chainCutoffNumber > d.syncStartBlock { + fetchedBlocks = block.Number.Uint64() - d.chainCutoffNumber + } // Retrieve the current chain head and calculate the ETA latest, _, _, err := d.skeleton.Bounds() if err != nil { @@ -1103,7 +1161,7 @@ func (d *Downloader) reportSnapSyncProgress(force bool) { } var ( left = latest.Number.Uint64() - block.Number.Uint64() - eta = time.Since(d.syncStartTime) / time.Duration(syncedBlocks) * time.Duration(left) + eta = time.Since(d.syncStartTime) / time.Duration(fetchedBlocks) * time.Duration(left) progress = fmt.Sprintf("%.2f%%", float64(block.Number.Uint64())*100/float64(latest.Number.Uint64())) headers = fmt.Sprintf("%v@%v", log.FormatLogfmtUint64(header.Number.Uint64()), common.StorageSize(headerBytes).TerminalString()) diff --git a/eth/downloader/metrics.go b/eth/downloader/metrics.go index 23c033a8ad..bfe80ddbf1 100644 --- a/eth/downloader/metrics.go +++ b/eth/downloader/metrics.go @@ -25,7 +25,6 @@ import ( var ( headerInMeter = metrics.NewRegisteredMeter("eth/downloader/headers/in", nil) headerReqTimer = metrics.NewRegisteredTimer("eth/downloader/headers/req", nil) - headerDropMeter = metrics.NewRegisteredMeter("eth/downloader/headers/drop", nil) headerTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/headers/timeout", nil) bodyInMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/in", nil) diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index a1c114f057..000ad97ca9 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -73,7 +73,7 @@ type fetchResult struct { Withdrawals types.Withdrawals } -func newFetchResult(header *types.Header, fastSync bool) *fetchResult { +func newFetchResult(header *types.Header, snapSync bool) *fetchResult { item := &fetchResult{ Header: header, } @@ -82,7 +82,7 @@ func newFetchResult(header *types.Header, fastSync bool) *fetchResult { } else if header.WithdrawalsHash != nil { item.Withdrawals = make(types.Withdrawals, 0) } - if fastSync && !header.EmptyReceipts() { + if snapSync && !header.EmptyReceipts() { item.pending.Store(item.pending.Load() | (1 << receiptType)) } return item @@ -124,19 +124,8 @@ func (f *fetchResult) Done(kind uint) bool { // queue represents hashes that are either need fetching or are being fetched type queue struct { - mode SyncMode // Synchronisation mode to decide on the block parts to schedule for fetching - - // Headers are "special", they download in batches, supported by a skeleton chain - headerHead common.Hash // Hash of the last queued header to verify order - headerTaskPool map[uint64]*types.Header // Pending header retrieval tasks, mapping starting indexes to skeleton headers - headerTaskQueue *prque.Prque[int64, uint64] // Priority queue of the skeleton indexes to fetch the filling headers for - headerPeerMiss map[string]map[uint64]struct{} // Set of per-peer header batches known to be unavailable - headerPendPool map[string]*fetchRequest // Currently pending header retrieval operations - headerResults []*types.Header // Result cache accumulating the completed headers - headerHashes []common.Hash // Result cache accumulating the completed header hashes - headerProced int // Number of headers already processed from the results - headerOffset uint64 // Number of the first header in the result cache - headerContCh chan bool // Channel to notify when header download finishes + mode SyncMode // Synchronisation mode to decide on the block parts to schedule for fetching + headerHead common.Hash // Hash of the last queued header to verify order // All data retrievals below are based on an already assembles header chain blockTaskPool map[common.Hash]*types.Header // Pending block (body) retrieval tasks, mapping hashes to headers @@ -163,7 +152,6 @@ type queue struct { func newQueue(blockCacheLimit int, thresholdInitialSize int) *queue { lock := new(sync.RWMutex) q := &queue{ - headerContCh: make(chan bool, 1), blockTaskQueue: prque.New[int64, *types.Header](nil), blockWakeCh: make(chan bool, 1), receiptTaskQueue: prque.New[int64, *types.Header](nil), @@ -182,9 +170,7 @@ func (q *queue) Reset(blockCacheLimit int, thresholdInitialSize int) { q.closed = false q.mode = ethconfig.FullSync - q.headerHead = common.Hash{} - q.headerPendPool = make(map[string]*fetchRequest) q.blockTaskPool = make(map[common.Hash]*types.Header) q.blockTaskQueue.Reset() @@ -207,14 +193,6 @@ func (q *queue) Close() { q.lock.Unlock() } -// PendingHeaders retrieves the number of header requests pending for retrieval. -func (q *queue) PendingHeaders() int { - q.lock.Lock() - defer q.lock.Unlock() - - return q.headerTaskQueue.Size() -} - // PendingBodies retrieves the number of block body requests pending for retrieval. func (q *queue) PendingBodies() int { q.lock.Lock() @@ -260,54 +238,14 @@ func (q *queue) Idle() bool { return (queued + pending) == 0 } -// ScheduleSkeleton adds a batch of header retrieval tasks to the queue to fill -// up an already retrieved header skeleton. -func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) { - q.lock.Lock() - defer q.lock.Unlock() - - // No skeleton retrieval can be in progress, fail hard if so (huge implementation bug) - if q.headerResults != nil { - panic("skeleton assembly already in progress") - } - // Schedule all the header retrieval tasks for the skeleton assembly - q.headerTaskPool = make(map[uint64]*types.Header) - q.headerTaskQueue = prque.New[int64, uint64](nil) - q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains - q.headerResults = make([]*types.Header, len(skeleton)*MaxHeaderFetch) - q.headerHashes = make([]common.Hash, len(skeleton)*MaxHeaderFetch) - q.headerProced = 0 - q.headerOffset = from - q.headerContCh = make(chan bool, 1) - - for i, header := range skeleton { - index := from + uint64(i*MaxHeaderFetch) - - q.headerTaskPool[index] = header - q.headerTaskQueue.Push(index, -int64(index)) - } -} - -// RetrieveHeaders retrieves the header chain assemble based on the scheduled -// skeleton. -func (q *queue) RetrieveHeaders() ([]*types.Header, []common.Hash, int) { - q.lock.Lock() - defer q.lock.Unlock() - - headers, hashes, proced := q.headerResults, q.headerHashes, q.headerProced - q.headerResults, q.headerHashes, q.headerProced = nil, nil, 0 - - return headers, hashes, proced -} - // Schedule adds a set of headers for the download queue for scheduling, returning // the new headers encountered. -func (q *queue) Schedule(headers []*types.Header, hashes []common.Hash, from uint64) []*types.Header { +func (q *queue) Schedule(headers []*types.Header, hashes []common.Hash, from uint64) int { q.lock.Lock() defer q.lock.Unlock() // Insert all the headers prioritised by the contained block number - inserts := make([]*types.Header, 0, len(headers)) + var inserts int for i, header := range headers { // Make sure chain order is honoured and preserved throughout hash := hashes[i] @@ -337,7 +275,7 @@ func (q *queue) Schedule(headers []*types.Header, hashes []common.Hash, from uin q.receiptTaskQueue.Push(header, -int64(header.Number.Uint64())) } } - inserts = append(inserts, header) + inserts++ q.headerHead = hash from++ } @@ -390,7 +328,7 @@ func (q *queue) Results(block bool) []*fetchResult { q.resultSize = common.StorageSize(blockCacheSizeWeight)*size + (1-common.StorageSize(blockCacheSizeWeight))*q.resultSize } - // Using the newly calibrated resultsize, figure out the new throttle limit + // Using the newly calibrated result size, figure out the new throttle limit // on the result cache throttleThreshold := uint64((common.StorageSize(blockCacheMemory) + q.resultSize - 1) / q.resultSize) throttleThreshold = q.resultCache.SetThrottleThreshold(throttleThreshold) @@ -428,46 +366,6 @@ func (q *queue) stats() []interface{} { } } -// ReserveHeaders reserves a set of headers for the given peer, skipping any -// previously failed batches. -func (q *queue) ReserveHeaders(p *peerConnection, count int) *fetchRequest { - q.lock.Lock() - defer q.lock.Unlock() - - // Short circuit if the peer's already downloading something (sanity check to - // not corrupt state) - if _, ok := q.headerPendPool[p.id]; ok { - return nil - } - // Retrieve a batch of hashes, skipping previously failed ones - send, skip := uint64(0), []uint64{} - for send == 0 && !q.headerTaskQueue.Empty() { - from, _ := q.headerTaskQueue.Pop() - if q.headerPeerMiss[p.id] != nil { - if _, ok := q.headerPeerMiss[p.id][from]; ok { - skip = append(skip, from) - continue - } - } - send = from - } - // Merge all the skipped batches back - for _, from := range skip { - q.headerTaskQueue.Push(from, -int64(from)) - } - // Assemble and return the block download request - if send == 0 { - return nil - } - request := &fetchRequest{ - Peer: p, - From: send, - Time: time.Now(), - } - q.headerPendPool[p.id] = request - return request -} - // ReserveBodies reserves a set of body fetches for the given peer, skipping any // previously failed downloads. Beside the next batch of needed fetches, it also // returns a flag whether empty blocks were queued requiring processing. @@ -594,10 +492,6 @@ func (q *queue) Revoke(peerID string) { q.lock.Lock() defer q.lock.Unlock() - if request, ok := q.headerPendPool[peerID]; ok { - q.headerTaskQueue.Push(request.From, -int64(request.From)) - delete(q.headerPendPool, peerID) - } if request, ok := q.blockPendPool[peerID]; ok { for _, header := range request.Headers { q.blockTaskQueue.Push(header, -int64(header.Number.Uint64())) @@ -612,16 +506,6 @@ func (q *queue) Revoke(peerID string) { } } -// ExpireHeaders cancels a request that timed out and moves the pending fetch -// task back into the queue for rescheduling. -func (q *queue) ExpireHeaders(peer string) int { - q.lock.Lock() - defer q.lock.Unlock() - - headerTimeoutMeter.Mark(1) - return q.expire(peer, q.headerPendPool, q.headerTaskQueue) -} - // ExpireBodies checks for in flight block body requests that exceeded a timeout // allowance, canceling them and returning the responsible peers for penalisation. func (q *queue) ExpireBodies(peer string) int { @@ -670,116 +554,6 @@ func (q *queue) expire(peer string, pendPool map[string]*fetchRequest, taskQueue return len(req.Headers) } -// DeliverHeaders injects a header retrieval response into the header results -// cache. This method either accepts all headers it received, or none of them -// if they do not map correctly to the skeleton. -// -// If the headers are accepted, the method makes an attempt to deliver the set -// of ready headers to the processor to keep the pipeline full. However, it will -// not block to prevent stalling other pending deliveries. -func (q *queue) DeliverHeaders(id string, headers []*types.Header, hashes []common.Hash, headerProcCh chan *headerTask) (int, error) { - q.lock.Lock() - defer q.lock.Unlock() - - var logger log.Logger - if len(id) < 16 { - // Tests use short IDs, don't choke on them - logger = log.New("peer", id) - } else { - logger = log.New("peer", id[:16]) - } - // Short circuit if the data was never requested - request := q.headerPendPool[id] - if request == nil { - headerDropMeter.Mark(int64(len(headers))) - return 0, errNoFetchesPending - } - delete(q.headerPendPool, id) - - headerReqTimer.UpdateSince(request.Time) - headerInMeter.Mark(int64(len(headers))) - - // Ensure headers can be mapped onto the skeleton chain - target := q.headerTaskPool[request.From].Hash() - - accepted := len(headers) == MaxHeaderFetch - if accepted { - if headers[0].Number.Uint64() != request.From { - logger.Trace("First header broke chain ordering", "number", headers[0].Number, "hash", hashes[0], "expected", request.From) - accepted = false - } else if hashes[len(headers)-1] != target { - logger.Trace("Last header broke skeleton structure ", "number", headers[len(headers)-1].Number, "hash", hashes[len(headers)-1], "expected", target) - accepted = false - } - } - if accepted { - parentHash := hashes[0] - for i, header := range headers[1:] { - hash := hashes[i+1] - if want := request.From + 1 + uint64(i); header.Number.Uint64() != want { - logger.Warn("Header broke chain ordering", "number", header.Number, "hash", hash, "expected", want) - accepted = false - break - } - if parentHash != header.ParentHash { - logger.Warn("Header broke chain ancestry", "number", header.Number, "hash", hash) - accepted = false - break - } - // Set-up parent hash for next round - parentHash = hash - } - } - // If the batch of headers wasn't accepted, mark as unavailable - if !accepted { - logger.Trace("Skeleton filling not accepted", "from", request.From) - headerDropMeter.Mark(int64(len(headers))) - - miss := q.headerPeerMiss[id] - if miss == nil { - q.headerPeerMiss[id] = make(map[uint64]struct{}) - miss = q.headerPeerMiss[id] - } - miss[request.From] = struct{}{} - - q.headerTaskQueue.Push(request.From, -int64(request.From)) - return 0, errors.New("delivery not accepted") - } - // Clean up a successful fetch and try to deliver any sub-results - copy(q.headerResults[request.From-q.headerOffset:], headers) - copy(q.headerHashes[request.From-q.headerOffset:], hashes) - - delete(q.headerTaskPool, request.From) - - ready := 0 - for q.headerProced+ready < len(q.headerResults) && q.headerResults[q.headerProced+ready] != nil { - ready += MaxHeaderFetch - } - if ready > 0 { - // Headers are ready for delivery, gather them and push forward (non blocking) - processHeaders := make([]*types.Header, ready) - copy(processHeaders, q.headerResults[q.headerProced:q.headerProced+ready]) - - processHashes := make([]common.Hash, ready) - copy(processHashes, q.headerHashes[q.headerProced:q.headerProced+ready]) - - select { - case headerProcCh <- &headerTask{ - headers: processHeaders, - hashes: processHashes, - }: - logger.Trace("Pre-scheduled new headers", "count", len(processHeaders), "from", processHeaders[0].Number) - q.headerProced += len(processHeaders) - default: - } - } - // Check for termination and return - if len(q.headerTaskPool) == 0 { - q.headerContCh <- false - } - return len(headers), nil -} - // DeliverBodies injects a block body retrieval response into the results queue. // The method returns the number of blocks bodies accepted from the delivery and // also wakes any threads waiting for data delivery. diff --git a/eth/downloader/resultstore.go b/eth/downloader/resultstore.go index e4323c04eb..36c382fefc 100644 --- a/eth/downloader/resultstore.go +++ b/eth/downloader/resultstore.go @@ -76,7 +76,7 @@ func (r *resultStore) SetThrottleThreshold(threshold uint64) uint64 { // throttled - if true, the store is at capacity, this particular header is not prio now // item - the result to store data into // err - any error that occurred -func (r *resultStore) AddFetch(header *types.Header, fastSync bool) (stale, throttled bool, item *fetchResult, err error) { +func (r *resultStore) AddFetch(header *types.Header, snapSync bool) (stale, throttled bool, item *fetchResult, err error) { r.lock.Lock() defer r.lock.Unlock() @@ -86,7 +86,7 @@ func (r *resultStore) AddFetch(header *types.Header, fastSync bool) (stale, thro return stale, throttled, item, err } if item == nil { - item = newFetchResult(header, fastSync) + item = newFetchResult(header, snapSync) r.items[index] = item } return stale, throttled, item, err From db9be56bab751e3d75fcc29e282ff7e51bd95b8b Mon Sep 17 00:00:00 2001 From: Mobin Mohanan <47410557+tr1sm0s1n@users.noreply.github.com> Date: Thu, 3 Apr 2025 18:48:35 +0530 Subject: [PATCH 15/35] build: upgrade to golangci-lint v2 (#31530) --- .golangci.yml | 131 +++++++++++++++++++++++++------------------- build/checksums.txt | 106 +++++++++++++++++------------------ 2 files changed, 127 insertions(+), 110 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index a702da524b..5ed06537bd 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,31 +1,25 @@ # This file configures github.com/golangci/golangci-lint. - +version: '2' run: - timeout: 20m tests: true - linters: - disable-all: true + default: none enable: - - goimports - - gosimple - - govet - - ineffassign - - misspell - - unconvert - - typecheck - - unused - - staticcheck - bidichk - - durationcheck - copyloopvar - - whitespace - - revive # only certain checks enabled - durationcheck - gocheckcompilerdirectives - - reassign + - govet + - ineffassign - mirror + - misspell + - reassign + - revive # only certain checks enabled + - staticcheck + - unconvert + - unused - usetesting + - whitespace ### linters we tried and will not be using: ### # - structcheck # lots of false positives @@ -36,44 +30,67 @@ linters: # - exhaustive # silly check # - makezero # false positives # - nilerr # several intentional - -linters-settings: - gofmt: - simplify: true - revive: - enable-all-rules: false - # here we enable specific useful rules - # see https://golangci-lint.run/usage/linters/#revive for supported rules + settings: + staticcheck: + checks: + # disable Quickfixes + - -QF1* + revive: + enable-all-rules: false + # here we enable specific useful rules + # see https://golangci-lint.run/usage/linters/#revive for supported rules + rules: + - name: receiver-naming + severity: warning + disabled: false + exclude: + - '' + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling rules: - - name: receiver-naming - severity: warning - disabled: false - exclude: [""] - -issues: - # default is true. Enables skipping of directories: - # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ - exclude-dirs-use-default: true - exclude-files: - - core/genesis_alloc.go - exclude-rules: - - path: crypto/bn256/cloudflare/optate.go - linters: - - deadcode - - staticcheck - - path: crypto/bn256/ - linters: - - revive - - path: cmd/utils/flags.go - text: "SA1019: cfg.TxLookupLimit is deprecated: use 'TransactionHistory' instead." - - path: cmd/utils/flags.go - text: "SA1019: ethconfig.Defaults.TxLookupLimit is deprecated: use 'TransactionHistory' instead." - - path: internal/build/pgp.go - text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.' - - path: core/vm/contracts.go - text: 'SA1019: "golang.org/x/crypto/ripemd160" is deprecated: RIPEMD-160 is a legacy hash and should not be used for new applications.' - exclude: - - 'SA1019: event.TypeMux is deprecated: use Feed' - - 'SA1019: strings.Title is deprecated' - - 'SA1019: strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead.' - - 'SA1029: should not use built-in type string as key for value' + - linters: + - deadcode + - staticcheck + path: crypto/bn256/cloudflare/optate.go + - linters: + - revive + path: crypto/bn256/ + - path: cmd/utils/flags.go + text: "SA1019: cfg.TxLookupLimit is deprecated: use 'TransactionHistory' instead." + - path: cmd/utils/flags.go + text: "SA1019: ethconfig.Defaults.TxLookupLimit is deprecated: use 'TransactionHistory' instead." + - path: internal/build/pgp.go + text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.' + - path: core/vm/contracts.go + text: 'SA1019: "golang.org/x/crypto/ripemd160" is deprecated: RIPEMD-160 is a legacy hash and should not be used for new applications.' + - path: (.+)\.go$ + text: 'SA1019: event.TypeMux is deprecated: use Feed' + - path: (.+)\.go$ + text: 'SA1019: strings.Title is deprecated' + - path: (.+)\.go$ + text: 'SA1019: strings.Title has been deprecated since Go 1.18 and an alternative has been available since Go 1.0: The rule Title uses for word boundaries does not handle Unicode punctuation properly. Use golang.org/x/text/cases instead.' + - path: (.+)\.go$ + text: 'SA1029: should not use built-in type string as key for value' + paths: + - core/genesis_alloc.go + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - goimports + settings: + gofmt: + simplify: true + exclusions: + generated: lax + paths: + - core/genesis_alloc.go + - third_party$ + - builtin$ + - examples$ diff --git a/build/checksums.txt b/build/checksums.txt index 5ccb85bba6..6d3b718c3c 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -54,60 +54,60 @@ db128981033ac82a64688a123f631e61297b6b8f52ca913145e57caa8ce94cc3 go1.24.1.windo e28c4e6d0b913955765b46157ab88ae59bb636acaa12d7bec959aa6900f1cebd go1.24.1.windows-arm64.zip 6d352c1f154a102a5b90c480cc64bab205ccf2681e34e78a3a4d3f1ddfbc81e4 go1.24.1.windows-arm64.msi -# version:golangci 1.64.6 +# version:golangci 2.0.2 # https://github.com/golangci/golangci-lint/releases/ -# https://github.com/golangci/golangci-lint/releases/download/v1.64.6/ -08f9459e7125fed2612abd71596e04d172695921aff82120d1c1e5e9b6fff2a3 golangci-lint-1.64.6-darwin-amd64.tar.gz -8c10d0c7c3935b8c2269d628b6a06a8f48d8fb4cc31af02fe4ce0cd18b0704c1 golangci-lint-1.64.6-darwin-arm64.tar.gz -c07fcabb55deb8d2c4d390025568e76162d7f91b1d14bd2311be45d8d440a624 golangci-lint-1.64.6-freebsd-386.tar.gz -8ed1ef1102e1a42983ffcaae8e06de6a540334c3a69e205c610b8a7c92d469cd golangci-lint-1.64.6-freebsd-amd64.tar.gz -8f8dda66d1b4a85cc8a1daf1b69af6559c3eb0a41dd8033148a9ad85cfc0e1d9 golangci-lint-1.64.6-freebsd-armv6.tar.gz -59e8ca1fa254661b2a55e96515b14a10cd02221d443054deac5b28c3c3e12d6b golangci-lint-1.64.6-freebsd-armv7.tar.gz -e3d323d5f132e9b7593141bfe1d19c8460e65a55cea1008ec96945fed563f981 golangci-lint-1.64.6-illumos-amd64.tar.gz -5ce910f2a864c5fbeb32a30cbd506e1b2ef215f7a0f422cd5be6370b13db6f03 golangci-lint-1.64.6-linux-386.deb -2caab682a26b9a5fb94ba24e3a7e1404fa9eab2c12e36ae1c5548d80a1be190c golangci-lint-1.64.6-linux-386.rpm -2d82d0a4716e6d9b70c95103054855cb4b5f20f7bbdee42297f0189955bd14b6 golangci-lint-1.64.6-linux-386.tar.gz -9cd82503e9841abcaa57663fc899587fe90363c26d86a793a98d3080fd25e907 golangci-lint-1.64.6-linux-amd64.deb -981aaca5e5202d4fbb162ec7080490eb67ef5d32add5fb62fb02210597acc9da golangci-lint-1.64.6-linux-amd64.rpm -71e290acbacb7b3ba4f68f0540fb74dc180c4336ac8a6f3bdcd7fcc48b15148d golangci-lint-1.64.6-linux-amd64.tar.gz -718016bb06c1f598a8d23c7964e2643de621dbe5808688cb38857eb0bb773c84 golangci-lint-1.64.6-linux-arm64.deb -ddc0e7b4b10b47cf894aea157e89e3674bbb60f8f5c480110c21c49cc2c1634d golangci-lint-1.64.6-linux-arm64.rpm -99a7ff13dec7a8781a68408b6ecb8a1c5e82786cba3189eaa91d5cdcc24362ce golangci-lint-1.64.6-linux-arm64.tar.gz -90e360f89c244394912b8709fb83a900b6d56cf19141df2afaf9e902ef3057b1 golangci-lint-1.64.6-linux-armv6.deb -46546ff7c98d873ffcdbee06b39dc1024fc08db4fbf1f6309360a44cf976b5c2 golangci-lint-1.64.6-linux-armv6.rpm -e45c1a42868aca0b0ee54d14fb89da216f3b4409367cd7bce3b5f36788b4fc92 golangci-lint-1.64.6-linux-armv6.tar.gz -3cf6ddbbbf358db3de4b64a24f9683bbe2da3f003cfdee86cb610124b57abafb golangci-lint-1.64.6-linux-armv7.deb -508b6712710da10f11aab9ea5e63af415c932dfe7fff718e649d3100b838f797 golangci-lint-1.64.6-linux-armv7.rpm -da9a8bbee86b4dfee73fbc17e0856ec84c5b04919eb09bf3dd5904c39ce41753 golangci-lint-1.64.6-linux-armv7.tar.gz -ad496a58284e1e9c8ac6f993fec429dcd96c85a8c4715dbb6530a5af0dae7fbd golangci-lint-1.64.6-linux-loong64.deb -3bd70fa737b224750254dce616d9a188570e49b894b0cdb2fd04155e2c061350 golangci-lint-1.64.6-linux-loong64.rpm -a535af973499729f2215a84303eb0de61399057aad6901ddea1b4f73f68d5f2c golangci-lint-1.64.6-linux-loong64.tar.gz -6ad2a1cd37ca30303a488abfca2de52aff57519901c6d8d2608656fe9fb05292 golangci-lint-1.64.6-linux-mips64.deb -5f18369f0ca010a02c267352ebe6e3e0514c6debff49899c9e5520906c0da287 golangci-lint-1.64.6-linux-mips64.rpm -3449d6c13307b91b0e8817f8911c07c3412cdb6931b8d101e42db3e9762e91ad golangci-lint-1.64.6-linux-mips64.tar.gz -d4712a348f65dcaf9f6c58f1cfd6d0984d54a902873dca5e76f0d686f5c59499 golangci-lint-1.64.6-linux-mips64le.deb -57cea4538894558cb8c956d6b69c5509e4304546abe4a467478fc9ada0c736d6 golangci-lint-1.64.6-linux-mips64le.rpm -bc030977d44535b2152fddb2732f056d193c043992fe638ddecea21a40ef09fe golangci-lint-1.64.6-linux-mips64le.tar.gz -1ceb4e492efa527d246c61798c581f49113756fab8c39bb3eefdb1fa97041f92 golangci-lint-1.64.6-linux-ppc64le.deb -454e1c2b3583a8644e0c33a970fb4ce35b8f11848e1a770d9095d99d159ad0ab golangci-lint-1.64.6-linux-ppc64le.rpm -fddf30d35923eb6c7bb57520d8645768f802bf86c4cbf5a3a13049efb9e71b82 golangci-lint-1.64.6-linux-ppc64le.tar.gz -bd75f0dd6f65bee5821c433803b28f3c427ef3582764c3d0e7f7fae1c9d468b6 golangci-lint-1.64.6-linux-riscv64.deb -58457207c225cbd5340c8dcb75d9a44aa890d604e28464115a3a9762febaed04 golangci-lint-1.64.6-linux-riscv64.rpm -82639518a613a6705b7e2de5b28c278e875d782a5c97e9c1b2ac10b4ecd7852f golangci-lint-1.64.6-linux-riscv64.tar.gz -131d69204f8ced200b1437731e987cc886edac30dc87e6e1dcbd4f833d351713 golangci-lint-1.64.6-linux-s390x.deb -ca6caf28ca7a1df7cff720f8fd6ac4b8f2eac10c8cbfe7d2eade70876aded570 golangci-lint-1.64.6-linux-s390x.rpm -ed966d28a6512bc2b1120029a9f78ed77f2015e330b589a731d67d59be30b0b1 golangci-lint-1.64.6-linux-s390x.tar.gz -b181132b41943fc77ace7f9f5523085d12d3613f27774a0e35e17dd5e65ba589 golangci-lint-1.64.6-netbsd-386.tar.gz -f7b81c426006e3c699dc8665797a462aecba8c2cd23ac4971472e55184d95bc9 golangci-lint-1.64.6-netbsd-amd64.tar.gz -663d6fb4c3bef57b8aacdb1b1a4634075e55d294ed170724b443374860897ca6 golangci-lint-1.64.6-netbsd-arm64.tar.gz -8c7a76ee822568cc19352bbb9b2b0863dc5e185eb07782adbbaf648afd02ebcd golangci-lint-1.64.6-netbsd-armv6.tar.gz -0ce26d56ce78e516529e087118ba3f1bcd71d311b4c5b2bde6ec24a6e8966d85 golangci-lint-1.64.6-netbsd-armv7.tar.gz -135d5d998168683f46e4fab308cef5aa3c282025b7de6b258f976aadfb820db7 golangci-lint-1.64.6-source.tar.gz -ccdb0cc249531a205304a76308cfa7cda830083d083d557884416a2461148263 golangci-lint-1.64.6-windows-386.zip -2d88f282e08e1853c70bc7c914b5f58beaa6509903d56098aeb9bc3df30ea9d5 golangci-lint-1.64.6-windows-amd64.zip -6a3c6e7afc6916392679d7d6b95ac239827644e3e50ec4e8ca6ab1e4e6db65fe golangci-lint-1.64.6-windows-arm64.zip -9c9c1ef9687651566987f93e76252f7526c386901d7d8aeee044ca88115da4b1 golangci-lint-1.64.6-windows-armv6.zip -4f0df114155791799dfde8cd8cb6307fecfb17fed70b44205486ec925e2f39cf golangci-lint-1.64.6-windows-armv7.zip +# https://github.com/golangci/golangci-lint/releases/download/v2.0.2/ +a88cbdc86b483fe44e90bf2dcc3fec2af8c754116e6edf0aa6592cac5baa7a0e golangci-lint-2.0.2-darwin-amd64.tar.gz +664550e7954f5f4451aae99b4f7382c1a47039c66f39ca605f5d9af1a0d32b49 golangci-lint-2.0.2-darwin-arm64.tar.gz +bda0f0f27d300502faceda8428834a76ca25986f6d9fc2bd41d201c3ed73f08e golangci-lint-2.0.2-freebsd-386.tar.gz +1cbd0c7ade3fb027d61d38a646ec1b51be5846952b4b04a5330e7f4687f2270c golangci-lint-2.0.2-freebsd-amd64.tar.gz +1e828a597726198b2e35acdbcc5f3aad85244d79846d2d2bdb05241c5a535f9e golangci-lint-2.0.2-freebsd-armv6.tar.gz +848b49315dc5cddd0c9ce35e96ab33d584db0ea8fb57bcbf9784f1622bec0269 golangci-lint-2.0.2-freebsd-armv7.tar.gz +cabf9a6beab574c7f98581eb237919e580024759e3cdc05c4d516b044dce6770 golangci-lint-2.0.2-illumos-amd64.tar.gz +2fde80d15ed6527791f106d606120620e913c3a663c90a8596861d0a4461169e golangci-lint-2.0.2-linux-386.deb +804bc6e350a8c613aaa0a33d8d45414a80157b0ba1b2c2335ac859f85ad98ebd golangci-lint-2.0.2-linux-386.rpm +e64beb72fecf581e57d88ae3adb1c9d4bf022694b6bd92e3c8e460910bbdc37d golangci-lint-2.0.2-linux-386.tar.gz +9c55aed174d7a52bb1d4006b36e7edee9023631f6b814a80cb39c9860d6f75c3 golangci-lint-2.0.2-linux-amd64.deb +c55a2ef741a687b4c679696931f7fd4a467babd64c9457cf17bb9632fd1cecd1 golangci-lint-2.0.2-linux-amd64.rpm +89cc8a7810dc63b9a37900da03e37c3601caf46d42265d774e0f1a5d883d53e2 golangci-lint-2.0.2-linux-amd64.tar.gz +a3e78583c4e7ea1b63e82559f126bb3a5b12788676f158526752d53e67824b99 golangci-lint-2.0.2-linux-arm64.deb +bd5dd52b5c9f18aa7a2904eda9a9f91c628e98623fe70b7afcbb847e2de84422 golangci-lint-2.0.2-linux-arm64.rpm +789d5b91219ac68c2336f77d41cd7e33a910420594780f455893f8453d09595b golangci-lint-2.0.2-linux-arm64.tar.gz +534cd4c464a66178714ed68152c1ed7aa73e5700bf409e4ed1a8363adf96afca golangci-lint-2.0.2-linux-armv6.deb +cf7d02905a5fc80b96c9a64621693b4cc7337b1ce29986c19fd72608dafe66c5 golangci-lint-2.0.2-linux-armv6.rpm +a0d81cb527d8fe878377f2356b5773e219b0b91832a6b59e7b9bcf9a90fe0b0e golangci-lint-2.0.2-linux-armv6.tar.gz +dedd5be7fff8cba8fe15b658a59347ea90d7d02a9fff87f09c7687e6da05a8b6 golangci-lint-2.0.2-linux-armv7.deb +85521b6f3ad2f5a2bc9bfe14b9b08623f764964048f75ed6dfcfaf8eb7d57cc1 golangci-lint-2.0.2-linux-armv7.rpm +96471046c7780dda4ea680f65e92c2ef56ff58d40bcffaf6cfe9d6d48e3c27aa golangci-lint-2.0.2-linux-armv7.tar.gz +815d914a7738e4362466b2d11004e8618b696b49e8ace13df2c2b25f28fb1e17 golangci-lint-2.0.2-linux-loong64.deb +f16381e3d8a0f011b95e086d83d620248432b915d01f4beab4d29cfe4dc388b0 golangci-lint-2.0.2-linux-loong64.rpm +1bd8d7714f9c92db6a0f23bae89f39c85ba047bec8eeb42b8748d30ae3228d18 golangci-lint-2.0.2-linux-loong64.tar.gz +ea6e9d4aabb526aa298e47e8b026d8893d918c5eb919ba0ab403e315def74cc5 golangci-lint-2.0.2-linux-mips64.deb +519d8d53af83fdc9c25cc3fba8b663331ac22ef68131d4b0084cb6f425b6f79a golangci-lint-2.0.2-linux-mips64.rpm +80d655a0a1ac1b19dcef4b58fa2a7dadb646cc50ad08d460b5c53cdb421165e4 golangci-lint-2.0.2-linux-mips64.tar.gz +aa0e75384bb482c865d4dfc95d23ceb25666bf20461b67a832f0eea6670312ec golangci-lint-2.0.2-linux-mips64le.deb +f2a8b500fb69bdea1b01df6267aaa5218fa4a58aeb781c1a20d0d802fe465a52 golangci-lint-2.0.2-linux-mips64le.rpm +e66a0c0c9a275f02d27a7caa9576112622306f001d73dfc082cf1ae446fc1242 golangci-lint-2.0.2-linux-mips64le.tar.gz +e85ad51aac6428be2d8a37000d053697371a538a5bcbc1644caa7c5e77f6d0af golangci-lint-2.0.2-linux-ppc64le.deb +906798365eac1944af2a9b9a303e6fd49ec9043307bc681b7a96277f7f8beea5 golangci-lint-2.0.2-linux-ppc64le.rpm +f7f1a271b0af274d6c9ce000f5dc6e1fb194350c67bcc62494f96f791882ba92 golangci-lint-2.0.2-linux-ppc64le.tar.gz +eea8bf643a42bf05de9780530db22923e5ab0d588f0e173594dc6518f2a25d2a golangci-lint-2.0.2-linux-riscv64.deb +4ff40f9fe2954400836e2a011ba4744d00ffab5068a51368552dfce6aba3b81b golangci-lint-2.0.2-linux-riscv64.rpm +531d8f225866674977d630afbf0533eb02f9bec607fb13895f7a2cd7b2e0a648 golangci-lint-2.0.2-linux-riscv64.tar.gz +6f827647046c603f40d97ea5aadc6f48cd0bb5d19f7a3d56500c3b833d2a0342 golangci-lint-2.0.2-linux-s390x.deb +387a090e9576d19ca86aac738172e58e07c19f2784a13bb387f4f0d75fb9c8d3 golangci-lint-2.0.2-linux-s390x.rpm +57de1fb7722a9feb2d11ed0a007a93959d05b9db5929a392abc222e30012467e golangci-lint-2.0.2-linux-s390x.tar.gz +ed95e0492ea86bf79eb661f0334474b2a4255093685ff587eccd797c5a54db7e golangci-lint-2.0.2-netbsd-386.tar.gz +eab81d729778166415d349a80e568b2f2b3a781745a9be3212a92abb1e732daf golangci-lint-2.0.2-netbsd-amd64.tar.gz +d20add73f7c2de2c3b01ed4fd7b63ffcf0a6597d5ea228d1699e92339a3cd047 golangci-lint-2.0.2-netbsd-arm64.tar.gz +4e4f44e6057879cd62424ff1800a767d25a595c0e91d6d48809eea9186b4c739 golangci-lint-2.0.2-netbsd-armv6.tar.gz +51ec17b16d8743ae4098a0171f04f0ed4d64561e3051b982778b0e6c306a1b03 golangci-lint-2.0.2-netbsd-armv7.tar.gz +5482cf27b93fae1765c70ee2a95d4074d038e9dee61bdd61d017ce8893d3a4a8 golangci-lint-2.0.2-source.tar.gz +a35d8fdf3e14079a10880dbbb7586b46faec89be96f086b244b3e565aac80313 golangci-lint-2.0.2-windows-386.zip +fe4b946cc01366b989001215687003a9c4a7098589921f75e6228d6d8cffc15c golangci-lint-2.0.2-windows-amd64.zip +646bd9250ef8c771d85cd22fe8e6f2397ae39599179755e3bbfa9ef97ad44090 golangci-lint-2.0.2-windows-arm64.zip +ce1dc0bad6f8a61d64e6b3779eeb773479c175125d6f686b0e67ef9c8432d16e golangci-lint-2.0.2-windows-armv6.zip +92684a48faabe792b11ac27ca8b25551eff940b0a1e84ad7244e98b4994962db golangci-lint-2.0.2-windows-armv7.zip # This is the builder on PPA that will build Go itself (inception-y), don't modify! # From a0620f114c469503985376e7e4bdc28547d4ea5a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 3 Apr 2025 15:44:07 +0200 Subject: [PATCH 16/35] eth: fix calls to HistoryPruningCutoff (#31552) These were caused by crossed merges of recent PRs #31414 and #31361 --- eth/api_backend.go | 15 ++++++++------- internal/ethapi/api_test.go | 5 ++++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/eth/api_backend.go b/eth/api_backend.go index 944c357e78..b39dd4cbdb 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -94,7 +94,7 @@ func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumb } var bn uint64 if number == rpc.EarliestBlockNumber { - bn = b.eth.blockchain.HistoryPruningCutoff() + bn = b.HistoryPruningCutoff() } else { bn = uint64(number) } @@ -152,10 +152,10 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe } bn := uint64(number) // the resolved number if number == rpc.EarliestBlockNumber { - bn = b.eth.blockchain.HistoryPruningCutoff() + bn = b.HistoryPruningCutoff() } block := b.eth.blockchain.GetBlockByNumber(bn) - if block == nil && bn < b.eth.blockchain.HistoryPruningCutoff() { + if block == nil && bn < b.HistoryPruningCutoff() { return nil, ðconfig.PrunedHistoryError{} } return block, nil @@ -167,7 +167,7 @@ func (b *EthAPIBackend) BlockByHash(ctx context.Context, hash common.Hash) (*typ return nil, nil } block := b.eth.blockchain.GetBlock(hash, *number) - if block == nil && *number < b.eth.blockchain.HistoryPruningCutoff() { + if block == nil && *number < b.HistoryPruningCutoff() { return nil, ðconfig.PrunedHistoryError{} } return block, nil @@ -180,7 +180,7 @@ func (b *EthAPIBackend) GetBody(ctx context.Context, hash common.Hash, number rp } body := b.eth.blockchain.GetBody(hash) if body == nil { - if uint64(number) < b.eth.blockchain.HistoryPruningCutoff() { + if uint64(number) < b.HistoryPruningCutoff() { return nil, ðconfig.PrunedHistoryError{} } return nil, errors.New("block body not found") @@ -202,7 +202,7 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r } block := b.eth.blockchain.GetBlock(hash, header.Number.Uint64()) if block == nil { - if header.Number.Uint64() < b.eth.blockchain.HistoryPruningCutoff() { + if header.Number.Uint64() < b.HistoryPruningCutoff() { return nil, ðconfig.PrunedHistoryError{} } return nil, errors.New("header found, but block body is missing") @@ -265,7 +265,8 @@ func (b *EthAPIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockN } func (b *EthAPIBackend) HistoryPruningCutoff() uint64 { - return b.eth.blockchain.HistoryPruningCutoff() + bn, _ := b.eth.blockchain.HistoryPruningCutoff() + return bn } func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 37210aa78b..3fbf32e22e 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -623,7 +623,10 @@ func (b testBackend) NewMatcherBackend() filtermaps.MatcherBackend { panic("implement me") } -func (b testBackend) HistoryPruningCutoff() uint64 { return b.chain.HistoryPruningCutoff() } +func (b testBackend) HistoryPruningCutoff() uint64 { + bn, _ := b.chain.HistoryPruningCutoff() + return bn +} func TestEstimateGas(t *testing.T) { t.Parallel() From 49f0d49e89af34eef7ea4d712b6936c4e7440311 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 3 Apr 2025 15:58:37 +0200 Subject: [PATCH 17/35] cmd/devp2p/internal/v5test: log test descriptions (#31551) This adds the test description text to the output, instead of keeping it as a Go comment. Logs are visible in the hive UI where these tests run, while Go comments are not. --- cmd/devp2p/internal/v5test/discv5tests.go | 35 ++++++++++++++++------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/cmd/devp2p/internal/v5test/discv5tests.go b/cmd/devp2p/internal/v5test/discv5tests.go index 7dbd3c3be5..2139cd8ca6 100644 --- a/cmd/devp2p/internal/v5test/discv5tests.go +++ b/cmd/devp2p/internal/v5test/discv5tests.go @@ -59,8 +59,9 @@ func (s *Suite) AllTests() []utesting.Test { } } -// TestPing sends PING and expects a PONG response. func (s *Suite) TestPing(t *utesting.T) { + t.Log(`This test is just a sanity check. It sends PING and expects a PONG response.`) + conn, l1 := s.listen1(t) defer conn.close() @@ -85,9 +86,10 @@ func checkPong(t *utesting.T, pong *v5wire.Pong, ping *v5wire.Ping, c net.Packet } } -// TestPingLargeRequestID sends PING with a 9-byte request ID, which isn't allowed by the spec. -// The remote node should not respond. func (s *Suite) TestPingLargeRequestID(t *utesting.T) { + t.Log(`This test sends PING with a 9-byte request ID, which isn't allowed by the spec. +The remote node should not respond.`) + conn, l1 := s.listen1(t) defer conn.close() @@ -104,10 +106,11 @@ func (s *Suite) TestPingLargeRequestID(t *utesting.T) { } } -// TestPingMultiIP establishes a session from one IP as usual. The session is then reused -// on another IP, which shouldn't work. The remote node should respond with WHOAREYOU for -// the attempt from a different IP. func (s *Suite) TestPingMultiIP(t *utesting.T) { + t.Log(`This test establishes a session from one IP as usual. The session is then reused +on another IP, which shouldn't work. The remote node should respond with WHOAREYOU for +the attempt from a different IP.`) + conn, l1, l2 := s.listen2(t) defer conn.close() @@ -120,6 +123,7 @@ func (s *Suite) TestPingMultiIP(t *utesting.T) { checkPong(t, resp.(*v5wire.Pong), ping, l1) // Send on l2. This reuses the session because there is only one codec. + t.Log("sending ping from alternate IP", l2.LocalAddr()) ping2 := &v5wire.Ping{ReqID: conn.nextReqID()} conn.write(l2, ping2, nil) switch resp := conn.read(l2).(type) { @@ -158,6 +162,10 @@ func (s *Suite) TestPingMultiIP(t *utesting.T) { // packet instead of a handshake message packet. The remote node should respond with // another WHOAREYOU challenge for the second packet. func (s *Suite) TestPingHandshakeInterrupted(t *utesting.T) { + t.Log(`TestPingHandshakeInterrupted starts a handshake, but doesn't finish it and sends a second ordinary message +packet instead of a handshake message packet. The remote node should respond with +another WHOAREYOU challenge for the second packet.`) + conn, l1 := s.listen1(t) defer conn.close() @@ -181,8 +189,10 @@ func (s *Suite) TestPingHandshakeInterrupted(t *utesting.T) { } } -// TestTalkRequest sends TALKREQ and expects an empty TALKRESP response. func (s *Suite) TestTalkRequest(t *utesting.T) { + t.Log(`This test sends some examples of TALKREQ with a protocol-id of "test-protocol" +and expects an empty TALKRESP response.`) + conn, l1 := s.listen1(t) defer conn.close() @@ -202,6 +212,7 @@ func (s *Suite) TestTalkRequest(t *utesting.T) { } // Empty request ID. + t.Log("sending TALKREQ with empty request-id") resp = conn.reqresp(l1, &v5wire.TalkRequest{Protocol: "test-protocol"}) switch resp := resp.(type) { case *v5wire.TalkResponse: @@ -216,8 +227,9 @@ func (s *Suite) TestTalkRequest(t *utesting.T) { } } -// TestFindnodeZeroDistance checks that the remote node returns itself for FINDNODE with distance zero. func (s *Suite) TestFindnodeZeroDistance(t *utesting.T) { + t.Log(`This test checks that the remote node returns itself for FINDNODE with distance zero.`) + conn, l1 := s.listen1(t) defer conn.close() @@ -233,9 +245,11 @@ func (s *Suite) TestFindnodeZeroDistance(t *utesting.T) { } } -// TestFindnodeResults pings the node under test from multiple nodes. After waiting for them to be -// accepted into the remote table, the test checks that they are returned by FINDNODE. func (s *Suite) TestFindnodeResults(t *utesting.T) { + t.Log(`This test pings the node under test from multiple other endpoints and node identities +(the 'bystanders'). After waiting for them to be accepted into the remote table, the test checks +that they are returned by FINDNODE.`) + // Create bystanders. nodes := make([]*bystander, 5) added := make(chan enode.ID, len(nodes)) @@ -273,6 +287,7 @@ func (s *Suite) TestFindnodeResults(t *utesting.T) { } // Send FINDNODE for all distances. + t.Log("requesting nodes") conn, l1 := s.listen1(t) defer conn.close() foundNodes, err := conn.findnode(l1, dists) From 553183e5de5dd6aa73038f3a24f90fb65e141402 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 3 Apr 2025 22:03:03 +0800 Subject: [PATCH 18/35] core, eth, node: use sync write option in pebble (#31519) Fixes #31499 --- core/bench_test.go | 8 ++++---- core/blockchain_repair_test.go | 8 ++++---- core/blockchain_sethead_test.go | 2 +- core/blockchain_snapshot_test.go | 4 ++-- core/blockchain_test.go | 2 +- ethdb/pebble/pebble.go | 4 ++-- node/database.go | 13 +++++++++---- 7 files changed, 23 insertions(+), 18 deletions(-) diff --git a/core/bench_test.go b/core/bench_test.go index 155fa6c3b5..00f924076a 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -183,7 +183,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { if !disk { db = rawdb.NewMemoryDatabase() } else { - pdb, err := pebble.New(b.TempDir(), 128, 128, "", false) + pdb, err := pebble.New(b.TempDir(), 128, 128, "", false, true) if err != nil { b.Fatalf("cannot create temporary database: %v", err) } @@ -303,7 +303,7 @@ func makeChainForBench(db ethdb.Database, genesis *Genesis, full bool, count uin func benchWriteChain(b *testing.B, full bool, count uint64) { genesis := &Genesis{Config: params.AllEthashProtocolChanges} for i := 0; i < b.N; i++ { - pdb, err := pebble.New(b.TempDir(), 1024, 128, "", false) + pdb, err := pebble.New(b.TempDir(), 1024, 128, "", false, true) if err != nil { b.Fatalf("error opening database: %v", err) } @@ -316,7 +316,7 @@ func benchWriteChain(b *testing.B, full bool, count uint64) { func benchReadChain(b *testing.B, full bool, count uint64) { dir := b.TempDir() - pdb, err := pebble.New(dir, 1024, 128, "", false) + pdb, err := pebble.New(dir, 1024, 128, "", false, true) if err != nil { b.Fatalf("error opening database: %v", err) } @@ -332,7 +332,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { b.ResetTimer() for i := 0; i < b.N; i++ { - pdb, err = pebble.New(dir, 1024, 128, "", false) + pdb, err = pebble.New(dir, 1024, 128, "", false, true) if err != nil { b.Fatalf("error opening database: %v", err) } diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 6c52d057ad..3ff1d77fc8 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -1765,7 +1765,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s datadir := t.TempDir() ancient := filepath.Join(datadir, "ancient") - pdb, err := pebble.New(datadir, 0, 0, "", false) + pdb, err := pebble.New(datadir, 0, 0, "", false, true) if err != nil { t.Fatalf("Failed to create persistent key-value database: %v", err) } @@ -1850,7 +1850,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s chain.stopWithoutSaving() // Start a new blockchain back up and see where the repair leads us - pdb, err = pebble.New(datadir, 0, 0, "", false) + pdb, err = pebble.New(datadir, 0, 0, "", false, true) if err != nil { t.Fatalf("Failed to reopen persistent key-value database: %v", err) } @@ -1915,7 +1915,7 @@ func testIssue23496(t *testing.T, scheme string) { datadir := t.TempDir() ancient := filepath.Join(datadir, "ancient") - pdb, err := pebble.New(datadir, 0, 0, "", false) + pdb, err := pebble.New(datadir, 0, 0, "", false, true) if err != nil { t.Fatalf("Failed to create persistent key-value database: %v", err) } @@ -1973,7 +1973,7 @@ func testIssue23496(t *testing.T, scheme string) { chain.stopWithoutSaving() // Start a new blockchain back up and see where the repair leads us - pdb, err = pebble.New(datadir, 0, 0, "", false) + pdb, err = pebble.New(datadir, 0, 0, "", false, true) if err != nil { t.Fatalf("Failed to reopen persistent key-value database: %v", err) } diff --git a/core/blockchain_sethead_test.go b/core/blockchain_sethead_test.go index 424854b2bf..e998b510df 100644 --- a/core/blockchain_sethead_test.go +++ b/core/blockchain_sethead_test.go @@ -1969,7 +1969,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme datadir := t.TempDir() ancient := filepath.Join(datadir, "ancient") - pdb, err := pebble.New(datadir, 0, 0, "", false) + pdb, err := pebble.New(datadir, 0, 0, "", false, true) if err != nil { t.Fatalf("Failed to create persistent key-value database: %v", err) } diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index 1a6fe38af6..23effea15e 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -66,7 +66,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo datadir := t.TempDir() ancient := filepath.Join(datadir, "ancient") - pdb, err := pebble.New(datadir, 0, 0, "", false) + pdb, err := pebble.New(datadir, 0, 0, "", false, true) if err != nil { t.Fatalf("Failed to create persistent key-value database: %v", err) } @@ -257,7 +257,7 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) { chain.triedb.Close() // Start a new blockchain back up and see where the repair leads us - pdb, err := pebble.New(snaptest.datadir, 0, 0, "", false) + pdb, err := pebble.New(snaptest.datadir, 0, 0, "", false, true) if err != nil { t.Fatalf("Failed to create persistent key-value database: %v", err) } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 8f5a64e206..3f7c03b93c 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -2491,7 +2491,7 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) { datadir := t.TempDir() ancient := path.Join(datadir, "ancient") - pdb, err := pebble.New(datadir, 0, 0, "", false) + pdb, err := pebble.New(datadir, 0, 0, "", false, true) if err != nil { t.Fatalf("Failed to create persistent key-value database: %v", err) } diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index b87ecb2595..969e67af5a 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -144,7 +144,7 @@ func (l panicLogger) Fatalf(format string, args ...interface{}) { // New returns a wrapped pebble DB object. The namespace is the prefix that the // metrics reporting should use for surfacing internal stats. -func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) { +func New(file string, cache int, handles int, namespace string, readonly bool, ephemeral bool) (*Database, error) { // Ensure we have some minimal caching and file guarantees if cache < minCache { cache = minCache @@ -185,7 +185,7 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( fn: file, log: logger, quitChan: make(chan chan error), - writeOptions: &pebble.WriteOptions{Sync: false}, + writeOptions: &pebble.WriteOptions{Sync: !ephemeral}, } opt := &pebble.Options{ // Pebble has a single combined cache area and the write diff --git a/node/database.go b/node/database.go index e3ccb91066..b7d0d856cb 100644 --- a/node/database.go +++ b/node/database.go @@ -36,6 +36,11 @@ type openOptions struct { Cache int // the capacity(in megabytes) of the data caching Handles int // number of files to be open simultaneously ReadOnly bool + + // Ephemeral means that filesystem sync operations should be avoided: + // data integrity in the face of a crash is not important. This option + // should typically be used in tests. + Ephemeral bool } // openDatabase opens both a disk-based key-value database such as leveldb or pebble, but also @@ -78,7 +83,7 @@ func openKeyValueDatabase(o openOptions) (ethdb.Database, error) { } if o.Type == rawdb.DBPebble || existingDb == rawdb.DBPebble { log.Info("Using pebble as the backing database") - return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) + return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) } if o.Type == rawdb.DBLeveldb || existingDb == rawdb.DBLeveldb { log.Info("Using leveldb as the backing database") @@ -86,7 +91,7 @@ func openKeyValueDatabase(o openOptions) (ethdb.Database, error) { } // No pre-existing database, no user-requested one either. Default to Pebble. log.Info("Defaulting to pebble as the backing database") - return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) + return newPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.Ephemeral) } // newLevelDBDatabase creates a persistent key-value database without a freezer @@ -102,8 +107,8 @@ func newLevelDBDatabase(file string, cache int, handles int, namespace string, r // newPebbleDBDatabase creates a persistent key-value database without a freezer // moving immutable chain segments into cold storage. -func newPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { - db, err := pebble.New(file, cache, handles, namespace, readonly) +func newPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool, ephemeral bool) (ethdb.Database, error) { + db, err := pebble.New(file, cache, handles, namespace, readonly, ephemeral) if err != nil { return nil, err } From 9f83e9e6734f82facbc66fd4cd591dd648aaec92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Thu, 3 Apr 2025 16:04:11 +0200 Subject: [PATCH 19/35] beacon/blsync: add checkpoint import/export file feature (#31469) This PR adds a new `--beacon.checkpoint.file` config flag to geth and blsync which specifies a checkpoint import/export file. If a file with an existing checkpoint is specified, it is used for initialization instead of the hardcoded one (except when `--beacon.checkpoint` is also specified simultaneously). Whenever the client encounters a new valid finality update with a suitable finalized beacon block root at an epoch boundary, it saves the block root in hex format to the checkpoint file. --- beacon/blsync/client.go | 10 +++++++++- beacon/light/head_tracker.go | 9 ++++++++- beacon/params/config.go | 34 ++++++++++++++++++++++++++++++++++ cmd/blsync/main.go | 1 + cmd/geth/main.go | 1 + cmd/utils/flags.go | 32 ++++++++++++++++++++++++++------ 6 files changed, 79 insertions(+), 8 deletions(-) diff --git a/beacon/blsync/client.go b/beacon/blsync/client.go index 3c93754d3d..744f469124 100644 --- a/beacon/blsync/client.go +++ b/beacon/blsync/client.go @@ -23,9 +23,11 @@ import ( "github.com/ethereum/go-ethereum/beacon/light/sync" "github.com/ethereum/go-ethereum/beacon/params" "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -46,7 +48,13 @@ func NewClient(config params.ClientConfig) *Client { var ( db = memorydb.New() committeeChain = light.NewCommitteeChain(db, &config.ChainConfig, config.Threshold, !config.NoFilter) - headTracker = light.NewHeadTracker(committeeChain, config.Threshold) + headTracker = light.NewHeadTracker(committeeChain, config.Threshold, func(checkpoint common.Hash) { + if saved, err := config.SaveCheckpointToFile(checkpoint); saved { + log.Debug("Saved beacon checkpoint", "file", config.CheckpointFile, "checkpoint", checkpoint) + } else if err != nil { + log.Error("Failed to save beacon checkpoint", "file", config.CheckpointFile, "checkpoint", checkpoint, "error", err) + } + }) ) headSync := sync.NewHeadSync(headTracker, committeeChain) diff --git a/beacon/light/head_tracker.go b/beacon/light/head_tracker.go index 010e548ddb..62faf1dbc1 100644 --- a/beacon/light/head_tracker.go +++ b/beacon/light/head_tracker.go @@ -21,7 +21,9 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/beacon/params" "github.com/ethereum/go-ethereum/beacon/types" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" ) @@ -38,13 +40,15 @@ type HeadTracker struct { hasFinalityUpdate bool prefetchHead types.HeadInfo changeCounter uint64 + saveCheckpoint func(common.Hash) } // NewHeadTracker creates a new HeadTracker. -func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int) *HeadTracker { +func NewHeadTracker(committeeChain *CommitteeChain, minSignerCount int, saveCheckpoint func(common.Hash)) *HeadTracker { return &HeadTracker{ committeeChain: committeeChain, minSignerCount: minSignerCount, + saveCheckpoint: saveCheckpoint, } } @@ -100,6 +104,9 @@ func (h *HeadTracker) ValidateFinality(update types.FinalityUpdate) (bool, error if replace { h.finalityUpdate, h.hasFinalityUpdate = update, true h.changeCounter++ + if h.saveCheckpoint != nil && update.Finalized.Slot%params.EpochLength == 0 { + h.saveCheckpoint(update.Finalized.Hash()) + } } return replace, err } diff --git a/beacon/params/config.go b/beacon/params/config.go index be2a40f171..2f6ba082c5 100644 --- a/beacon/params/config.go +++ b/beacon/params/config.go @@ -54,6 +54,7 @@ type ChainConfig struct { GenesisValidatorsRoot common.Hash // Root hash of the genesis validator set, used for signature domain calculation Forks Forks Checkpoint common.Hash + CheckpointFile string } // ForkAtEpoch returns the latest active fork at the given epoch. @@ -211,3 +212,36 @@ func (f Forks) Less(i, j int) bool { } return f[i].knownIndex < f[j].knownIndex } + +// SetCheckpointFile sets the checkpoint import/export file name and attempts to +// read the checkpoint from the file if it already exists. It returns true if +// a checkpoint has been loaded. +func (c *ChainConfig) SetCheckpointFile(checkpointFile string) (bool, error) { + c.CheckpointFile = checkpointFile + file, err := os.ReadFile(checkpointFile) + if os.IsNotExist(err) { + return false, nil // did not load checkpoint + } + if err != nil { + return false, fmt.Errorf("failed to read beacon checkpoint file: %v", err) + } + cp, err := hexutil.Decode(string(file)) + if err != nil { + return false, fmt.Errorf("failed to decode hex string in beacon checkpoint file: %v", err) + } + if len(cp) != 32 { + return false, fmt.Errorf("invalid hex string length in beacon checkpoint file: %d", len(cp)) + } + copy(c.Checkpoint[:len(cp)], cp) + return true, nil +} + +// SaveCheckpointToFile saves the given checkpoint to file if a checkpoint +// import/export file has been specified. +func (c *ChainConfig) SaveCheckpointToFile(checkpoint common.Hash) (bool, error) { + if c.CheckpointFile == "" { + return false, nil + } + err := os.WriteFile(c.CheckpointFile, []byte(checkpoint.Hex()), 0600) + return err == nil, err +} diff --git a/cmd/blsync/main.go b/cmd/blsync/main.go index 60caa4aa2a..39a9407304 100644 --- a/cmd/blsync/main.go +++ b/cmd/blsync/main.go @@ -43,6 +43,7 @@ func main() { utils.BeaconGenesisRootFlag, utils.BeaconGenesisTimeFlag, utils.BeaconCheckpointFlag, + utils.BeaconCheckpointFileFlag, //TODO datadir for optional permanent database utils.MainnetFlag, utils.SepoliaFlag, diff --git a/cmd/geth/main.go b/cmd/geth/main.go index cd74fb7b6a..ab46e059f3 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -157,6 +157,7 @@ var ( utils.BeaconGenesisRootFlag, utils.BeaconGenesisTimeFlag, utils.BeaconCheckpointFlag, + utils.BeaconCheckpointFileFlag, }, utils.NetworkFlags, utils.DatabaseFlags) rpcFlags = []cli.Flag{ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index fb2892d2c1..f5fc94cebc 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -342,6 +342,11 @@ var ( Usage: "Beacon chain weak subjectivity checkpoint block hash", Category: flags.BeaconCategory, } + BeaconCheckpointFileFlag = &cli.StringFlag{ + Name: "beacon.checkpoint.file", + Usage: "Beacon chain weak subjectivity checkpoint import/export file", + Category: flags.BeaconCategory, + } BlsyncApiFlag = &cli.StringFlag{ Name: "blsync.engine.api", Usage: "Target EL engine API URL", @@ -1890,7 +1895,7 @@ func MakeBeaconLightConfig(ctx *cli.Context) bparams.ClientConfig { if !ctx.IsSet(BeaconGenesisTimeFlag.Name) { Fatalf("Custom beacon chain config is specified but genesis time is missing") } - if !ctx.IsSet(BeaconCheckpointFlag.Name) { + if !ctx.IsSet(BeaconCheckpointFlag.Name) && !ctx.IsSet(BeaconCheckpointFileFlag.Name) { Fatalf("Custom beacon chain config is specified but checkpoint is missing") } config.ChainConfig = bparams.ChainConfig{ @@ -1915,13 +1920,28 @@ func MakeBeaconLightConfig(ctx *cli.Context) bparams.ClientConfig { } } // Checkpoint is required with custom chain config and is optional with pre-defined config - if ctx.IsSet(BeaconCheckpointFlag.Name) { - if c, err := hexutil.Decode(ctx.String(BeaconCheckpointFlag.Name)); err == nil && len(c) <= 32 { - copy(config.Checkpoint[:len(c)], c) - } else { - Fatalf("Invalid hex string", "beacon.checkpoint", ctx.String(BeaconCheckpointFlag.Name), "error", err) + // If both checkpoint block hash and checkpoint file are specified then the + // client is initialized with the specified block hash and new checkpoints + // are saved to the specified file. + if ctx.IsSet(BeaconCheckpointFileFlag.Name) { + if _, err := config.SetCheckpointFile(ctx.String(BeaconCheckpointFileFlag.Name)); err != nil { + Fatalf("Could not load beacon checkpoint file", "beacon.checkpoint.file", ctx.String(BeaconCheckpointFileFlag.Name), "error", err) } } + if ctx.IsSet(BeaconCheckpointFlag.Name) { + hex := ctx.String(BeaconCheckpointFlag.Name) + c, err := hexutil.Decode(hex) + if err != nil { + Fatalf("Invalid hex string", "beacon.checkpoint", hex, "error", err) + } + if len(c) != 32 { + Fatalf("Invalid hex string length", "beacon.checkpoint", hex, "length", len(c)) + } + copy(config.Checkpoint[:len(c)], c) + } + if config.Checkpoint == (common.Hash{}) { + Fatalf("Beacon checkpoint not specified") + } config.Apis = ctx.StringSlice(BeaconApiFlag.Name) if config.Apis == nil { Fatalf("Beacon node light client API URL not specified") From ff365afc63f047c09ff04f41e12678c6b6698b72 Mon Sep 17 00:00:00 2001 From: Nathan Jo <162083209+qqqeck@users.noreply.github.com> Date: Fri, 4 Apr 2025 17:56:55 +0900 Subject: [PATCH 20/35] p2p/nat: remove forceful port mapping in upnp (#30265) Here we are modifying the port mapping logic so that existing port mappings will only be removed when they were previously created by geth. The AddAnyPortMapping functionality has been adapted to work consistently between the IGDv1 and IGDv2 backends. --- p2p/nat/natpmp.go | 3 +++ p2p/nat/natupnp.go | 36 +++++++++++++++++++++++------------- p2p/server_nat.go | 13 ++++--------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/p2p/nat/natpmp.go b/p2p/nat/natpmp.go index 4a9644ac1a..ee07eb4ff6 100644 --- a/p2p/nat/natpmp.go +++ b/p2p/nat/natpmp.go @@ -49,6 +49,9 @@ func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lif if lifetime <= 0 { return 0, errors.New("lifetime must not be <= 0") } + if extport == 0 { + extport = intport + } // Note order of port arguments is switched between our // AddMapping and the client's AddPortMapping. res, err := n.c.AddPortMapping(strings.ToLower(protocol), intport, extport, int(lifetime/time.Second)) diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index 1014dc69d2..ed00b8eeb6 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -86,15 +86,15 @@ func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, li } protocol = strings.ToUpper(protocol) lifetimeS := uint32(lifetime / time.Second) - n.DeleteMapping(protocol, extport, intport) - err = n.withRateLimit(func() error { - return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) - }) - if err == nil { - return uint16(extport), nil + if extport == 0 { + extport = intport + } else { + // Only delete port mapping if the external port was already used by geth. + n.DeleteMapping(protocol, extport, intport) } - // Try addAnyPortMapping if mapping specified port didn't work. + + // Try to add port mapping, preferring the specified external port. err = n.withRateLimit(func() error { p, err := n.addAnyPortMapping(protocol, extport, intport, ip, desc, lifetimeS) if err == nil { @@ -105,18 +105,28 @@ func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, li return uint16(extport), err } +// addAnyPortMapping tries to add a port mapping with the specified external port. +// If the external port is already in use, it will try to assign another port. func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.IP, desc string, lifetimeS uint32) (uint16, error) { if client, ok := n.client.(*internetgateway2.WANIPConnection2); ok { return client.AddAnyPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) } - // It will retry with a random port number if the client does - // not support AddAnyPortMapping. - extport = n.randomPort() + // For IGDv1 and v1 services we should first try to add with extport. err := n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) - if err != nil { - return 0, err + if err == nil { + return uint16(extport), nil } - return uint16(extport), nil + + // If above fails, we retry with a random port. + // We retry several times because of possible port conflicts. + for i := 0; i < 3; i++ { + extport = n.randomPort() + err := n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + if err == nil { + return uint16(extport), nil + } + } + return 0, err } func (n *upnp) randomPort() int { diff --git a/p2p/server_nat.go b/p2p/server_nat.go index 933993bc1f..5830f950e1 100644 --- a/p2p/server_nat.go +++ b/p2p/server_nat.go @@ -150,14 +150,9 @@ func (srv *Server) portMappingLoop() { continue } - external := m.port - if m.extPort != 0 { - external = m.extPort - } - log := newLogger(m.protocol, external, m.port) - + log := newLogger(m.protocol, m.extPort, m.port) log.Trace("Attempting port mapping") - p, err := srv.NAT.AddMapping(m.protocol, external, m.port, m.name, portMapDuration) + p, err := srv.NAT.AddMapping(m.protocol, m.extPort, m.port, m.name, portMapDuration) if err != nil { log.Debug("Couldn't add port mapping", "err", err) m.extPort = 0 @@ -167,8 +162,8 @@ func (srv *Server) portMappingLoop() { // It was mapped! m.extPort = int(p) m.nextTime = srv.clock.Now().Add(portMapRefreshInterval) - if external != m.extPort { - log = newLogger(m.protocol, m.extPort, m.port) + log = newLogger(m.protocol, m.extPort, m.port) + if m.port != m.extPort { log.Info("NAT mapped alternative port") } else { log.Info("NAT mapped port") From 77dc1acafaad69e6adc98293541ee49644ed9218 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Fri, 4 Apr 2025 14:07:46 +0200 Subject: [PATCH 21/35] internal/era: random access to header and receipts (#31544) Co-authored-by: lightclient Add GetHeaderByNumber and GetReceiptsByNumber to allow more efficient API request filling from Era files. --- internal/era/e2store/e2store.go | 12 +++++++ internal/era/era.go | 64 ++++++++++++++++++++++++++++----- internal/era/era_test.go | 53 ++++++++++++++++++++++----- 3 files changed, 112 insertions(+), 17 deletions(-) diff --git a/internal/era/e2store/e2store.go b/internal/era/e2store/e2store.go index 9832b72d48..b0d43bf55a 100644 --- a/internal/era/e2store/e2store.go +++ b/internal/era/e2store/e2store.go @@ -219,3 +219,15 @@ func (r *Reader) FindAll(want uint16) ([]*Entry, error) { off += int64(headerSize + length) } } + +// SkipN skips `n` entries starting from `offset` and returns the new offset. +func (r *Reader) SkipN(offset int64, n uint64) (int64, error) { + for i := uint64(0); i < n; i++ { + length, err := r.LengthAt(offset) + if err != nil { + return 0, err + } + offset += length + } + return offset, nil +} diff --git a/internal/era/era.go b/internal/era/era.go index daf337963d..5129186fe7 100644 --- a/internal/era/era.go +++ b/internal/era/era.go @@ -70,7 +70,7 @@ func ReadDir(dir, network string) ([]string, error) { } parts := strings.Split(entry.Name(), "-") if len(parts) != 3 || parts[0] != network { - // invalid era1 filename, skip + // Invalid era1 filename, skip. continue } if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil { @@ -126,6 +126,29 @@ func (e *Era) Close() error { return e.f.Close() } +// GetHeaderByNumber returns the header for the given block number. +func (e *Era) GetHeaderByNumber(num uint64) (*types.Header, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, errors.New("out-of-bounds") + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + + // Read and decompress header. + r, _, err := newSnappyReader(e.s, TypeCompressedHeader, off) + if err != nil { + return nil, err + } + var header types.Header + if err := rlp.Decode(r, &header); err != nil { + return nil, err + } + return &header, nil +} + +// GetBlockByNumber returns the block for the given block number. func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { if e.m.start > num || e.m.start+e.m.count <= num { return nil, errors.New("out-of-bounds") @@ -154,6 +177,34 @@ func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { return types.NewBlockWithHeader(&header).WithBody(body), nil } +// GetReceiptsByNumber returns the receipts for the given block number. +func (e *Era) GetReceiptsByNumber(num uint64) (types.Receipts, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, errors.New("out-of-bounds") + } + off, err := e.readOffset(num) + if err != nil { + return nil, err + } + + // Skip over header and body. + off, err = e.s.SkipN(off, 2) + if err != nil { + return nil, err + } + + // Read and decompress receipts. + r, _, err := newSnappyReader(e.s, TypeCompressedReceipts, off) + if err != nil { + return nil, err + } + var receipts types.Receipts + if err := rlp.Decode(r, &receipts); err != nil { + return nil, err + } + return receipts, nil +} + // Accumulator reads the accumulator entry in the Era1 file. func (e *Era) Accumulator() (common.Hash, error) { entry, err := e.s.Find(TypeAccumulator) @@ -187,13 +238,10 @@ func (e *Era) InitialTD() (*big.Int, error) { } off += n - // Skip over next two records. - for i := 0; i < 2; i++ { - length, err := e.s.LengthAt(off) - if err != nil { - return nil, err - } - off += length + // Skip over header and body. + off, err = e.s.SkipN(off, 2) + if err != nil { + return nil, err } // Read total difficulty after first block. diff --git a/internal/era/era_test.go b/internal/era/era_test.go index 72c3b385dd..46fc2e91f3 100644 --- a/internal/era/era_test.go +++ b/internal/era/era_test.go @@ -18,12 +18,15 @@ package era import ( "bytes" + "fmt" "io" "math/big" "os" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" ) type testchain struct { @@ -48,9 +51,9 @@ func TestEra1Builder(t *testing.T) { chain = testchain{} ) for i := 0; i < 128; i++ { - chain.headers = append(chain.headers, []byte{byte('h'), byte(i)}) - chain.bodies = append(chain.bodies, []byte{byte('b'), byte(i)}) - chain.receipts = append(chain.receipts, []byte{byte('r'), byte(i)}) + chain.headers = append(chain.headers, mustEncode(&types.Header{Number: big.NewInt(int64(i))})) + chain.bodies = append(chain.bodies, mustEncode(&types.Body{Transactions: []*types.Transaction{types.NewTransaction(0, common.Address{byte(i)}, nil, 0, nil, nil)}})) + chain.receipts = append(chain.receipts, mustEncode(&types.Receipts{{CumulativeGasUsed: uint64(i)}})) chain.tds = append(chain.tds, big.NewInt(int64(i))) } @@ -91,13 +94,25 @@ func TestEra1Builder(t *testing.T) { t.Fatalf("unexpected error %v", it.Error()) } // Check headers. - header, err := io.ReadAll(it.Header) + rawHeader, err := io.ReadAll(it.Header) + if err != nil { + t.Fatalf("error reading header from iterator: %v", err) + } + if !bytes.Equal(rawHeader, chain.headers[i]) { + t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], rawHeader) + } + header, err := e.GetHeaderByNumber(i) if err != nil { t.Fatalf("error reading header: %v", err) } - if !bytes.Equal(header, chain.headers[i]) { - t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], header) + encHeader, err := rlp.EncodeToBytes(header) + if err != nil { + t.Fatalf("error encoding header: %v", err) } + if !bytes.Equal(encHeader, chain.headers[i]) { + t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], encHeader) + } + // Check bodies. body, err := io.ReadAll(it.Body) if err != nil { @@ -106,13 +121,25 @@ func TestEra1Builder(t *testing.T) { if !bytes.Equal(body, chain.bodies[i]) { t.Fatalf("mismatched body: want %s, got %s", chain.bodies[i], body) } + // Check receipts. - receipts, err := io.ReadAll(it.Receipts) + rawReceipts, err := io.ReadAll(it.Receipts) + if err != nil { + t.Fatalf("error reading receipts from iterator: %v", err) + } + if !bytes.Equal(rawReceipts, chain.receipts[i]) { + t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], rawReceipts) + } + receipts, err := e.GetReceiptsByNumber(i) if err != nil { t.Fatalf("error reading receipts: %v", err) } - if !bytes.Equal(receipts, chain.receipts[i]) { - t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], receipts) + encReceipts, err := rlp.EncodeToBytes(receipts) + if err != nil { + t.Fatalf("error encoding receipts: %v", err) + } + if !bytes.Equal(encReceipts, chain.receipts[i]) { + t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], encReceipts) } // Check total difficulty. @@ -144,3 +171,11 @@ func TestEraFilename(t *testing.T) { } } } + +func mustEncode(obj any) []byte { + b, err := rlp.EncodeToBytes(obj) + if err != nil { + panic(fmt.Sprintf("failed in encode obj: %v", err)) + } + return b +} From 21b035eb29f6489ffee66d8ee2451873bc96dd2d Mon Sep 17 00:00:00 2001 From: Delweng Date: Mon, 7 Apr 2025 13:16:26 +0800 Subject: [PATCH 22/35] cmd/geth: set trie,gc and other cache flags for import chain (#31577) Signed-off-by: jsvisa --- cmd/geth/chaincmd.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 0c36b82af5..2279509542 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -79,11 +79,15 @@ if one is set. Otherwise it prints the genesis from the datadir.`, Usage: "Import a blockchain file", ArgsUsage: " ( ... ) ", Flags: slices.Concat([]cli.Flag{ - utils.CacheFlag, utils.GCModeFlag, utils.SnapshotFlag, + utils.CacheFlag, utils.CacheDatabaseFlag, + utils.CacheTrieFlag, utils.CacheGCFlag, + utils.CacheSnapshotFlag, + utils.CacheNoPrefetchFlag, + utils.CachePreimagesFlag, utils.NoCompactionFlag, utils.MetricsEnabledFlag, utils.MetricsEnabledExpensiveFlag, From ec6d1044045bcd6f8fe96892d840fe35c16ca7c8 Mon Sep 17 00:00:00 2001 From: Ocenka Date: Tue, 8 Apr 2025 15:44:13 +0300 Subject: [PATCH 23/35] eth/remotedb: improve error handling (#31331) This PR improves error handling in the remotedb package by fixing two issues: 1. In the `Has` method, we now properly propagate errors instead of silently returning false. This makes the behavior more predictable and helps clients better understand when there are connection issues. 2. In the `New` constructor, we add a nil check for the client parameter to prevent potential panics. This follows Go best practices for constructor functions. These changes make the code more robust and follow Go's error handling idioms without requiring any changes to other parts of the codebase. Changes: - Modified `Has` method to return errors instead of silently returning false - Added nil check in `New` constructor - Fixed field name in constructor to match struct definition --- ethdb/remotedb/remotedb.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index 247a4392db..8a91fdbcf2 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -34,7 +34,7 @@ type Database struct { func (db *Database) Has(key []byte) (bool, error) { if _, err := db.Get(key); err != nil { - return false, nil + return false, err } return true, nil } @@ -50,7 +50,7 @@ func (db *Database) Get(key []byte) ([]byte, error) { func (db *Database) HasAncient(kind string, number uint64) (bool, error) { if _, err := db.Ancient(kind, number); err != nil { - return false, nil + return false, err } return true, nil } @@ -144,7 +144,8 @@ func (db *Database) Close() error { } func New(client *rpc.Client) ethdb.Database { - return &Database{ - remote: client, + if client == nil { + return nil } + return &Database{remote: client} } From 2e739fce584a3cc8d84bf4c4604cd81c07294e2b Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 8 Apr 2025 21:46:27 +0800 Subject: [PATCH 24/35] core/txpool: add 7702 protection to blobpool (#31526) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This pull request introduces two constraints in the blobPool: (a) If the sender has a pending authorization or delegation, only one in-flight executable transaction can be cached. (b) If the authority address in a SetCode transaction is already reserved by the blobPool, the transaction will be rejected. These constraints mitigate an attack where an attacker spams the pool with numerous blob transactions, evicts other transactions, and then cancels all pending blob transactions by draining the sender’s funds if they have a delegation. Note, because there is no exclusive lock held between different subpools when processing transactions, it's totally possible the SetCode transaction and blob transactions with conflict sender and authorities are accepted simultaneously. I think it's acceptable as it's very hard to be exploited. --------- Co-authored-by: lightclient --- core/txpool/blobpool/blobpool.go | 66 ++++++++--- core/txpool/blobpool/blobpool_test.go | 60 +++------- core/txpool/errors.go | 4 + core/txpool/legacypool/legacypool.go | 60 ++++++---- core/txpool/legacypool/legacypool2_test.go | 8 +- core/txpool/legacypool/legacypool_test.go | 82 +++++--------- core/txpool/reserver.go | 124 +++++++++++++++++++++ core/txpool/subpool.go | 6 +- core/txpool/txpool.go | 77 ++----------- eth/backend.go | 10 +- eth/protocols/eth/handler_test.go | 2 +- 11 files changed, 286 insertions(+), 213 deletions(-) create mode 100644 core/txpool/reserver.go diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 5a20c3ce5a..b1966905a8 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -298,8 +298,9 @@ func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transac // minimums will need to be done only starting at the swapped in/out nonce // and leading up to the first no-change. type BlobPool struct { - config Config // Pool configuration - reserve txpool.AddressReserver // Address reserver to ensure exclusivity across subpools + config Config // Pool configuration + reserver *txpool.Reserver // Address reserver to ensure exclusivity across subpools + hasPendingAuth func(common.Address) bool // Determine whether the specified address has a pending 7702-auth store billy.Database // Persistent data store for the tx metadata and blobs stored uint64 // Useful data size of all transactions on disk @@ -329,13 +330,14 @@ type BlobPool struct { // New creates a new blob transaction pool to gather, sort and filter inbound // blob transactions from the network. -func New(config Config, chain BlockChain) *BlobPool { +func New(config Config, chain BlockChain, hasPendingAuth func(common.Address) bool) *BlobPool { // Sanitize the input to ensure no vulnerable gas prices are set config = (&config).sanitize() // Create the transaction pool with its initial settings return &BlobPool{ config: config, + hasPendingAuth: hasPendingAuth, signer: types.LatestSigner(chain.Config()), chain: chain, lookup: newLookup(), @@ -353,8 +355,8 @@ func (p *BlobPool) Filter(tx *types.Transaction) bool { // Init sets the gas price needed to keep a transaction in the pool and the chain // head to allow balance / nonce checks. The transaction journal will be loaded // from disk and filtered based on the provided starting settings. -func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserve txpool.AddressReserver) error { - p.reserve = reserve +func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver *txpool.Reserver) error { + p.reserver = reserver var ( queuedir string @@ -499,7 +501,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { return err } if _, ok := p.index[sender]; !ok { - if err := p.reserve(sender, true); err != nil { + if err := p.reserver.Hold(sender); err != nil { return err } p.index[sender] = []*blobTxMeta{} @@ -554,7 +556,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 if inclusions != nil { // only during reorgs will the heap be initialized heap.Remove(p.evict, p.evict.index[addr]) } - p.reserve(addr, false) + p.reserver.Release(addr) if gapped { log.Warn("Dropping dangling blob transactions", "from", addr, "missing", next, "drop", nonces, "ids", ids) @@ -707,7 +709,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 if inclusions != nil { // only during reorgs will the heap be initialized heap.Remove(p.evict, p.evict.index[addr]) } - p.reserve(addr, false) + p.reserver.Release(addr) } else { p.index[addr] = txs } @@ -1006,7 +1008,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error { // Update the indices and metrics meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), tx) if _, ok := p.index[addr]; !ok { - if err := p.reserve(addr, true); err != nil { + if err := p.reserver.Hold(addr); err != nil { log.Warn("Failed to reserve account for blob pool", "tx", tx.Hash(), "from", addr, "err", err) return err } @@ -1066,7 +1068,7 @@ func (p *BlobPool) SetGasTip(tip *big.Int) { delete(p.spent, addr) heap.Remove(p.evict, p.evict.index[addr]) - p.reserve(addr, false) + p.reserver.Release(addr) } // Clear out the transactions from the data store log.Warn("Dropping underpriced blob transaction", "from", addr, "rejected", tx.nonce, "tip", tx.execTipCap, "want", tip, "drop", nonces, "ids", ids) @@ -1101,6 +1103,39 @@ func (p *BlobPool) ValidateTxBasics(tx *types.Transaction) error { return txpool.ValidateTransaction(tx, p.head, p.signer, opts) } +// checkDelegationLimit determines if the tx sender is delegated or has a +// pending delegation, and if so, ensures they have at most one in-flight +// **executable** transaction, e.g. disallow stacked and gapped transactions +// from the account. +func (p *BlobPool) checkDelegationLimit(tx *types.Transaction) error { + from, _ := types.Sender(p.signer, tx) // validated + + // Short circuit if the sender has neither delegation nor pending delegation. + if p.state.GetCodeHash(from) == types.EmptyCodeHash { + // Because there is no exclusive lock held between different subpools + // when processing transactions, a blob transaction may be accepted + // while other SetCode transactions with pending authorities from the + // same address are also accepted simultaneously. + // + // This scenario is considered acceptable, as the rule primarily ensures + // that attackers cannot easily and endlessly stack blob transactions + // with a delegated or pending delegated sender. + if p.hasPendingAuth == nil || !p.hasPendingAuth(from) { + return nil + } + } + // Allow a single in-flight pending transaction. + pending := p.index[from] + if len(pending) == 0 { + return nil + } + // If account already has a pending transaction, allow replacement only. + if len(pending) == 1 && pending[0].nonce == tx.Nonce() { + return nil + } + return txpool.ErrInflightTxLimitReached +} + // validateTx checks whether a transaction is valid according to the consensus // rules and adheres to some heuristic limits of the local node (price and size). func (p *BlobPool) validateTx(tx *types.Transaction) error { @@ -1141,6 +1176,9 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error { if err := txpool.ValidateTransactionWithState(tx, p.signer, stateOpts); err != nil { return err } + if err := p.checkDelegationLimit(tx); err != nil { + return err + } // If the transaction replaces an existing one, ensure that price bumps are // adhered to. var ( @@ -1369,7 +1407,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { // only by this subpool until all transactions are evicted from, _ := types.Sender(p.signer, tx) // already validated above if _, ok := p.index[from]; !ok { - if err := p.reserve(from, true); err != nil { + if err := p.reserver.Hold(from); err != nil { addNonExclusiveMeter.Mark(1) return err } @@ -1381,7 +1419,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { // by a return statement before running deferred methods. Take care with // removing or subscoping err as it will break this clause. if err != nil { - p.reserve(from, false) + p.reserver.Release(from) } }() } @@ -1513,7 +1551,7 @@ func (p *BlobPool) drop() { if last { delete(p.index, from) delete(p.spent, from) - p.reserve(from, false) + p.reserver.Release(from) } else { txs[len(txs)-1] = nil txs = txs[:len(txs)-1] @@ -1789,7 +1827,7 @@ func (p *BlobPool) Clear() { // can't happen until Clear releases the reservation lock. Clear cannot // acquire the subpool lock until the transaction addition is completed. for acct := range p.index { - p.reserve(acct, false) + p.reserver.Release(acct) } p.lookup = newLookup() p.index = make(map[common.Address][]*blobTxMeta) diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index b7c6cfa51e..4dfba3b52b 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -26,7 +26,6 @@ import ( "math/big" "os" "path/filepath" - "sync" "testing" "github.com/ethereum/go-ethereum/common" @@ -168,33 +167,6 @@ func (bc *testBlockChain) StateAt(common.Hash) (*state.StateDB, error) { return bc.statedb, nil } -// makeAddressReserver is a utility method to sanity check that accounts are -// properly reserved by the blobpool (no duplicate reserves or unreserves). -func makeAddressReserver() txpool.AddressReserver { - var ( - reserved = make(map[common.Address]struct{}) - lock sync.Mutex - ) - return func(addr common.Address, reserve bool) error { - lock.Lock() - defer lock.Unlock() - - _, exists := reserved[addr] - if reserve { - if exists { - panic("already reserved") - } - reserved[addr] = struct{}{} - return nil - } - if !exists { - panic("not reserved") - } - delete(reserved, addr) - return nil - } -} - // makeTx is a utility method to construct a random blob transaction and sign it // with a valid key, only setting the interesting fields from the perspective of // the blob pool. @@ -433,6 +405,10 @@ func verifyBlobRetrievals(t *testing.T, pool *BlobPool) { } } +func newReserver() *txpool.Reserver { + return txpool.NewReservationTracker().NewHandle(42) +} + // Tests that transactions can be loaded from disk on startup and that they are // correctly discarded if invalid. // @@ -699,8 +675,8 @@ func TestOpenDrops(t *testing.T) { blobfee: uint256.NewInt(params.BlobTxMinBlobGasprice), statedb: statedb, } - pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + pool := New(Config{Datadir: storage}, chain, nil) + if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } defer pool.Close() @@ -817,8 +793,8 @@ func TestOpenIndex(t *testing.T) { blobfee: uint256.NewInt(params.BlobTxMinBlobGasprice), statedb: statedb, } - pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + pool := New(Config{Datadir: storage}, chain, nil) + if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } defer pool.Close() @@ -918,8 +894,8 @@ func TestOpenHeap(t *testing.T) { blobfee: uint256.NewInt(105), statedb: statedb, } - pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + pool := New(Config{Datadir: storage}, chain, nil) + if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } defer pool.Close() @@ -997,8 +973,8 @@ func TestOpenCap(t *testing.T) { blobfee: uint256.NewInt(105), statedb: statedb, } - pool := New(Config{Datadir: storage, Datacap: datacap}, chain) - if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + pool := New(Config{Datadir: storage, Datacap: datacap}, chain, nil) + if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } // Verify that enough transactions have been dropped to get the pool's size @@ -1098,8 +1074,8 @@ func TestChangingSlotterSize(t *testing.T) { blobfee: uint256.NewInt(105), statedb: statedb, } - pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + pool := New(Config{Datadir: storage}, chain, nil) + if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil { t.Fatalf("failed to create blob pool: %v", err) } @@ -1541,8 +1517,8 @@ func TestAdd(t *testing.T) { blobfee: uint256.NewInt(105), statedb: statedb, } - pool := New(Config{Datadir: storage}, chain) - if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + pool := New(Config{Datadir: storage}, chain, nil) + if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil { t.Fatalf("test %d: failed to create blob pool: %v", i, err) } verifyPoolInternals(t, pool) @@ -1638,10 +1614,10 @@ func benchmarkPoolPending(b *testing.B, datacap uint64) { blobfee: uint256.NewInt(blobfee), statedb: statedb, } - pool = New(Config{Datadir: ""}, chain) + pool = New(Config{Datadir: ""}, chain, nil) ) - if err := pool.Init(1, chain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(1, chain.CurrentBlock(), newReserver()); err != nil { b.Fatalf("failed to create blob pool: %v", err) } // Make the pool not use disk (just drop everything). This test never reads diff --git a/core/txpool/errors.go b/core/txpool/errors.go index c38644857e..02f5703b6c 100644 --- a/core/txpool/errors.go +++ b/core/txpool/errors.go @@ -56,4 +56,8 @@ var ( // input transaction of non-blob type when a blob transaction from this sender // remains pending (and vice-versa). ErrAlreadyReserved = errors.New("address already reserved") + + // ErrInflightTxLimitReached is returned when the maximum number of in-flight + // transactions is reached for specific accounts. + ErrInflightTxLimitReached = errors.New("in-flight transaction limit reached for delegated accounts") ) diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 9066f3e16b..278ad0791f 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -63,10 +63,6 @@ var ( // another remote transaction. ErrTxPoolOverflow = errors.New("txpool is full") - // ErrInflightTxLimitReached is returned when the maximum number of in-flight - // transactions is reached for specific accounts. - ErrInflightTxLimitReached = errors.New("in-flight transaction limit reached for delegated accounts") - // ErrOutOfOrderTxFromDelegated is returned when the transaction with gapped // nonce received from the accounts with delegation or pending delegation. ErrOutOfOrderTxFromDelegated = errors.New("gapped-nonce tx from delegated accounts") @@ -241,8 +237,8 @@ type LegacyPool struct { currentHead atomic.Pointer[types.Header] // Current head of the blockchain currentState *state.StateDB // Current state in the blockchain head pendingNonces *noncer // Pending state tracking virtual nonces + reserver *txpool.Reserver // Address reserver to ensure exclusivity across subpools - reserve txpool.AddressReserver // Address reserver to ensure exclusivity across subpools pending map[common.Address]*list // All currently processable transactions queue map[common.Address]*list // Queued but non-processable transactions beats map[common.Address]time.Time // Last heartbeat from each known account @@ -306,9 +302,9 @@ func (pool *LegacyPool) Filter(tx *types.Transaction) bool { // Init sets the gas price needed to keep a transaction in the pool and the chain // head to allow balance / nonce checks. The internal // goroutines will be spun up and the pool deemed operational afterwards. -func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserve txpool.AddressReserver) error { +func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserver *txpool.Reserver) error { // Set the address reserver to request exclusive access to pooled accounts - pool.reserve = reserve + pool.reserver = reserver // Set the basic pool parameters pool.gasTip.Store(uint256.NewInt(gasTip)) @@ -618,7 +614,7 @@ func (pool *LegacyPool) checkDelegationLimit(tx *types.Transaction) error { from, _ := types.Sender(pool.signer, tx) // validated // Short circuit if the sender has neither delegation nor pending delegation. - if pool.currentState.GetCodeHash(from) == types.EmptyCodeHash && pool.all.delegationTxsCount(from) == 0 { + if pool.currentState.GetCodeHash(from) == types.EmptyCodeHash && !pool.all.hasAuth(from) { return nil } pending := pool.pending[from] @@ -633,7 +629,7 @@ func (pool *LegacyPool) checkDelegationLimit(tx *types.Transaction) error { if pending.Contains(tx.Nonce()) { return nil } - return ErrInflightTxLimitReached + return txpool.ErrInflightTxLimitReached } // validateAuth verifies that the transaction complies with code authorization @@ -644,12 +640,24 @@ func (pool *LegacyPool) validateAuth(tx *types.Transaction) error { if err := pool.checkDelegationLimit(tx); err != nil { return err } - // Authorities cannot conflict with any pending or queued transactions. + // Authorities must not conflict with any pending or queued transactions, + // nor with addresses that have already been reserved. if auths := tx.SetCodeAuthorities(); len(auths) > 0 { for _, auth := range auths { if pool.pending[auth] != nil || pool.queue[auth] != nil { return ErrAuthorityReserved } + // Because there is no exclusive lock held between different subpools + // when processing transactions, the SetCode transaction may be accepted + // while other transactions with the same sender address are also + // accepted simultaneously in the other pools. + // + // This scenario is considered acceptable, as the rule primarily ensures + // that attackers cannot easily stack a SetCode transaction when the sender + // is reserved by other pools. + if pool.reserver.Has(auth) { + return ErrAuthorityReserved + } } } return nil @@ -683,7 +691,7 @@ func (pool *LegacyPool) add(tx *types.Transaction) (replaced bool, err error) { _, hasQueued = pool.queue[from] ) if !hasPending && !hasQueued { - if err := pool.reserve(from, true); err != nil { + if err := pool.reserver.Hold(from); err != nil { return false, err } defer func() { @@ -694,7 +702,7 @@ func (pool *LegacyPool) add(tx *types.Transaction) (replaced bool, err error) { // by a return statement before running deferred methods. Take care with // removing or subscoping err as it will break this clause. if err != nil { - pool.reserve(from, false) + pool.reserver.Release(from) } }() } @@ -1087,7 +1095,7 @@ func (pool *LegacyPool) removeTx(hash common.Hash, outofbound bool, unreserve bo _, hasQueued = pool.queue[addr] ) if !hasPending && !hasQueued { - pool.reserve(addr, false) + pool.reserver.Release(addr) } }() } @@ -1467,7 +1475,7 @@ func (pool *LegacyPool) promoteExecutables(accounts []common.Address) []*types.T delete(pool.queue, addr) delete(pool.beats, addr) if _, ok := pool.pending[addr]; !ok { - pool.reserve(addr, false) + pool.reserver.Release(addr) } } } @@ -1653,7 +1661,7 @@ func (pool *LegacyPool) demoteUnexecutables() { if list.Empty() { delete(pool.pending, addr) if _, ok := pool.queue[addr]; !ok { - pool.reserve(addr, false) + pool.reserver.Release(addr) } } } @@ -1862,11 +1870,13 @@ func (t *lookup) removeAuthorities(tx *types.Transaction) { } } -// delegationTxsCount returns the number of pending authorizations for the specified address. -func (t *lookup) delegationTxsCount(addr common.Address) int { +// hasAuth returns a flag indicating whether there are pending authorizations +// from the specified address. +func (t *lookup) hasAuth(addr common.Address) bool { t.lock.RLock() defer t.lock.RUnlock() - return len(t.auths[addr]) + + return len(t.auths[addr]) > 0 } // numSlots calculates the number of slots needed for a single transaction. @@ -1880,8 +1890,8 @@ func (pool *LegacyPool) Clear() { pool.mu.Lock() defer pool.mu.Unlock() - // unreserve each tracked account. Ideally, we could just clear the - // reservation map in the parent txpool context. However, if we clear in + // unreserve each tracked account. Ideally, we could just clear the + // reservation map in the parent txpool context. However, if we clear in // parent context, to avoid exposing the subpool lock, we have to lock the // reservations and then lock each subpool. // @@ -1892,11 +1902,11 @@ func (pool *LegacyPool) Clear() { // * TxPool.Clear attempts to lock subpool mutex // // The transaction addition may attempt to reserve the sender addr which - // can't happen until Clear releases the reservation lock. Clear cannot + // can't happen until Clear releases the reservation lock. Clear cannot // acquire the subpool lock until the transaction addition is completed. for _, tx := range pool.all.txs { senderAddr, _ := types.Sender(pool.signer, tx) - pool.reserve(senderAddr, false) + pool.reserver.Release(senderAddr) } pool.all = newLookup() pool.priced = newPricedList(pool.all) @@ -1904,3 +1914,9 @@ func (pool *LegacyPool) Clear() { pool.queue = make(map[common.Address]*list) pool.pendingNonces = newNoncer(pool.currentState) } + +// HasPendingAuth returns a flag indicating whether there are pending +// authorizations from the specific address cached in the pool. +func (pool *LegacyPool) HasPendingAuth(addr common.Address) bool { + return pool.all.hasAuth(addr) +} diff --git a/core/txpool/legacypool/legacypool2_test.go b/core/txpool/legacypool/legacypool2_test.go index d55e85d74f..3f210e3d1b 100644 --- a/core/txpool/legacypool/legacypool2_test.go +++ b/core/txpool/legacypool/legacypool2_test.go @@ -86,7 +86,7 @@ func TestTransactionFutureAttack(t *testing.T) { config.GlobalQueue = 100 config.GlobalSlots = 100 pool := New(config, blockchain) - pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() fillPool(t, pool) pending, _ := pool.Stats() @@ -120,7 +120,7 @@ func TestTransactionFuture1559(t *testing.T) { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create a number of test accounts, fund them and make transactions @@ -153,7 +153,7 @@ func TestTransactionZAttack(t *testing.T) { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create a number of test accounts, fund them and make transactions fillPool(t, pool) @@ -224,7 +224,7 @@ func BenchmarkFutureAttack(b *testing.B) { config.GlobalQueue = 100 config.GlobalSlots = 100 pool := New(config, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() fillPool(b, pool) diff --git a/core/txpool/legacypool/legacypool_test.go b/core/txpool/legacypool/legacypool_test.go index 3f269bd69e..c47a655204 100644 --- a/core/txpool/legacypool/legacypool_test.go +++ b/core/txpool/legacypool/legacypool_test.go @@ -24,7 +24,6 @@ import ( "math/big" "math/rand" "slices" - "sync" "sync/atomic" "testing" "time" @@ -168,42 +167,21 @@ func pricedSetCodeTxWithAuth(nonce uint64, gaslimit uint64, gasFee, tip *uint256 }) } -func makeAddressReserver() txpool.AddressReserver { - var ( - reserved = make(map[common.Address]struct{}) - lock sync.Mutex - ) - return func(addr common.Address, reserve bool) error { - lock.Lock() - defer lock.Unlock() - - _, exists := reserved[addr] - if reserve { - if exists { - panic("already reserved") - } - reserved[addr] = struct{}{} - return nil - } - if !exists { - panic("not reserved") - } - delete(reserved, addr) - return nil - } -} - func setupPool() (*LegacyPool, *ecdsa.PrivateKey) { return setupPoolWithConfig(params.TestChainConfig) } +func newReserver() *txpool.Reserver { + return txpool.NewReservationTracker().NewHandle(42) +} + func setupPoolWithConfig(config *params.ChainConfig) (*LegacyPool, *ecdsa.PrivateKey) { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabaseForTesting()) blockchain := newTestBlockChain(config, 10000000, statedb, new(event.Feed)) key, _ := crypto.GenerateKey() pool := New(testTxPoolConfig, blockchain) - if err := pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()); err != nil { + if err := pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()); err != nil { panic(err) } // wait for the pool to initialize @@ -336,7 +314,7 @@ func TestStateChangeDuringReset(t *testing.T) { tx1 := transaction(1, 100000, key) pool := New(testTxPoolConfig, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() nonce := pool.Nonce(address) @@ -753,7 +731,7 @@ func TestPostponing(t *testing.T) { blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create two test accounts to produce different gap profiles with @@ -963,7 +941,7 @@ func TestQueueGlobalLimiting(t *testing.T) { config.GlobalQueue = config.AccountQueue*3 - 1 // reduce the queue limits to shorten test time (-1 to make it non divisible) pool := New(config, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create a number of test accounts and fund them (last one will be the local) @@ -1015,7 +993,7 @@ func TestQueueTimeLimiting(t *testing.T) { config.Lifetime = time.Second pool := New(config, blockchain) - pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create a test account to ensure remotes expire @@ -1176,7 +1154,7 @@ func TestPendingGlobalLimiting(t *testing.T) { config.GlobalSlots = config.AccountSlots * 10 pool := New(config, blockchain) - pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create a number of test accounts and fund them @@ -1275,7 +1253,7 @@ func TestCapClearsFromAll(t *testing.T) { config.GlobalSlots = 8 pool := New(config, blockchain) - pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create a number of test accounts and fund them @@ -1308,7 +1286,7 @@ func TestPendingMinimumAllowance(t *testing.T) { config.GlobalSlots = 1 pool := New(config, blockchain) - pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create a number of test accounts and fund them @@ -1352,7 +1330,7 @@ func TestRepricing(t *testing.T) { blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Keep track of transaction events to ensure all executables get announced @@ -1457,7 +1435,7 @@ func TestMinGasPriceEnforced(t *testing.T) { txPoolConfig := DefaultConfig txPoolConfig.NoLocals = true pool := New(txPoolConfig, blockchain) - pool.Init(txPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(txPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() key, _ := crypto.GenerateKey() @@ -1606,7 +1584,7 @@ func TestUnderpricing(t *testing.T) { config.GlobalQueue = 2 pool := New(config, blockchain) - pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Keep track of transaction events to ensure all executables get announced @@ -1696,7 +1674,7 @@ func TestStableUnderpricing(t *testing.T) { config.GlobalQueue = 0 pool := New(config, blockchain) - pool.Init(config.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(config.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Keep track of transaction events to ensure all executables get announced @@ -1899,7 +1877,7 @@ func TestDeduplication(t *testing.T) { blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create a test account to add transactions with @@ -1966,7 +1944,7 @@ func TestReplacement(t *testing.T) { blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Keep track of transaction events to ensure all executables get announced @@ -2157,7 +2135,7 @@ func TestStatusCheck(t *testing.T) { blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create the test accounts to check various transaction statuses with @@ -2230,7 +2208,7 @@ func TestSetCodeTransactions(t *testing.T) { blockchain := newTestBlockChain(params.MergedTestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create the test accounts @@ -2271,12 +2249,12 @@ func TestSetCodeTransactions(t *testing.T) { if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), keyA)); err != nil { t.Fatalf("%s: failed to add remote transaction: %v", name, err) } - if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyA)); !errors.Is(err, ErrInflightTxLimitReached) { - t.Fatalf("%s: error mismatch: want %v, have %v", name, ErrInflightTxLimitReached, err) + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyA)); !errors.Is(err, txpool.ErrInflightTxLimitReached) { + t.Fatalf("%s: error mismatch: want %v, have %v", name, txpool.ErrInflightTxLimitReached, err) } // Check gapped transaction again. - if err := pool.addRemoteSync(pricedTransaction(2, 100000, big.NewInt(1), keyA)); !errors.Is(err, ErrInflightTxLimitReached) { - t.Fatalf("%s: error mismatch: want %v, have %v", name, ErrInflightTxLimitReached, err) + if err := pool.addRemoteSync(pricedTransaction(2, 100000, big.NewInt(1), keyA)); !errors.Is(err, txpool.ErrInflightTxLimitReached) { + t.Fatalf("%s: error mismatch: want %v, have %v", name, txpool.ErrInflightTxLimitReached, err) } // Replace by fee. if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(10), keyA)); err != nil { @@ -2310,8 +2288,8 @@ func TestSetCodeTransactions(t *testing.T) { t.Fatalf("%s: failed to add with pending delegatio: %v", name, err) } // Also check gapped transaction is rejected. - if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyC)); !errors.Is(err, ErrInflightTxLimitReached) { - t.Fatalf("%s: error mismatch: want %v, have %v", name, ErrInflightTxLimitReached, err) + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyC)); !errors.Is(err, txpool.ErrInflightTxLimitReached) { + t.Fatalf("%s: error mismatch: want %v, have %v", name, txpool.ErrInflightTxLimitReached, err) } }, }, @@ -2386,7 +2364,7 @@ func TestSetCodeTransactions(t *testing.T) { if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1000), keyC)); err != nil { t.Fatalf("%s: failed to added single pooled for account with pending delegation: %v", name, err) } - if err, want := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1000), keyC)), ErrInflightTxLimitReached; !errors.Is(err, want) { + if err, want := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1000), keyC)), txpool.ErrInflightTxLimitReached; !errors.Is(err, want) { t.Fatalf("%s: error mismatch: want %v, have %v", name, want, err) } }, @@ -2454,7 +2432,7 @@ func TestSetCodeTransactionsReorg(t *testing.T) { blockchain := newTestBlockChain(params.MergedTestChainConfig, 1000000, statedb, new(event.Feed)) pool := New(testTxPoolConfig, blockchain) - pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), makeAddressReserver()) + pool.Init(testTxPoolConfig.PriceLimit, blockchain.CurrentBlock(), newReserver()) defer pool.Close() // Create the test accounts @@ -2489,8 +2467,8 @@ func TestSetCodeTransactionsReorg(t *testing.T) { t.Fatalf("failed to add with remote setcode transaction: %v", err) } // Try to add a transactions in - if err := pool.addRemoteSync(pricedTransaction(2, 100000, big.NewInt(1000), keyA)); !errors.Is(err, ErrInflightTxLimitReached) { - t.Fatalf("unexpected error %v, expecting %v", err, ErrInflightTxLimitReached) + if err := pool.addRemoteSync(pricedTransaction(2, 100000, big.NewInt(1000), keyA)); !errors.Is(err, txpool.ErrInflightTxLimitReached) { + t.Fatalf("unexpected error %v, expecting %v", err, txpool.ErrInflightTxLimitReached) } // Simulate the chain moving blockchain.statedb.SetNonce(addrA, 2, tracing.NonceChangeAuthorization) diff --git a/core/txpool/reserver.go b/core/txpool/reserver.go new file mode 100644 index 0000000000..76ead0f3bb --- /dev/null +++ b/core/txpool/reserver.go @@ -0,0 +1,124 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package txpool + +import ( + "errors" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +var ( + // reservationsGaugeName is the prefix of a per-subpool address reservation + // metric. + // + // This is mostly a sanity metric to ensure there's no bug that would make + // some subpool hog all the reservations due to mis-accounting. + reservationsGaugeName = "txpool/reservations" +) + +// ReservationTracker is a struct shared between different subpools. It is used to reserve +// the account and ensure that one address cannot initiate transactions, authorizations, +// and other state-changing behaviors in different pools at the same time. +type ReservationTracker struct { + accounts map[common.Address]int + lock sync.RWMutex +} + +// NewReservationTracker initializes the account reservation tracker. +func NewReservationTracker() *ReservationTracker { + return &ReservationTracker{ + accounts: make(map[common.Address]int), + } +} + +// NewHandle creates a named handle on the ReservationTracker. The handle +// identifies the subpool so ownership of reservations can be determined. +func (r *ReservationTracker) NewHandle(id int) *Reserver { + return &Reserver{r, id} +} + +// Reserver is a named handle on ReservationTracker. It is held by subpools to +// make reservations for accounts it is tracking. The id is used to determine +// which pool owns an address and disallows non-owners to hold or release +// addresses it doesn't own. +type Reserver struct { + tracker *ReservationTracker + id int +} + +// Hold attempts to reserve the specified account address for the given pool. +// Returns an error if the account is already reserved. +func (h *Reserver) Hold(addr common.Address) error { + h.tracker.lock.Lock() + defer h.tracker.lock.Unlock() + + // Double reservations are forbidden even from the same pool to + // avoid subtle bugs in the long term. + owner, exists := h.tracker.accounts[addr] + if exists { + if owner == h.id { + log.Error("pool attempted to reserve already-owned address", "address", addr) + return nil // Ignore fault to give the pool a chance to recover while the bug gets fixed + } + return ErrAlreadyReserved + } + h.tracker.accounts[addr] = h.id + if metrics.Enabled() { + m := fmt.Sprintf("%s/%d", reservationsGaugeName, h.id) + metrics.GetOrRegisterGauge(m, nil).Inc(1) + } + return nil +} + +// Release attempts to release the reservation for the specified account. +// Returns an error if the address is not reserved or is reserved by another pool. +func (h *Reserver) Release(addr common.Address) error { + h.tracker.lock.Lock() + defer h.tracker.lock.Unlock() + + // Ensure subpools only attempt to unreserve their own owned addresses, + // otherwise flag as a programming error. + owner, exists := h.tracker.accounts[addr] + if !exists { + log.Error("pool attempted to unreserve non-reserved address", "address", addr) + return errors.New("address not reserved") + } + if owner != h.id { + log.Error("pool attempted to unreserve non-owned address", "address", addr) + return errors.New("address not owned") + } + delete(h.tracker.accounts, addr) + if metrics.Enabled() { + m := fmt.Sprintf("%s/%d", reservationsGaugeName, h.id) + metrics.GetOrRegisterGauge(m, nil).Dec(1) + } + return nil +} + +// Has returns a flag indicating if the address has been reserved or not. +func (h *Reserver) Has(address common.Address) bool { + h.tracker.lock.RLock() + defer h.tracker.lock.RUnlock() + + _, exists := h.tracker.accounts[address] + return exists +} diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index f5cb852d8f..2cb5103875 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -67,10 +67,6 @@ type LazyResolver interface { Get(hash common.Hash) *types.Transaction } -// AddressReserver is passed by the main transaction pool to subpools, so they -// may request (and relinquish) exclusive access to certain addresses. -type AddressReserver func(addr common.Address, reserve bool) error - // PendingFilter is a collection of filter rules to allow retrieving a subset // of transactions for announcement or mining. // @@ -109,7 +105,7 @@ type SubPool interface { // These should not be passed as a constructor argument - nor should the pools // start by themselves - in order to keep multiple subpools in lockstep with // one another. - Init(gasTip uint64, head *types.Header, reserve AddressReserver) error + Init(gasTip uint64, head *types.Header, reserver *Reserver) error // Close terminates any background processing threads and releases any held // resources. diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 083aac92c6..47d83e03d4 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" ) @@ -43,15 +42,6 @@ const ( TxStatusIncluded ) -var ( - // reservationsGaugeName is the prefix of a per-subpool address reservation - // metric. - // - // This is mostly a sanity metric to ensure there's no bug that would make - // some subpool hog all the reservations due to mis-accounting. - reservationsGaugeName = "txpool/reservations" -) - // BlockChain defines the minimal set of methods needed to back a tx pool with // a chain. Exists to allow mocking the live chain out of tests. type BlockChain interface { @@ -81,9 +71,6 @@ type TxPool struct { stateLock sync.RWMutex // The lock for protecting state instance state *state.StateDB // Current state at the blockchain head - reservations map[common.Address]SubPool // Map with the account to pool reservations - reserveLock sync.Mutex // Lock protecting the account reservations - subs event.SubscriptionScope // Subscription scope to unsubscribe all on shutdown quit chan chan error // Quit channel to tear down the head updater term chan struct{} // Termination channel to detect a closed pool @@ -110,17 +97,17 @@ func New(gasTip uint64, chain BlockChain, subpools []SubPool) (*TxPool, error) { return nil, err } pool := &TxPool{ - subpools: subpools, - chain: chain, - signer: types.LatestSigner(chain.Config()), - state: statedb, - reservations: make(map[common.Address]SubPool), - quit: make(chan chan error), - term: make(chan struct{}), - sync: make(chan chan error), + subpools: subpools, + chain: chain, + signer: types.LatestSigner(chain.Config()), + state: statedb, + quit: make(chan chan error), + term: make(chan struct{}), + sync: make(chan chan error), } + reserver := NewReservationTracker() for i, subpool := range subpools { - if err := subpool.Init(gasTip, head, pool.reserver(i, subpool)); err != nil { + if err := subpool.Init(gasTip, head, reserver.NewHandle(i)); err != nil { for j := i - 1; j >= 0; j-- { subpools[j].Close() } @@ -131,52 +118,6 @@ func New(gasTip uint64, chain BlockChain, subpools []SubPool) (*TxPool, error) { return pool, nil } -// reserver is a method to create an address reservation callback to exclusively -// assign/deassign addresses to/from subpools. This can ensure that at any point -// in time, only a single subpool is able to manage an account, avoiding cross -// subpool eviction issues and nonce conflicts. -func (p *TxPool) reserver(id int, subpool SubPool) AddressReserver { - return func(addr common.Address, reserve bool) error { - p.reserveLock.Lock() - defer p.reserveLock.Unlock() - - owner, exists := p.reservations[addr] - if reserve { - // Double reservations are forbidden even from the same pool to - // avoid subtle bugs in the long term. - if exists { - if owner == subpool { - log.Error("pool attempted to reserve already-owned address", "address", addr) - return nil // Ignore fault to give the pool a chance to recover while the bug gets fixed - } - return ErrAlreadyReserved - } - p.reservations[addr] = subpool - if metrics.Enabled() { - m := fmt.Sprintf("%s/%d", reservationsGaugeName, id) - metrics.GetOrRegisterGauge(m, nil).Inc(1) - } - return nil - } - // Ensure subpools only attempt to unreserve their own owned addresses, - // otherwise flag as a programming error. - if !exists { - log.Error("pool attempted to unreserve non-reserved address", "address", addr) - return errors.New("address not reserved") - } - if subpool != owner { - log.Error("pool attempted to unreserve non-owned address", "address", addr) - return errors.New("address not owned") - } - delete(p.reservations, addr) - if metrics.Enabled() { - m := fmt.Sprintf("%s/%d", reservationsGaugeName, id) - metrics.GetOrRegisterGauge(m, nil).Dec(1) - } - return nil - } -} - // Close terminates the transaction pool and all its subpools. func (p *TxPool) Close() error { var errs []error diff --git a/eth/backend.go b/eth/backend.go index 79759710b6..6716a77562 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -260,16 +260,16 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { eth.filterMaps = filtermaps.NewFilterMaps(chainDb, chainView, historyCutoff, finalBlock, filtermaps.DefaultParams, fmConfig) eth.closeFilterMaps = make(chan chan struct{}) - if config.BlobPool.Datadir != "" { - config.BlobPool.Datadir = stack.ResolvePath(config.BlobPool.Datadir) - } - blobPool := blobpool.New(config.BlobPool, eth.blockchain) - if config.TxPool.Journal != "" { config.TxPool.Journal = stack.ResolvePath(config.TxPool.Journal) } legacyPool := legacypool.New(config.TxPool, eth.blockchain) + if config.BlobPool.Datadir != "" { + config.BlobPool.Datadir = stack.ResolvePath(config.BlobPool.Datadir) + } + blobPool := blobpool.New(config.BlobPool, eth.blockchain, legacyPool.HasPendingAuth) + eth.txPool, err = txpool.New(config.TxPool.PriceLimit, eth.blockchain, []txpool.SubPool{legacyPool, blobPool}) if err != nil { return nil, err diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index bbbd888701..fa031d9899 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -135,7 +135,7 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, cancun bool, generat storage, _ := os.MkdirTemp("", "blobpool-") defer os.RemoveAll(storage) - blobPool := blobpool.New(blobpool.Config{Datadir: storage}, chain) + blobPool := blobpool.New(blobpool.Config{Datadir: storage}, chain, nil) legacyPool := legacypool.New(txconfig, chain) txpool, _ := txpool.New(txconfig.PriceLimit, chain, []txpool.SubPool{legacyPool, blobPool}) From 5cc9137c9cec8a977b845a025d3deced9457fd48 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 8 Apr 2025 19:57:45 +0200 Subject: [PATCH 25/35] core/vm: optimize push2 opcode (#31267) During my benchmarks on Holesky, around 10% of all CPU time was spent in PUSH2 ``` ROUTINE ======================== github.com/ethereum/go-ethereum/core/vm.newFrontierInstructionSet.makePush.func1 in github.com/ethereum/go-ethereum/core/vm/instructions.go 16.38s 20.35s (flat, cum) 10.31% of Total 740ms 740ms 976: return func(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { . . 977: var ( 40ms 40ms 978: codeLen = len(scope.Contract.Code) 970ms 970ms 979: start = min(codeLen, int(*pc+1)) 200ms 200ms 980: end = min(codeLen, start+pushByteSize) . . 981: ) 670ms 2.39s 982: a := new(uint256.Int).SetBytes(scope.Contract.Code[start:end]) . . 983: . . 984: // Missing bytes: pushByteSize - len(pushData) 410ms 410ms 985: if missing := pushByteSize - (end - start); missing > 0 { . . 986: a.Lsh(a, uint(8*missing)) . . 987: } 12.69s 14.94s 988: scope.Stack.push2(*a) 10ms 10ms 989: *pc += size 650ms 650ms 990: return nil, nil . . 991: } . . 992:} ``` Which is quite crazy. We have a handwritten encoder for PUSH1 already, this PR adds one for PUSH2. PUSH2 is the second most used opcode as shown here: https://gist.github.com/shemnon/fb9b292a103abb02d98d64df6fbd35c8 since it is used by solidity quite significantly. Its used ~20 times as much as PUSH20 and PUSH32. # Benchmarks ``` BenchmarkPush/makePush-14 94196547 12.27 ns/op 0 B/op 0 allocs/op BenchmarkPush/push-14 429976924 2.829 ns/op 0 B/op 0 allocs/op ``` --------- Co-authored-by: jwasinger --- core/vm/instructions.go | 17 +++++++++++++++++ core/vm/jump_table.go | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 1785ffc139..0b3b1d1569 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -971,6 +971,23 @@ func opPush1(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]by return nil, nil } +// opPush2 is a specialized version of pushN +func opPush2(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + var ( + codeLen = uint64(len(scope.Contract.Code)) + integer = new(uint256.Int) + ) + if *pc+2 < codeLen { + scope.Stack.push(integer.SetBytes2(scope.Contract.Code[*pc+1 : *pc+3])) + } else if *pc+1 < codeLen { + scope.Stack.push(integer.SetUint64(uint64(scope.Contract.Code[*pc+1]) << 8)) + } else { + scope.Stack.push(integer.Clear()) + } + *pc += 2 + return nil, nil +} + // make push instruction function func makePush(size uint64, pushByteSize int) executionFunc { return func(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index 6610fa7f9a..ee811b447e 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -631,7 +631,7 @@ func newFrontierInstructionSet() JumpTable { maxStack: maxStack(0, 1), }, PUSH2: { - execute: makePush(2, 2), + execute: opPush2, constantGas: GasFastestStep, minStack: minStack(0, 1), maxStack: maxStack(0, 1), From a7f24c26c0a6a73357d76adf64d98f343bafd0e5 Mon Sep 17 00:00:00 2001 From: Csaba Kiraly Date: Wed, 9 Apr 2025 11:28:29 +0200 Subject: [PATCH 26/35] p2p/nat: fix UPnP port reset (#31566) Make UPnP more robust - Once a random port was mapped, we try to stick to it even if a UPnP refresh fails. Previously we were immediately moving back to try the default port, leading to frequent ENR changes. - We were deleting port mappings before refresh as a possible workaround. This created issues in some UPnP servers. The UPnP (and PMP) specification is explicit about the refresh requirements, and delete is clearly not needed (see https://github.com/ethereum/go-ethereum/pull/30265#issuecomment-2766987859). From now on we only delete when closing. - We were trying to add port mappings only once, and then moved on to random ports. Now we insist a bit more, so that a simple failed request won't lead to ENR changes. Fixes https://github.com/ethereum/go-ethereum/issues/31418 --------- Signed-off-by: Csaba Kiraly Co-authored-by: Felix Lange --- p2p/nat/natupnp.go | 49 +++++++++++++++++++++++++-------------- p2p/server_nat.go | 57 ++++++++++++++++++++++++++++++++-------------- 2 files changed, 72 insertions(+), 34 deletions(-) diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index ed00b8eeb6..d79677db55 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -26,6 +26,7 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/log" "github.com/huin/goupnp" "github.com/huin/goupnp/dcps/internetgateway1" "github.com/huin/goupnp/dcps/internetgateway2" @@ -34,6 +35,8 @@ import ( const ( soapRequestTimeout = 3 * time.Second rateLimit = 200 * time.Millisecond + retryCount = 3 // number of retries after a failed AddPortMapping + randomCount = 3 // number of random ports to try ) type upnp struct { @@ -89,42 +92,43 @@ func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, li if extport == 0 { extport = intport - } else { - // Only delete port mapping if the external port was already used by geth. - n.DeleteMapping(protocol, extport, intport) } // Try to add port mapping, preferring the specified external port. - err = n.withRateLimit(func() error { - p, err := n.addAnyPortMapping(protocol, extport, intport, ip, desc, lifetimeS) - if err == nil { - extport = int(p) - } - return err - }) - return uint16(extport), err + return n.addAnyPortMapping(protocol, extport, intport, ip, desc, lifetimeS) } // addAnyPortMapping tries to add a port mapping with the specified external port. // If the external port is already in use, it will try to assign another port. func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.IP, desc string, lifetimeS uint32) (uint16, error) { if client, ok := n.client.(*internetgateway2.WANIPConnection2); ok { - return client.AddAnyPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + return n.portWithRateLimit(func() (uint16, error) { + return client.AddAnyPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + }) } // For IGDv1 and v1 services we should first try to add with extport. - err := n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) - if err == nil { - return uint16(extport), nil + for i := 0; i < retryCount+1; i++ { + err := n.withRateLimit(func() error { + return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + }) + if err == nil { + return uint16(extport), nil + } + log.Debug("Failed to add port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", err) } // If above fails, we retry with a random port. // We retry several times because of possible port conflicts. - for i := 0; i < 3; i++ { + var err error + for i := 0; i < randomCount; i++ { extport = n.randomPort() - err := n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + err := n.withRateLimit(func() error { + return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + }) if err == nil { return uint16(extport), nil } + log.Debug("Failed to add random port mapping", "protocol", protocol, "extport", extport, "intport", intport, "err", err) } return 0, err } @@ -169,6 +173,17 @@ func (n *upnp) String() string { return "UPNP " + n.service } +func (n *upnp) portWithRateLimit(pfn func() (uint16, error)) (uint16, error) { + var port uint16 + var err error + fn := func() error { + port, err = pfn() + return err + } + n.withRateLimit(fn) + return port, err +} + func (n *upnp) withRateLimit(fn func() error) error { n.mu.Lock() defer n.mu.Unlock() diff --git a/p2p/server_nat.go b/p2p/server_nat.go index 5830f950e1..298c454a4a 100644 --- a/p2p/server_nat.go +++ b/p2p/server_nat.go @@ -31,12 +31,14 @@ const ( portMapRefreshInterval = 8 * time.Minute portMapRetryInterval = 5 * time.Minute extipRetryInterval = 2 * time.Minute + maxRetries = 5 // max number of failed attempts to refresh the mapping ) type portMapping struct { protocol string name string port int + retries int // number of failed attempts to refresh the mapping // for use by the portMappingLoop goroutine: extPort int // the mapped port returned by the NAT interface @@ -154,28 +156,49 @@ func (srv *Server) portMappingLoop() { log.Trace("Attempting port mapping") p, err := srv.NAT.AddMapping(m.protocol, m.extPort, m.port, m.name, portMapDuration) if err != nil { - log.Debug("Couldn't add port mapping", "err", err) - m.extPort = 0 + // Failed to add or refresh port mapping. + if m.extPort == 0 { + log.Debug("Couldn't add port mapping", "err", err) + } else { + // Failed refresh. Since UPnP implementation are often buggy, + // and lifetime is larger than the retry interval, this does not + // mean we lost our existing mapping. We do not reset the external + // port, as it is still our best chance, but we do retry soon. + // We could check the error code, but UPnP implementations are buggy. + log.Debug("Couldn't refresh port mapping", "err", err) + m.retries++ + if m.retries > maxRetries { + m.retries = 0 + err := srv.NAT.DeleteMapping(m.protocol, m.extPort, m.port) + log.Debug("Couldn't refresh port mapping, trying to delete it:", "err", err) + m.extPort = 0 + } + } m.nextTime = srv.clock.Now().Add(portMapRetryInterval) + // Note ENR is not updated here, i.e. we keep the last port. continue } - // It was mapped! - m.extPort = int(p) - m.nextTime = srv.clock.Now().Add(portMapRefreshInterval) - log = newLogger(m.protocol, m.extPort, m.port) - if m.port != m.extPort { - log.Info("NAT mapped alternative port") - } else { - log.Info("NAT mapped port") - } - // Update port in local ENR. - switch m.protocol { - case "TCP": - srv.localnode.Set(enr.TCP(m.extPort)) - case "UDP": - srv.localnode.SetFallbackUDP(m.extPort) + // It was mapped! + m.retries = 0 + log = newLogger(m.protocol, int(p), m.port) + if int(p) != m.extPort { + m.extPort = int(p) + if m.port != m.extPort { + log.Info("NAT mapped alternative port") + } else { + log.Info("NAT mapped port") + } + + // Update port in local ENR. + switch m.protocol { + case "TCP": + srv.localnode.Set(enr.TCP(m.extPort)) + case "UDP": + srv.localnode.SetFallbackUDP(m.extPort) + } } + m.nextTime = srv.clock.Now().Add(portMapRefreshInterval) } } } From 60b922fd529237ac5070a1750c93a4c132ed32fb Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Wed, 9 Apr 2025 14:19:28 -0600 Subject: [PATCH 27/35] core/txpool: add notice to` Clear` that is not meant for production code (#31567) The `Sync(..)` function explicitly says not to rely on in production code, but it is used in `Clear(..)` so should add a similar mention. --- core/txpool/blobpool/blobpool.go | 6 ++++++ core/txpool/legacypool/legacypool.go | 7 +++++-- core/txpool/txpool.go | 11 +++++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index b1966905a8..1770066a8d 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1349,6 +1349,9 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844. // Add inserts a set of blob transactions into the pool if they pass validation (both // consensus validity and pool restrictions). +// +// Note, if sync is set the method will block until all internal maintenance +// related to the add is finished. Only use this during tests for determinism. func (p *BlobPool) Add(txs []*types.Transaction, sync bool) []error { var ( adds = make([]*types.Transaction, 0, len(txs)) @@ -1792,6 +1795,9 @@ func (p *BlobPool) Status(hash common.Hash) txpool.TxStatus { // Clear implements txpool.SubPool, removing all tracked transactions // from the blob pool and persistent store. +// +// Note, do not use this in production / live code. In live code, the pool is +// meant to reset on a separate thread to avoid DoS vectors. func (p *BlobPool) Clear() { p.lock.Lock() defer p.lock.Unlock() diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 278ad0791f..75dc4a8461 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -927,8 +927,8 @@ func (pool *LegacyPool) addRemoteSync(tx *types.Transaction) error { // Add enqueues a batch of transactions into the pool if they are valid. // -// If sync is set, the method will block until all internal maintenance related -// to the add is finished. Only use this during tests for determinism! +// Note, if sync is set the method will block until all internal maintenance +// related to the add is finished. Only use this during tests for determinism. func (pool *LegacyPool) Add(txs []*types.Transaction, sync bool) []error { // Filter out known ones without obtaining the pool lock or recovering signatures var ( @@ -1886,6 +1886,9 @@ func numSlots(tx *types.Transaction) int { // Clear implements txpool.SubPool, removing all tracked txs from the pool // and rotating the journal. +// +// Note, do not use this in production / live code. In live code, the pool is +// meant to reset on a separate thread to avoid DoS vectors. func (pool *LegacyPool) Clear() { pool.mu.Lock() defer pool.mu.Unlock() diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 47d83e03d4..2ed38772ce 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -350,6 +350,9 @@ func (p *TxPool) ValidateTxBasics(tx *types.Transaction) error { // Add enqueues a batch of transactions into the pool if they are valid. Due // to the large transaction churn, add may postpone fully integrating the tx // to a later point to batch multiple ones together. +// +// Note, if sync is set the method will block until all internal maintenance +// related to the add is finished. Only use this during tests for determinism. func (p *TxPool) Add(txs []*types.Transaction, sync bool) []error { // Split the input transactions between the subpools. It shouldn't really // happen that we receive merged batches, but better graceful than strange @@ -503,8 +506,8 @@ func (p *TxPool) Status(hash common.Hash) TxStatus { // internal background reset operations. This method will run an explicit reset // operation to ensure the pool stabilises, thus avoiding flakey behavior. // -// Note, do not use this in production / live code. In live code, the pool is -// meant to reset on a separate thread to avoid DoS vectors. +// Note, this method is only used for testing and is susceptible to DoS vectors. +// In production code, the pool is meant to reset on a separate thread. func (p *TxPool) Sync() error { sync := make(chan error) select { @@ -516,6 +519,10 @@ func (p *TxPool) Sync() error { } // Clear removes all tracked txs from the subpools. +// +// Note, this method invokes Sync() and is only used for testing, because it is +// susceptible to DoS vectors. In production code, the pool is meant to reset on +// a separate thread. func (p *TxPool) Clear() { // Invoke Sync to ensure that txs pending addition don't get added to the pool after // the subpools are subsequently cleared From 0c2ad076734e87fcc008e4718ba7b1f4f881b1fa Mon Sep 17 00:00:00 2001 From: Luis Ayuso Date: Thu, 10 Apr 2025 03:11:24 +0200 Subject: [PATCH 28/35] core/txpool: allow tx and authority regardless of admission order (#31373) This PR proposes a change to the authorizations' validation introduced in commit cdb66c8. These changes make the expected behavior independent of the order of admission of authorizations, improving the predictability of the resulting state and the usability of the system with it. The current implementation behavior is dependent on the transaction submission order: This issue is related to authorities and the sender of a transaction, and can be reproduced respecting the normal nonce rules. The issue can be reproduced by the two following cases: **First case** - Given an empty pool. - Submit transaction `{ from: B, auths [ A ] }`: is accepted. - Submit transaction `{ from: A }`: Is accepted: it becomes the one in-flight transaction allowed. **Second case** - Given an empty pool. - Submit transaction `{ from: A }`: is accepted - Submit transaction `{ from: B, auths [ A ] }`: is rejected since there is already a queued/pending transaction from A. The expected behavior is that both sequences of events would lead to the same sets of accepted and rejected transactions. **Proposed changes** The queued/pending transactions issued from any authority of the transaction being validated have to be counted, allowing one transaction from accounts submitting an authorization. - Notice that the expected behavior was explicitly forbidden in the case "reject-delegation-from-pending-account", I believe that this behavior conflicts to the definition of the limitation, and it is removed in this PR. The expected behavior is tested in "accept-authorization-from-sender-of-one-inflight-tx". - Replacement tests have been separated to improve readability of the acceptance test. - The test "allow-more-than-one-tx-from-replaced-authority" has been extended with one extra transaction, since the system would always have accepted one transaction (but not two). - The test "accept-one-inflight-tx-of-delegated-account" is extended to clean-up state, avoiding leaking the delegation used into the other tests. Additionally, replacement check is removed to be tested in its own test case. **Expected behavior** The expected behavior of the authorizations' validation shall be as follows: ![image](https://github.com/user-attachments/assets/dbde7a1f-9679-4691-94eb-c197a0cbb823) Notice that replacement shall be allowed, and behavior shall remain coherent with the table, according to the replaced transaction. --------- Co-authored-by: lightclient --- core/txpool/blobpool/blobpool.go | 4 +- core/txpool/blobpool/blobpool_test.go | 43 ++++- core/txpool/legacypool/legacypool.go | 28 ++- core/txpool/legacypool/legacypool_test.go | 198 +++++++++++++++++----- core/txpool/reserver.go | 42 +++-- core/txpool/subpool.go | 2 +- 6 files changed, 242 insertions(+), 75 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 1770066a8d..12a4133b40 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -299,7 +299,7 @@ func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transac // and leading up to the first no-change. type BlobPool struct { config Config // Pool configuration - reserver *txpool.Reserver // Address reserver to ensure exclusivity across subpools + reserver txpool.Reserver // Address reserver to ensure exclusivity across subpools hasPendingAuth func(common.Address) bool // Determine whether the specified address has a pending 7702-auth store billy.Database // Persistent data store for the tx metadata and blobs @@ -355,7 +355,7 @@ func (p *BlobPool) Filter(tx *types.Transaction) bool { // Init sets the gas price needed to keep a transaction in the pool and the chain // head to allow balance / nonce checks. The transaction journal will be loaded // from disk and filtered based on the provided starting settings. -func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver *txpool.Reserver) error { +func (p *BlobPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reserver) error { p.reserver = reserver var ( diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 4dfba3b52b..76d21a0c9e 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -26,6 +26,7 @@ import ( "math/big" "os" "path/filepath" + "sync" "testing" "github.com/ethereum/go-ethereum/common" @@ -167,6 +168,44 @@ func (bc *testBlockChain) StateAt(common.Hash) (*state.StateDB, error) { return bc.statedb, nil } +// reserver is a utility struct to sanity check that accounts are +// properly reserved by the blobpool (no duplicate reserves or unreserves). +type reserver struct { + accounts map[common.Address]struct{} + lock sync.RWMutex +} + +func newReserver() txpool.Reserver { + return &reserver{accounts: make(map[common.Address]struct{})} +} + +func (r *reserver) Hold(addr common.Address) error { + r.lock.Lock() + defer r.lock.Unlock() + if _, exists := r.accounts[addr]; exists { + panic("already reserved") + } + r.accounts[addr] = struct{}{} + return nil +} + +func (r *reserver) Release(addr common.Address) error { + r.lock.Lock() + defer r.lock.Unlock() + if _, exists := r.accounts[addr]; !exists { + panic("not reserved") + } + delete(r.accounts, addr) + return nil +} + +func (r *reserver) Has(address common.Address) bool { + r.lock.RLock() + defer r.lock.RUnlock() + _, exists := r.accounts[address] + return exists +} + // makeTx is a utility method to construct a random blob transaction and sign it // with a valid key, only setting the interesting fields from the perspective of // the blob pool. @@ -405,10 +444,6 @@ func verifyBlobRetrievals(t *testing.T, pool *BlobPool) { } } -func newReserver() *txpool.Reserver { - return txpool.NewReservationTracker().NewHandle(42) -} - // Tests that transactions can be loaded from disk on startup and that they are // correctly discarded if invalid. // diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 75dc4a8461..04f1a2234c 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -237,7 +237,7 @@ type LegacyPool struct { currentHead atomic.Pointer[types.Header] // Current head of the blockchain currentState *state.StateDB // Current state in the blockchain head pendingNonces *noncer // Pending state tracking virtual nonces - reserver *txpool.Reserver // Address reserver to ensure exclusivity across subpools + reserver txpool.Reserver // Address reserver to ensure exclusivity across subpools pending map[common.Address]*list // All currently processable transactions queue map[common.Address]*list // Queued but non-processable transactions @@ -302,7 +302,7 @@ func (pool *LegacyPool) Filter(tx *types.Transaction) bool { // Init sets the gas price needed to keep a transaction in the pool and the chain // head to allow balance / nonce checks. The internal // goroutines will be spun up and the pool deemed operational afterwards. -func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserver *txpool.Reserver) error { +func (pool *LegacyPool) Init(gasTip uint64, head *types.Header, reserver txpool.Reserver) error { // Set the address reserver to request exclusive access to pooled accounts pool.reserver = reserver @@ -640,11 +640,18 @@ func (pool *LegacyPool) validateAuth(tx *types.Transaction) error { if err := pool.checkDelegationLimit(tx); err != nil { return err } - // Authorities must not conflict with any pending or queued transactions, - // nor with addresses that have already been reserved. + // For symmetry, allow at most one in-flight tx for any authority with a + // pending transaction. if auths := tx.SetCodeAuthorities(); len(auths) > 0 { for _, auth := range auths { - if pool.pending[auth] != nil || pool.queue[auth] != nil { + var count int + if pending := pool.pending[auth]; pending != nil { + count += pending.Len() + } + if queue := pool.queue[auth]; queue != nil { + count += queue.Len() + } + if count > 1 { return ErrAuthorityReserved } // Because there is no exclusive lock held between different subpools @@ -1907,9 +1914,14 @@ func (pool *LegacyPool) Clear() { // The transaction addition may attempt to reserve the sender addr which // can't happen until Clear releases the reservation lock. Clear cannot // acquire the subpool lock until the transaction addition is completed. - for _, tx := range pool.all.txs { - senderAddr, _ := types.Sender(pool.signer, tx) - pool.reserver.Release(senderAddr) + + for addr := range pool.pending { + if _, ok := pool.queue[addr]; !ok { + pool.reserver.Release(addr) + } + } + for addr := range pool.queue { + pool.reserver.Release(addr) } pool.all = newLookup() pool.priced = newPricedList(pool.all) diff --git a/core/txpool/legacypool/legacypool_test.go b/core/txpool/legacypool/legacypool_test.go index c47a655204..bb1323a7d1 100644 --- a/core/txpool/legacypool/legacypool_test.go +++ b/core/txpool/legacypool/legacypool_test.go @@ -24,6 +24,7 @@ import ( "math/big" "math/rand" "slices" + "sync" "sync/atomic" "testing" "time" @@ -171,8 +172,39 @@ func setupPool() (*LegacyPool, *ecdsa.PrivateKey) { return setupPoolWithConfig(params.TestChainConfig) } -func newReserver() *txpool.Reserver { - return txpool.NewReservationTracker().NewHandle(42) +// reserver is a utility struct to sanity check that accounts are +// properly reserved by the blobpool (no duplicate reserves or unreserves). +type reserver struct { + accounts map[common.Address]struct{} + lock sync.RWMutex +} + +func newReserver() txpool.Reserver { + return &reserver{accounts: make(map[common.Address]struct{})} +} + +func (r *reserver) Hold(addr common.Address) error { + r.lock.Lock() + defer r.lock.Unlock() + if _, exists := r.accounts[addr]; exists { + panic("already reserved") + } + r.accounts[addr] = struct{}{} + return nil +} + +func (r *reserver) Release(addr common.Address) error { + r.lock.Lock() + defer r.lock.Unlock() + if _, exists := r.accounts[addr]; !exists { + panic("not reserved") + } + delete(r.accounts, addr) + return nil +} + +func (r *reserver) Has(address common.Address) bool { + return false // reserver only supports a single pool } func setupPoolWithConfig(config *params.ChainConfig) (*LegacyPool, *ecdsa.PrivateKey) { @@ -2232,9 +2264,8 @@ func TestSetCodeTransactions(t *testing.T) { }{ { // Check that only one in-flight transaction is allowed for accounts - // with delegation set. Also verify the accepted transaction can be - // replaced by fee. - name: "only-one-in-flight", + // with delegation set. + name: "accept-one-inflight-tx-of-delegated-account", pending: 1, run: func(name string) { aa := common.Address{0xaa, 0xaa} @@ -2249,6 +2280,7 @@ func TestSetCodeTransactions(t *testing.T) { if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), keyA)); err != nil { t.Fatalf("%s: failed to add remote transaction: %v", name, err) } + // Second and further transactions shall be rejected if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyA)); !errors.Is(err, txpool.ErrInflightTxLimitReached) { t.Fatalf("%s: error mismatch: want %v, have %v", name, txpool.ErrInflightTxLimitReached, err) } @@ -2260,6 +2292,70 @@ func TestSetCodeTransactions(t *testing.T) { if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(10), keyA)); err != nil { t.Fatalf("%s: failed to replace with remote transaction: %v", name, err) } + + // Reset the delegation, avoid leaking state into the other tests + statedb.SetCode(addrA, nil) + }, + }, + { + // This test is analogous to the previous one, but the delegation is pending + // instead of set. + name: "allow-one-tx-from-pooled-delegation", + pending: 2, + run: func(name string) { + // Create a pending delegation request from B. + if err := pool.addRemoteSync(setCodeTx(0, keyA, []unsignedAuth{{0, keyB}})); err != nil { + t.Fatalf("%s: failed to add with remote setcode transaction: %v", name, err) + } + // First transaction from B is accepted. + if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), keyB)); err != nil { + t.Fatalf("%s: failed to add remote transaction: %v", name, err) + } + // Second transaction fails due to limit. + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyB)); !errors.Is(err, txpool.ErrInflightTxLimitReached) { + t.Fatalf("%s: error mismatch: want %v, have %v", name, txpool.ErrInflightTxLimitReached, err) + } + // Replace by fee for first transaction from B works. + if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(2), keyB)); err != nil { + t.Fatalf("%s: failed to add remote transaction: %v", name, err) + } + }, + }, + { + // This is the symmetric case of the previous one, where the delegation request + // is received after the transaction. The resulting state shall be the same. + name: "accept-authorization-from-sender-of-one-inflight-tx", + pending: 2, + run: func(name string) { + // The first in-flight transaction is accepted. + if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), keyB)); err != nil { + t.Fatalf("%s: failed to add with pending delegation: %v", name, err) + } + // Delegation is accepted. + if err := pool.addRemoteSync(setCodeTx(0, keyA, []unsignedAuth{{0, keyB}})); err != nil { + t.Fatalf("%s: failed to add remote transaction: %v", name, err) + } + // The second in-flight transaction is rejected. + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyB)); !errors.Is(err, txpool.ErrInflightTxLimitReached) { + t.Fatalf("%s: error mismatch: want %v, have %v", name, txpool.ErrInflightTxLimitReached, err) + } + }, + }, + { + name: "reject-authorization-from-sender-with-more-than-one-inflight-tx", + pending: 2, + run: func(name string) { + // Submit two transactions. + if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), keyB)); err != nil { + t.Fatalf("%s: failed to add with pending delegation: %v", name, err) + } + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyB)); err != nil { + t.Fatalf("%s: failed to add with pending delegation: %v", name, err) + } + // Delegation rejected since two txs are already in-flight. + if err := pool.addRemoteSync(setCodeTx(0, keyA, []unsignedAuth{{0, keyB}})); !errors.Is(err, ErrAuthorityReserved) { + t.Fatalf("%s: error mismatch: want %v, have %v", name, ErrAuthorityReserved, err) + } }, }, { @@ -2267,7 +2363,7 @@ func TestSetCodeTransactions(t *testing.T) { pending: 2, run: func(name string) { // Send two transactions where the first has no conflicting delegations and - // the second should be allowed despite conflicting with the authorities in 1). + // the second should be allowed despite conflicting with the authorities in the first. if err := pool.addRemoteSync(setCodeTx(0, keyA, []unsignedAuth{{1, keyC}})); err != nil { t.Fatalf("%s: failed to add with remote setcode transaction: %v", name, err) } @@ -2276,28 +2372,10 @@ func TestSetCodeTransactions(t *testing.T) { } }, }, - { - name: "allow-one-tx-from-pooled-delegation", - pending: 2, - run: func(name string) { - // Verify C cannot originate another transaction when it has a pooled delegation. - if err := pool.addRemoteSync(setCodeTx(0, keyA, []unsignedAuth{{0, keyC}})); err != nil { - t.Fatalf("%s: failed to add with remote setcode transaction: %v", name, err) - } - if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), keyC)); err != nil { - t.Fatalf("%s: failed to add with pending delegatio: %v", name, err) - } - // Also check gapped transaction is rejected. - if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyC)); !errors.Is(err, txpool.ErrInflightTxLimitReached) { - t.Fatalf("%s: error mismatch: want %v, have %v", name, txpool.ErrInflightTxLimitReached, err) - } - }, - }, { name: "replace-by-fee-setcode-tx", pending: 1, run: func(name string) { - // 4. Fee bump the setcode tx send. if err := pool.addRemoteSync(setCodeTx(0, keyB, []unsignedAuth{{1, keyC}})); err != nil { t.Fatalf("%s: failed to add with remote setcode transaction: %v", name, err) } @@ -2307,44 +2385,85 @@ func TestSetCodeTransactions(t *testing.T) { }, }, { - name: "allow-tx-from-replaced-authority", - pending: 2, + name: "allow-more-than-one-tx-from-replaced-authority", + pending: 3, run: func(name string) { - // Fee bump with a different auth list. Make sure that unlocks the authorities. + // Send transaction from A with B as an authority. if err := pool.addRemoteSync(pricedSetCodeTx(0, 250000, uint256.NewInt(10), uint256.NewInt(3), keyA, []unsignedAuth{{0, keyB}})); err != nil { t.Fatalf("%s: failed to add with remote setcode transaction: %v", name, err) } + // Replace transaction with another having C as an authority. if err := pool.addRemoteSync(pricedSetCodeTx(0, 250000, uint256.NewInt(3000), uint256.NewInt(300), keyA, []unsignedAuth{{0, keyC}})); err != nil { t.Fatalf("%s: failed to add with remote setcode transaction: %v", name, err) } - // Now send a regular tx from B. + // B should not be considred as having an in-flight delegation, so + // should allow more than one pooled transaction. if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(10), keyB)); err != nil { t.Fatalf("%s: failed to replace with remote transaction: %v", name, err) } + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(10), keyB)); err != nil { + t.Fatalf("%s: failed to replace with remote transaction: %v", name, err) + } }, }, { + // This test is analogous to the previous one, but the the replaced + // transaction is self-sponsored. name: "allow-tx-from-replaced-self-sponsor-authority", - pending: 2, + pending: 3, run: func(name string) { - // + // Send transaction from A with A as an authority. if err := pool.addRemoteSync(pricedSetCodeTx(0, 250000, uint256.NewInt(10), uint256.NewInt(3), keyA, []unsignedAuth{{0, keyA}})); err != nil { t.Fatalf("%s: failed to add with remote setcode transaction: %v", name, err) } + // Replace transaction with a transaction with B as an authority. if err := pool.addRemoteSync(pricedSetCodeTx(0, 250000, uint256.NewInt(30), uint256.NewInt(30), keyA, []unsignedAuth{{0, keyB}})); err != nil { t.Fatalf("%s: failed to add with remote setcode transaction: %v", name, err) } - // Now send a regular tx from keyA. - if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1000), keyA)); err != nil { + // The one in-flight transaction limit from A no longer applies, so we + // can stack a second transaction for the account. + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1000), keyA)); err != nil { t.Fatalf("%s: failed to replace with remote transaction: %v", name, err) } - // Make sure we can still send from keyB. + // B should still be able to send transactions. if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1000), keyB)); err != nil { t.Fatalf("%s: failed to replace with remote transaction: %v", name, err) } + // However B still has the limitation to one in-flight transaction. + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyB)); !errors.Is(err, txpool.ErrInflightTxLimitReached) { + t.Fatalf("%s: error mismatch: want %v, have %v", name, txpool.ErrInflightTxLimitReached, err) + } }, }, { + name: "replacements-respect-inflight-tx-count", + pending: 2, + run: func(name string) { + // Send transaction from A with B as an authority. + if err := pool.addRemoteSync(pricedSetCodeTx(0, 250000, uint256.NewInt(10), uint256.NewInt(3), keyA, []unsignedAuth{{0, keyB}})); err != nil { + t.Fatalf("%s: failed to add with remote setcode transaction: %v", name, err) + } + // Send two transactions from B. Only the first should be accepted due + // to in-flight limit. + if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), keyB)); err != nil { + t.Fatalf("%s: failed to add remote transaction: %v", name, err) + } + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyB)); !errors.Is(err, txpool.ErrInflightTxLimitReached) { + t.Fatalf("%s: error mismatch: want %v, have %v", name, txpool.ErrInflightTxLimitReached, err) + } + // Replace the in-flight transaction from B. + if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(30), keyB)); err != nil { + t.Fatalf("%s: failed to replace with remote transaction: %v", name, err) + } + // Ensure the in-flight limit for B is still in place. + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(1), keyB)); !errors.Is(err, txpool.ErrInflightTxLimitReached) { + t.Fatalf("%s: error mismatch: want %v, have %v", name, txpool.ErrInflightTxLimitReached, err) + } + }, + }, + { + // Since multiple authorizations can be pending simultaneously, replacing + // one of them should not break the one in-flight-transaction limit. name: "track-multiple-conflicting-delegations", pending: 3, run: func(name string) { @@ -2369,19 +2488,6 @@ func TestSetCodeTransactions(t *testing.T) { } }, }, - { - name: "reject-delegation-from-pending-account", - pending: 1, - run: func(name string) { - // Attempt to submit a delegation from an account with a pending tx. - if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1000), keyC)); err != nil { - t.Fatalf("%s: failed to add with remote setcode transaction: %v", name, err) - } - if err, want := pool.addRemoteSync(setCodeTx(0, keyA, []unsignedAuth{{1, keyC}})), ErrAuthorityReserved; !errors.Is(err, want) { - t.Fatalf("%s: error mismatch: want %v, have %v", name, want, err) - } - }, - }, { name: "remove-hash-from-authority-tracker", pending: 10, diff --git a/core/txpool/reserver.go b/core/txpool/reserver.go index 76ead0f3bb..b6ecef9f1a 100644 --- a/core/txpool/reserver.go +++ b/core/txpool/reserver.go @@ -52,22 +52,37 @@ func NewReservationTracker() *ReservationTracker { // NewHandle creates a named handle on the ReservationTracker. The handle // identifies the subpool so ownership of reservations can be determined. -func (r *ReservationTracker) NewHandle(id int) *Reserver { - return &Reserver{r, id} +func (r *ReservationTracker) NewHandle(id int) *ReservationHandle { + return &ReservationHandle{r, id} } -// Reserver is a named handle on ReservationTracker. It is held by subpools to +// Reserver is an interface for creating and releasing owned reservations in the +// ReservationTracker struct, which is shared between subpools. +type Reserver interface { + // Hold attempts to reserve the specified account address for the given pool. + // Returns an error if the account is already reserved. + Hold(addr common.Address) error + + // Release attempts to release the reservation for the specified account. + // Returns an error if the address is not reserved or is reserved by another pool. + Release(addr common.Address) error + + // Has returns a flag indicating if the address has been reserved by a pool + // other than one with the current Reserver handle. + Has(address common.Address) bool +} + +// ReservationHandle is a named handle on ReservationTracker. It is held by subpools to // make reservations for accounts it is tracking. The id is used to determine // which pool owns an address and disallows non-owners to hold or release // addresses it doesn't own. -type Reserver struct { +type ReservationHandle struct { tracker *ReservationTracker id int } -// Hold attempts to reserve the specified account address for the given pool. -// Returns an error if the account is already reserved. -func (h *Reserver) Hold(addr common.Address) error { +// Hold implements the Reserver interface. +func (h *ReservationHandle) Hold(addr common.Address) error { h.tracker.lock.Lock() defer h.tracker.lock.Unlock() @@ -89,9 +104,8 @@ func (h *Reserver) Hold(addr common.Address) error { return nil } -// Release attempts to release the reservation for the specified account. -// Returns an error if the address is not reserved or is reserved by another pool. -func (h *Reserver) Release(addr common.Address) error { +// Release implements the Reserver interface. +func (h *ReservationHandle) Release(addr common.Address) error { h.tracker.lock.Lock() defer h.tracker.lock.Unlock() @@ -114,11 +128,11 @@ func (h *Reserver) Release(addr common.Address) error { return nil } -// Has returns a flag indicating if the address has been reserved or not. -func (h *Reserver) Has(address common.Address) bool { +// Has implements the Reserver interface. +func (h *ReservationHandle) Has(address common.Address) bool { h.tracker.lock.RLock() defer h.tracker.lock.RUnlock() - _, exists := h.tracker.accounts[address] - return exists + id, exists := h.tracker.accounts[address] + return exists && id != h.id } diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index 2cb5103875..8cfc14f164 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -105,7 +105,7 @@ type SubPool interface { // These should not be passed as a constructor argument - nor should the pools // start by themselves - in order to keep multiple subpools in lockstep with // one another. - Init(gasTip uint64, head *types.Header, reserver *Reserver) error + Init(gasTip uint64, head *types.Header, reserver Reserver) error // Close terminates any background processing threads and releases any held // resources. From 2547bb28a4a25969dd677f048155b64394fe59fd Mon Sep 17 00:00:00 2001 From: crStiv Date: Thu, 10 Apr 2025 12:26:35 +0300 Subject: [PATCH 29/35] eth/fetcher: Fix flaky TestTransactionForgotten test using mock clock (#31468) Fixes #31169 The TestTransactionForgotten test was flaky due to real time dependencies. This PR: - Replaces real time with mock clock for deterministic timing control - Adds precise state checks at timeout boundaries - Verifies underpriced cache states and cleanup - Improves test reliability by controlling transaction timestamps - Adds checks for transaction re-enqueueing behavior The changes ensure consistent test behavior without timing-related flakiness. --------- Co-authored-by: Zsolt Felfoldi --- eth/fetcher/tx_fetcher.go | 14 +-- eth/fetcher/tx_fetcher_test.go | 98 ++++++++++++++++++--- tests/fuzzers/txfetcher/txfetcher_fuzzer.go | 7 +- 3 files changed, 99 insertions(+), 20 deletions(-) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 97d1e29862..1c192d4112 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -202,22 +202,23 @@ type TxFetcher struct { fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer dropPeer func(string) // Drops a peer in case of announcement violation - step chan struct{} // Notification channel when the fetcher loop iterates - clock mclock.Clock // Time wrapper to simulate in tests - rand *mrand.Rand // Randomizer to use in tests instead of map range loops (soft-random) + step chan struct{} // Notification channel when the fetcher loop iterates + clock mclock.Clock // Monotonic clock or simulated clock for tests + realTime func() time.Time // Real system time or simulated time for tests + rand *mrand.Rand // Randomizer to use in tests instead of map range loops (soft-random) } // NewTxFetcher creates a transaction fetcher to retrieve transaction // based on hash announcements. func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string)) *TxFetcher { - return NewTxFetcherForTests(hasTx, addTxs, fetchTxs, dropPeer, mclock.System{}, nil) + return NewTxFetcherForTests(hasTx, addTxs, fetchTxs, dropPeer, mclock.System{}, time.Now, nil) } // NewTxFetcherForTests is a testing method to mock out the realtime clock with // a simulated version and the internal randomness with a deterministic one. func NewTxFetcherForTests( hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, dropPeer func(string), - clock mclock.Clock, rand *mrand.Rand) *TxFetcher { + clock mclock.Clock, realTime func() time.Time, rand *mrand.Rand) *TxFetcher { return &TxFetcher{ notify: make(chan *txAnnounce), cleanup: make(chan *txDelivery), @@ -237,6 +238,7 @@ func NewTxFetcherForTests( fetchTxs: fetchTxs, dropPeer: dropPeer, clock: clock, + realTime: realTime, rand: rand, } } @@ -293,7 +295,7 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c // isKnownUnderpriced reports whether a transaction hash was recently found to be underpriced. func (f *TxFetcher) isKnownUnderpriced(hash common.Hash) bool { prevTime, ok := f.underpriced.Peek(hash) - if ok && prevTime.Before(time.Now().Add(-maxTxUnderpricedTimeout)) { + if ok && prevTime.Before(f.realTime().Add(-maxTxUnderpricedTimeout)) { f.underpriced.Remove(hash) return false } diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index 52b3591086..7f3080f5f6 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -2150,9 +2150,22 @@ func containsHashInAnnounces(slice []announce, hash common.Hash) bool { return false } -// Tests that a transaction is forgotten after the timeout. +// TestTransactionForgotten verifies that underpriced transactions are properly +// forgotten after the timeout period, testing both the exact timeout boundary +// and the cleanup of the underpriced cache. func TestTransactionForgotten(t *testing.T) { - fetcher := NewTxFetcher( + // Test ensures that underpriced transactions are properly forgotten after a timeout period, + // including checks for timeout boundary and cache cleanup. + t.Parallel() + + // Create a mock clock for deterministic time control + mockClock := new(mclock.Simulated) + mockTime := func() time.Time { + nanoTime := int64(mockClock.Now()) + return time.Unix(nanoTime/1000000000, nanoTime%1000000000) + } + + fetcher := NewTxFetcherForTests( func(common.Hash) bool { return false }, func(txs []*types.Transaction) []error { errs := make([]error, len(txs)) @@ -2163,24 +2176,83 @@ func TestTransactionForgotten(t *testing.T) { }, func(string, []common.Hash) error { return nil }, func(string) {}, + mockClock, + mockTime, + rand.New(rand.NewSource(0)), // Use fixed seed for deterministic behavior ) fetcher.Start() defer fetcher.Stop() - // Create one TX which is 5 minutes old, and one which is recent - tx1 := types.NewTx(&types.LegacyTx{Nonce: 0}) - tx1.SetTime(time.Now().Add(-maxTxUnderpricedTimeout - 1*time.Second)) - tx2 := types.NewTx(&types.LegacyTx{Nonce: 1}) - // Enqueue both in the fetcher. They will be immediately tagged as underpriced - if err := fetcher.Enqueue("asdf", []*types.Transaction{tx1, tx2}, false); err != nil { + // Create two test transactions with the same timestamp + tx1 := types.NewTransaction(0, common.Address{}, big.NewInt(100), 21000, big.NewInt(1), nil) + tx2 := types.NewTransaction(1, common.Address{}, big.NewInt(100), 21000, big.NewInt(1), nil) + + now := mockTime() + tx1.SetTime(now) + tx2.SetTime(now) + + // Initial state: both transactions should be marked as underpriced + if err := fetcher.Enqueue("peer", []*types.Transaction{tx1, tx2}, false); err != nil { t.Fatal(err) } - // isKnownUnderpriced should trigger removal of the first tx (no longer be known underpriced) - if fetcher.isKnownUnderpriced(tx1.Hash()) { - t.Fatal("transaction should be forgotten by now") + if !fetcher.isKnownUnderpriced(tx1.Hash()) { + t.Error("tx1 should be underpriced") } - // isKnownUnderpriced should not trigger removal of the second if !fetcher.isKnownUnderpriced(tx2.Hash()) { - t.Fatal("transaction should be known underpriced") + t.Error("tx2 should be underpriced") + } + + // Verify cache size + if size := fetcher.underpriced.Len(); size != 2 { + t.Errorf("wrong underpriced cache size: got %d, want %d", size, 2) + } + + // Just before timeout: transactions should still be underpriced + mockClock.Run(maxTxUnderpricedTimeout - time.Second) + if !fetcher.isKnownUnderpriced(tx1.Hash()) { + t.Error("tx1 should still be underpriced before timeout") + } + if !fetcher.isKnownUnderpriced(tx2.Hash()) { + t.Error("tx2 should still be underpriced before timeout") + } + + // Exactly at timeout boundary: transactions should still be present + mockClock.Run(time.Second) + if !fetcher.isKnownUnderpriced(tx1.Hash()) { + t.Error("tx1 should be present exactly at timeout") + } + if !fetcher.isKnownUnderpriced(tx2.Hash()) { + t.Error("tx2 should be present exactly at timeout") + } + + // After timeout: transactions should be forgotten + mockClock.Run(time.Second) + if fetcher.isKnownUnderpriced(tx1.Hash()) { + t.Error("tx1 should be forgotten after timeout") + } + if fetcher.isKnownUnderpriced(tx2.Hash()) { + t.Error("tx2 should be forgotten after timeout") + } + + // Verify cache is empty + if size := fetcher.underpriced.Len(); size != 0 { + t.Errorf("wrong underpriced cache size after timeout: got %d, want 0", size) + } + + // Re-enqueue tx1 with updated timestamp + tx1.SetTime(mockTime()) + if err := fetcher.Enqueue("peer", []*types.Transaction{tx1}, false); err != nil { + t.Fatal(err) + } + if !fetcher.isKnownUnderpriced(tx1.Hash()) { + t.Error("tx1 should be underpriced after re-enqueueing with new timestamp") + } + if fetcher.isKnownUnderpriced(tx2.Hash()) { + t.Error("tx2 should remain forgotten") + } + + // Verify final cache state + if size := fetcher.underpriced.Len(); size != 1 { + t.Errorf("wrong final underpriced cache size: got %d, want 1", size) } } diff --git a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go index 51f2fc3b4d..c136253a62 100644 --- a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go +++ b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go @@ -84,7 +84,12 @@ func fuzz(input []byte) int { }, func(string, []common.Hash) error { return nil }, nil, - clock, rand, + clock, + func() time.Time { + nanoTime := int64(clock.Now()) + return time.Unix(nanoTime/1000000000, nanoTime%1000000000) + }, + rand, ) f.Start() defer f.Stop() From 4906c9911316c52ccd20e122d1e092abe3c7d6ca Mon Sep 17 00:00:00 2001 From: HackyMiner Date: Thu, 10 Apr 2025 19:46:54 +0900 Subject: [PATCH 30/35] accounts/usbwallet: full 32bit chainId support for Trezor (#17439) This fix allows Trezor to support full 32bit chainId in geth, with the next version of firmware. For `chainId > 2147483630` case, Trezor returns signature bit only. - Trezor returns only signature parity for `chainId > 2147483630` case. - for `chainId == 2147483630` case, Trezor returns `MAX_UINT32` or `0`, but it doesn't matter. (`2147483630 * 2 + 35` = `4294967295`(`MAX_UINT32`)) chainId | returned signature_v | compatible issue ---------|------------------------|-------------------- 0 < chainId <= 255 | chainId * 2 + 35 + v | no issue (firmware `1.6.2` for Trezor one) 255 < chainId <= 2147483630 | chainId * 2 + 35 + v | ***fixed.*** *firmware `1.6.3`* chainId > 2147483630 | v | *firmware `1.6.3`* Please see also: full 32bit chainId support for Trezor - Trezor one: https://github.com/trezor/trezor-mcu/pull/399 ***merged*** - Trezor model T: https://github.com/trezor/trezor-core/pull/311 ***merged*** --------- Signed-off-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Co-authored-by: Guillaume Ballet <3272758+gballet@users.noreply.github.com> --- accounts/usbwallet/trezor.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/accounts/usbwallet/trezor.go b/accounts/usbwallet/trezor.go index 1c4270d255..d4862d161b 100644 --- a/accounts/usbwallet/trezor.go +++ b/accounts/usbwallet/trezor.go @@ -25,6 +25,7 @@ import ( "errors" "fmt" "io" + "math" "math/big" "github.com/ethereum/go-ethereum/accounts" @@ -249,7 +250,11 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction } } // Extract the Ethereum signature and do a sanity validation - if len(response.GetSignatureR()) == 0 || len(response.GetSignatureS()) == 0 || response.GetSignatureV() == 0 { + if len(response.GetSignatureR()) == 0 || len(response.GetSignatureS()) == 0 { + return common.Address{}, nil, errors.New("reply lacks signature") + } else if response.GetSignatureV() == 0 && int(chainID.Int64()) <= (math.MaxUint32-36)/2 { + // for chainId >= (MaxUint32-36)/2, Trezor returns signature bit only + // https://github.com/trezor/trezor-mcu/pull/399 return common.Address{}, nil, errors.New("reply lacks signature") } signature := append(append(response.GetSignatureR(), response.GetSignatureS()...), byte(response.GetSignatureV())) @@ -261,7 +266,11 @@ func (w *trezorDriver) trezorSign(derivationPath []uint32, tx *types.Transaction } else { // Trezor backend does not support typed transactions yet. signer = types.NewEIP155Signer(chainID) - signature[64] -= byte(chainID.Uint64()*2 + 35) + // if chainId is above (MaxUint32 - 36) / 2 then the final v values is returned + // directly. Otherwise, the returned value is 35 + chainid * 2. + if signature[64] > 1 && int(chainID.Int64()) <= (math.MaxUint32-36)/2 { + signature[64] -= byte(chainID.Uint64()*2 + 35) + } } // Inject the final signature into the transaction and sanity check the sender From 9b4eab6a29704f55fa7b4b92e296094f0dbcee22 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Thu, 10 Apr 2025 19:49:54 +0800 Subject: [PATCH 31/35] eth/catalyst: in tests, manually sync txpool after initial chain insertion to prevent race between txpool head reset and promotion of txs that will be subsequently added (#31595) before this changes, this will result in numerous test failures: ``` > go test -run=Eth2AssembleBlock -c > stress ./catalyst.test ``` The reason is that after creating/inserting the test chain, there is a race between the txpool head reset and the promotion of txs added from tests. Ensuring that the txpool state is up to date with the head of the chain before proceeding fixes these flaky tests. --- eth/catalyst/api_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index e91e07d05d..cb6ae053b6 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -447,6 +447,9 @@ func startEthService(t *testing.T, genesis *core.Genesis, blocks []*types.Block) n.Close() t.Fatal("can't import test blocks:", err) } + if err := ethservice.TxPool().Sync(); err != nil { + t.Fatal("failed to sync txpool after initial blockchain import:", err) + } ethservice.SetSynced() return n, ethservice From f64aa6eaf7a021226648aeae5ceef800fb9b66d0 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Fri, 11 Apr 2025 00:21:32 +0800 Subject: [PATCH 32/35] internal/testlog: fix log output from sub-loggers (#31539) When we instantiate a sub-logger via `go-ethereum/internal/testlog/logger.With`, we copy the reference to the `bufHandler` from the parent logger. However, internally, `go-ethereum/internal/testlog/logger.With` calls `log/slog/Logger.With` which creates a new handler instance (via `internal/bufHandler.WithAttrs`). This PR modifies sub-logger instantiation to use the newly-instantiated handler, instead of copying the reference from the parent instance. The type cast from `slog.Handler` to `*bufHandler` in `internal/testlog/Logger.With` is safe here because a `internal/testlog/Logger` can only be instantiated with a `*bufHandler` as the underlying handler type. Note, that I've also removed a pre-existing method that broke the above assumption. However, this method is not used in our codebase. I'm not sure if the assumption holds for forks of geth (e.g. optimism has modified the testlogger somewhat allowing test loggers to accept arbitrary handler types), but it seems okay to break API compatibility given that this is in the `internal` package. closes https://github.com/ethereum/go-ethereum/issues/31533 --- internal/testlog/testlog.go | 28 +++++++------ internal/testlog/testlog_test.go | 67 ++++++++++++++++++++++++++++++++ tests/testdata | 2 +- 3 files changed, 81 insertions(+), 16 deletions(-) create mode 100644 internal/testlog/testlog_test.go diff --git a/internal/testlog/testlog.go b/internal/testlog/testlog.go index ad61af9eac..8a3ea85438 100644 --- a/internal/testlog/testlog.go +++ b/internal/testlog/testlog.go @@ -23,7 +23,6 @@ import ( "fmt" "log/slog" "sync" - "testing" "github.com/ethereum/go-ethereum/log" ) @@ -32,12 +31,21 @@ const ( termTimeFormat = "01-02|15:04:05.000" ) +// T wraps methods from testing.T used by the test logger into an interface. +// It is specified so that unit tests can instantiate the logger with an +// implementation of T which can capture the output of logging statements +// from T.Logf, as this cannot be using testing.T. +type T interface { + Logf(format string, args ...any) + Helper() +} + // logger implements log.Logger such that all output goes to the unit test log via // t.Logf(). All methods in between logger.Trace, logger.Debug, etc. are marked as test // helpers, so the file and line number in unit test output correspond to the call site // which emitted the log message. type logger struct { - t *testing.T + t T l log.Logger mu *sync.Mutex h *bufHandler @@ -78,7 +86,7 @@ func (h *bufHandler) WithGroup(_ string) slog.Handler { } // Logger returns a logger which logs to the unit test log of t. -func Logger(t *testing.T, level slog.Level) log.Logger { +func Logger(t T, level slog.Level) log.Logger { handler := bufHandler{ buf: []slog.Record{}, attrs: []slog.Attr{}, @@ -92,17 +100,6 @@ func Logger(t *testing.T, level slog.Level) log.Logger { } } -// LoggerWithHandler returns -func LoggerWithHandler(t *testing.T, handler slog.Handler) log.Logger { - var bh bufHandler - return &logger{ - t: t, - l: log.NewLogger(handler), - mu: new(sync.Mutex), - h: &bh, - } -} - func (l *logger) Handler() slog.Handler { return l.l.Handler() } @@ -170,7 +167,8 @@ func (l *logger) Crit(msg string, ctx ...interface{}) { } func (l *logger) With(ctx ...interface{}) log.Logger { - return &logger{l.t, l.l.With(ctx...), l.mu, l.h} + newLogger := l.l.With(ctx...) + return &logger{l.t, newLogger, l.mu, newLogger.Handler().(*bufHandler)} } func (l *logger) New(ctx ...interface{}) log.Logger { diff --git a/internal/testlog/testlog_test.go b/internal/testlog/testlog_test.go new file mode 100644 index 0000000000..7789a47270 --- /dev/null +++ b/internal/testlog/testlog_test.go @@ -0,0 +1,67 @@ +package testlog + +import ( + "bytes" + "fmt" + "io" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/log" +) + +type mockT struct { + out io.Writer +} + +func (t *mockT) Helper() { + // noop for the purposes of unit tests +} + +func (t *mockT) Logf(format string, args ...any) { + // we could gate this operation in a mutex, but because testlogger + // only calls Logf with its internal mutex held, we just write output here + var lineBuf bytes.Buffer + if _, err := fmt.Fprintf(&lineBuf, format, args...); err != nil { + panic(err) + } + // The timestamp is locale-dependent, so we want to trim that off + // "INFO [01-01|00:00:00.000] a message ..." -> "a message..." + sanitized := strings.Split(lineBuf.String(), "]")[1] + if _, err := t.out.Write([]byte(sanitized)); err != nil { + panic(err) + } +} + +func TestLogging(t *testing.T) { + tests := []struct { + name string + expected string + run func(t *mockT) + }{ + { + "SubLogger", + ` Visible + Hide and seek foobar=123 + Also visible +`, + func(t *mockT) { + l := Logger(t, log.LevelInfo) + subLogger := l.New("foobar", 123) + + l.Info("Visible") + subLogger.Info("Hide and seek") + l.Info("Also visible") + }, + }, + } + + for _, tc := range tests { + outp := bytes.Buffer{} + mock := mockT{&outp} + tc.run(&mock) + if outp.String() != tc.expected { + fmt.Printf("output mismatch.\nwant: '%s'\ngot: '%s'\n", tc.expected, outp.String()) + } + } +} diff --git a/tests/testdata b/tests/testdata index 81862e4848..faf33b4714 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 81862e4848585a438d64f911a19b3825f0f4cd95 +Subproject commit faf33b471465d3c6cdc3d04fbd690895f78d33f2 From a9444ea312eb4077a470cdb9b394ec0544cf116b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 10 Apr 2025 23:54:44 +0200 Subject: [PATCH 33/35] tests/testdata: revert to v17.0 The submodule was accidentally updated to another commit by f64aa6eaf7. --- tests/testdata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testdata b/tests/testdata index faf33b4714..81862e4848 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit faf33b471465d3c6cdc3d04fbd690895f78d33f2 +Subproject commit 81862e4848585a438d64f911a19b3825f0f4cd95 From 0059b0ab6676232dd3809baa664d1e5b0d5e85ab Mon Sep 17 00:00:00 2001 From: levisyin Date: Fri, 11 Apr 2025 17:28:14 +0800 Subject: [PATCH 34/35] build: upgrade -dlgo version to Go 1.24.2 (#31538) --- build/checksums.txt | 94 ++++++++++++++++++++++----------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index 6d3b718c3c..a914d79860 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -5,54 +5,54 @@ # https://github.com/ethereum/execution-spec-tests/releases/download/pectra-devnet-6%40v1.0.0/ b69211752a3029083c020dc635fe12156ca1a6725a08559da540a0337586a77e fixtures_pectra-devnet-6.tar.gz -# version:golang 1.24.1 +# version:golang 1.24.2 # https://go.dev/dl/ -8244ebf46c65607db10222b5806aeb31c1fcf8979c1b6b12f60c677e9a3c0656 go1.24.1.src.tar.gz -8d627dc163a4bffa2b1887112ad6194af175dce108d606ed1714a089fb806033 go1.24.1.aix-ppc64.tar.gz -addbfce2056744962e2d7436313ab93486660cf7a2e066d171b9d6f2da7c7abe go1.24.1.darwin-amd64.tar.gz -58d529334561cff11087cd4ab18fe0b46d8d5aad88f45c02b9645f847e014512 go1.24.1.darwin-amd64.pkg -295581b5619acc92f5106e5bcb05c51869337eb19742fdfa6c8346c18e78ff88 go1.24.1.darwin-arm64.tar.gz -78b0fc8ddc344eb499f1a952c687cb84cbd28ba2b739cfa0d4eb042f07e44e82 go1.24.1.darwin-arm64.pkg -e70053f56f7eb93806d80cbd5726f78509a0a467602f7bea0e2c4ee8ed7c3968 go1.24.1.dragonfly-amd64.tar.gz -3595e2674ed8fe72e604ca59c964d3e5277aafb08475c2b1aaca2d2fd69c24fc go1.24.1.freebsd-386.tar.gz -47d7de8bb64d5c3ee7b6723aa62d5ecb11e3568ef2249bbe1d4bbd432d37c00c go1.24.1.freebsd-amd64.tar.gz -04eec3bcfaa14c1370cdf98e8307fac7e4853496c3045afb9c3124a29cbca205 go1.24.1.freebsd-arm.tar.gz -51aa70146e40cfdc20927424083dc86e6223f85dc08089913a1651973b55665b go1.24.1.freebsd-arm64.tar.gz -3c131d8e3fc285a1340f87813153e24226d3ddbd6e54f3facbd6e4c46a84655e go1.24.1.freebsd-riscv64.tar.gz -201d09da737ba39d5367f87d4e8b31edaeeb3dc9b9c407cb8cfb40f90c5a727a go1.24.1.illumos-amd64.tar.gz -8c530ecedbc17e42ce10177bea07ccc96a3e77c792ea1ea72173a9675d16ffa5 go1.24.1.linux-386.tar.gz -cb2396bae64183cdccf81a9a6df0aea3bce9511fc21469fb89a0c00470088073 go1.24.1.linux-amd64.tar.gz -8df5750ffc0281017fb6070fba450f5d22b600a02081dceef47966ffaf36a3af go1.24.1.linux-arm64.tar.gz -6d95f8d7884bfe2364644c837f080f2b585903d0b771eb5b06044e226a4f120a go1.24.1.linux-armv6l.tar.gz -19304a4a56e46d04604547d2d83235dc4f9b192c79832560ce337d26cc7b835a go1.24.1.linux-loong64.tar.gz -6347be77fa5359c12a5308c8ab87147c1fc4717b0c216493d1706c3b9fcde22d go1.24.1.linux-mips.tar.gz -1647df415f7030b82d4105670192aa7e8910e18563bb0d505192d72800cc2d21 go1.24.1.linux-mips64.tar.gz -762da594e4ec0f9cf6defae6ef971f5f7901203ee6a2d979e317adec96657317 go1.24.1.linux-mips64le.tar.gz -9d8133c7b23a557399fab870b5cf464079c2b623a43b214a7567cf11c254a444 go1.24.1.linux-mipsle.tar.gz -132f10999abbaccbada47fa85462db30c423955913b14d6c692de25f4636c766 go1.24.1.linux-ppc64.tar.gz -0fb522efcefabae6e37e69bdc444094e75bfe824ea6d4cc3cbc70c7ae1b16858 go1.24.1.linux-ppc64le.tar.gz -eaef4323d5467ff97fb1979c8491764060dade19f02f3275a9313f9a0da3b9c0 go1.24.1.linux-riscv64.tar.gz -6c05e14d8f11094cb56a1c50f390b6b658bed8a7cbd8d1a57e926581b7eabfce go1.24.1.linux-s390x.tar.gz -5dbb287d343ea00d58a70b11629f32ee716dc50a6875c459ea2018df0f294cd8 go1.24.1.netbsd-386.tar.gz -617aa3faee50ce84c343db0888e9a210c310a7203666b4ed620f31030c9fb32f go1.24.1.netbsd-amd64.tar.gz -59a928b7080c4a6ac985946274b7c65ce1cecc0b468ecd992d17b7c12fab9296 go1.24.1.netbsd-arm.tar.gz -28daa8d0feb4aef2af60cefa3305bb9314de7e8a05cbca41ac548964cdfe89b7 go1.24.1.netbsd-arm64.tar.gz -b7382b2f5d99813aeac14db482faa3bfbd47a68880b607fa2a7e669e26bab9cd go1.24.1.openbsd-386.tar.gz -2513b6537c45deead5e641c7ce7502913e7d5e6f0b21c52542fb11f81578690f go1.24.1.openbsd-amd64.tar.gz -853c1917d4fc7b144c55a02842aa48542d5cc798dde8db96dc0fdbc263200e04 go1.24.1.openbsd-arm.tar.gz -6bc207b91e6f6ae3347fb54616a8fb2f5c11983713846a4cef111ff3f4f94d14 go1.24.1.openbsd-arm64.tar.gz -4279260e2f2b94ee94e81470d13db7367f4393b061fee60985528fa0fa430df4 go1.24.1.openbsd-ppc64.tar.gz -6fc4023a0a339ee0778522364a127d94c78e62122288d47d820dba703f81dc07 go1.24.1.openbsd-riscv64.tar.gz -b5eb9fafd77146e7e1f748acfd95559580ecc8d2f15abf432a20f58c929c7cd2 go1.24.1.plan9-386.tar.gz -24dcad6361b141fc8cced15b092351e12a99d2e58d7013204a3013c50daf9fdd go1.24.1.plan9-amd64.tar.gz -a026ac3b55aa1e6fdc2aaab30207a117eafbe965ed81d3aa0676409f280ddc37 go1.24.1.plan9-arm.tar.gz -8e4f6a77388dc6e5aa481efd5abdb3b9f5c9463bb82f4db074494e04e5c84992 go1.24.1.solaris-amd64.tar.gz -b799f4ab264eef12a014c759383ed934056608c483e0f73e34ea6caf9f1df5f9 go1.24.1.windows-386.zip -db128981033ac82a64688a123f631e61297b6b8f52ca913145e57caa8ce94cc3 go1.24.1.windows-386.msi -95666b551453209a2b8869d29d177285ff9573af10f085d961d7ae5440f645ce go1.24.1.windows-amd64.zip -5968e7adcf26e68a54f1cd41ad561275a670a8e2ca5263bc375b524638557dfb go1.24.1.windows-amd64.msi -e28c4e6d0b913955765b46157ab88ae59bb636acaa12d7bec959aa6900f1cebd go1.24.1.windows-arm64.zip -6d352c1f154a102a5b90c480cc64bab205ccf2681e34e78a3a4d3f1ddfbc81e4 go1.24.1.windows-arm64.msi +9dc77ffadc16d837a1bf32d99c624cb4df0647cee7b119edd9e7b1bcc05f2e00 go1.24.2.src.tar.gz +427b373540d8fd51dbcc46bdecd340af109cd41514443c000d3dcde72b2c65a3 go1.24.2.aix-ppc64.tar.gz +238d9c065d09ff6af229d2e3b8b5e85e688318d69f4006fb85a96e41c216ea83 go1.24.2.darwin-amd64.tar.gz +535ed9ff283fee39575a7fb9b6d8b1901b6dc640d06dc71fd7d3faeefdaf8030 go1.24.2.darwin-amd64.pkg +b70f8b3c5b4ccb0ad4ffa5ee91cd38075df20fdbd953a1daedd47f50fbcff47a go1.24.2.darwin-arm64.tar.gz +4732f607a47ce4d898c0af01ff68f07e0820a6b50603aef5d5c777d1102505e2 go1.24.2.darwin-arm64.pkg +c17686b5fd61a663fbfafccfa177961be59386cf294e935ce35866b9dcb8e78a go1.24.2.dragonfly-amd64.tar.gz +026f1dd906189acff714c7625686bbc4ed91042618ba010d45b671461acc9e63 go1.24.2.freebsd-386.tar.gz +49399ba759b570a8f87d12179133403da6c2dd296d63a8830dee309161b9c40c go1.24.2.freebsd-amd64.tar.gz +1f48f47183794d97c29736004247ab541177cf984ac6322c78bc43828daa1172 go1.24.2.freebsd-arm.tar.gz +ef856428b60a8c0bd9a2cba596e83024be6f1c2d5574e89cb1ff2262b08df8b9 go1.24.2.freebsd-arm64.tar.gz +ec2088823e16df00600a6d0f72e9a7dc6d2f80c9c140c2043c0cf20e1404d1a9 go1.24.2.freebsd-riscv64.tar.gz +e030e7cedbb8688f1d75cb80f3de6ee2e6617a67d34051e794e5992b53462147 go1.24.2.illumos-amd64.tar.gz +4c382776d52313266f3026236297a224a6688751256a2dffa3f524d8d6f6c0ba go1.24.2.linux-386.tar.gz +68097bd680839cbc9d464a0edce4f7c333975e27a90246890e9f1078c7e702ad go1.24.2.linux-amd64.tar.gz +756274ea4b68fa5535eb9fe2559889287d725a8da63c6aae4d5f23778c229f4b go1.24.2.linux-arm64.tar.gz +438d5d3d7dcb239b58d893a715672eabe670b9730b1fd1c8fc858a46722a598a go1.24.2.linux-armv6l.tar.gz +6aefd3bf59c3c5592eda4fb287322207f119c2210f3795afa9be48d3ccb73e1b go1.24.2.linux-loong64.tar.gz +93e49bb4692783b0e9a2deab9558c6e8d2867f35592aeff285adda60924167f3 go1.24.2.linux-mips.tar.gz +6e86e703675016f3faf6604b8f68f20dc1bba75849136e6dd4f43f69c8a4a9d9 go1.24.2.linux-mips64.tar.gz +f233d237538ca1559a7d7cf519a29f0147923a951377bc4e467af4c059e68851 go1.24.2.linux-mips64le.tar.gz +545e1b9a7939f923fd53bde98334b987ef42eb353ee3e0bfede8aa06079d6b24 go1.24.2.linux-mipsle.tar.gz +6eab31481f2f46187bc1b6c887662eef06fc9d7271a8390854072cdb387c8d74 go1.24.2.linux-ppc64.tar.gz +5fff857791d541c71d8ea0171c73f6f99770d15ff7e2ad979104856d01f36563 go1.24.2.linux-ppc64le.tar.gz +91bda1558fcbd1c92769ad86c8f5cf796f8c67b0d9d9c19f76eecfc75ce71527 go1.24.2.linux-riscv64.tar.gz +1cb3448166d6abb515a85a3ee5afbdf932081fb58ad7143a8fb666fbc06146d9 go1.24.2.linux-s390x.tar.gz +a9a2c0db2e826f20f00b02bee01dfdaeb49591c2f6ffacb78dc64a950894f7ff go1.24.2.netbsd-386.tar.gz +cd1a35b76ed9c7b6c0c1616741bd319699a77867ade0be9924f32496c0a87a3f go1.24.2.netbsd-amd64.tar.gz +8c666388d066e479155cc5116950eeb435df28087ef277c18f1dc7479f836e60 go1.24.2.netbsd-arm.tar.gz +5d42f0be04f58da5be788a1e260f8747c316b8ce182bf0b273c2e4c691feaa1a go1.24.2.netbsd-arm64.tar.gz +688effa23ea3973cc8b0fdf4246712cbeef55ff20c45f3a9e28b0c2db04246cf go1.24.2.openbsd-386.tar.gz +e5daf95f1048d8026b1366450a3f8044d668b0639db6422ad9a83755c6745cf7 go1.24.2.openbsd-amd64.tar.gz +aeadaf74bd544d1a12ba9b14c0e7cdb1964de3ba9a52acb4619e91dbae7def7b go1.24.2.openbsd-arm.tar.gz +9e222d9adb0ce836a5b3c8d5aadbd167c8869c030b113f4a81aa88e9a200f279 go1.24.2.openbsd-arm64.tar.gz +192fffa34536adc3cd1bb7c1ee785b8bc156ae7afd10bbf5db99ec8f2e93066e go1.24.2.openbsd-ppc64.tar.gz +a23e90b451a390549042c2a7efbec6f29ed98b2d5618c8d2a35704e21be96e09 go1.24.2.openbsd-riscv64.tar.gz +5cdcafe455d859b02779611a5a1e1d63e498b922e05818fb3debe410a5959e9e go1.24.2.plan9-386.tar.gz +81351659804fa505c1b3ec6fdf9599f7f88df08614307eeb96071bf5e2e74beb go1.24.2.plan9-amd64.tar.gz +6e337d5def14ed0123423c1c32e2e6d8b19161e5d5ffaa7356dad48ee0fd80b4 go1.24.2.plan9-arm.tar.gz +07e6926ebc476c044d7d5b17706abfc52be52bccc2073d1734174efe63c6b35e go1.24.2.solaris-amd64.tar.gz +13d86cb818bba331da75fcd18246ab31a1067b44fb4a243b6dfd93097eda7f37 go1.24.2.windows-386.zip +8a702d9f7104a15bd935f4191c58c24c0b6389e066b9d5661b93915114a2bef0 go1.24.2.windows-386.msi +29c553aabee0743e2ffa3e9fa0cda00ef3b3cc4ff0bc92007f31f80fd69892e1 go1.24.2.windows-amd64.zip +acefb191e72fea0bdb1a3f5f8f6f5ab18b42b3bbce0c7183f189f25953aff275 go1.24.2.windows-amd64.msi +ab267f7f9a3366d48d7664be9e627ce3e63273231430cce5f7783fb910f14148 go1.24.2.windows-arm64.zip +d187bfe539356c39573d2f46766d1d08122b4f33da00fd14d12485fa9e241ff5 go1.24.2.windows-arm64.msi # version:golangci 2.0.2 # https://github.com/golangci/golangci-lint/releases/ From f8ff24e8cfa95c7a1b55857dd5306a9d3b4e50ca Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 11 Apr 2025 11:31:16 +0200 Subject: [PATCH 35/35] version: release go-ethereum v1.15.8 stable --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index 72fd903a1f..945b3b58a2 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 15 // Minor version component of the current release - Patch = 8 // Patch version component of the current release - Meta = "unstable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 15 // Minor version component of the current release + Patch = 8 // Patch version component of the current release + Meta = "stable" // Version metadata to append to the version string )